Gulum
aa473b764e
Der Herkunfts-Verlauf nennt jetzt auch, wenn ein Wert aus einer Quelle VERWORFEN wurde — mit Grund und was stattdessen verwendet wurde (⚠-Markierung), z. B.: - „⚠ Vater-Kandidat ‚Danielle' (*04.03.2020) verworfen — falsches Geschlecht für die Vaterrolle; ‚Hagrid Rubeus' verwendet." (Molly) - „⚠ Geschlecht weiblich aus ‚Wurfchronik.docx' verworfen — abweichend; männlich aus ‚Alberto Kids.xlsx' verwendet (Mehrheit)." (Solice) - „⚠ Vater ‚Jasper Jacob' (*2018) verworfen — unplausibel (7 Jahre älter); kein Ersatz." / DOB-Remap: „⚠ Geburtsdatum 22.08.2018 verworfen — per manueller Entscheidung korrigiert; 14.07.2019 verwendet." (Kazuya) Import: generischer _format_discard()-Helfer; Verwerfungen aus Mehrheitsentscheid, pick_parent_ref-Ablehnungen (explain_pick_rejections), Rollen-Normalisierung, Eltern-Alter-Sanity-Check und DOB-Remaps werden über _discarded/_prov_args an die Tiere gehängt; Gerbil-Provenance wird in einem finalen Pass gebaut (nachdem alle Verwerfungen feststehen). Frontend hebt ⚠-Zeilen ab (+ Legende). Tests grün (python merge/extract, vitest 129, playwright 8, dotnet 212). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1199 lines
53 KiB
Python
1199 lines
53 KiB
Python
#!/usr/bin/env python3
|
||
"""FEAT-8b stages 1-2 — extract + dedup the GerbilManager source spreadsheets.
|
||
|
||
Stage 1: parse the 10 Stammbaum pedigree charts and the Wurfchronik litter
|
||
chronicle into normalised animals.json + litters.json, and extract
|
||
embedded photos (anchor-mapped to animals).
|
||
Stage 2: dedup animals (key = normalise(name)+DOB, corroborated by DOB+genotype)
|
||
and emit a German-friendly review-report.md for Julian's wife. No DB load.
|
||
|
||
Re-runnable per file (later Wurfchronik "Teil2+" / more charts just re-run).
|
||
Migration tooling — Python, not product code. Zero third-party deps.
|
||
|
||
See FEAT-8a-format-spec.md (in Pam's hive workspace) for the format analysis.
|
||
"""
|
||
import os
|
||
import re
|
||
import sys
|
||
import json
|
||
import glob
|
||
import shutil
|
||
import argparse
|
||
import unicodedata
|
||
from collections import Counter
|
||
|
||
import xlsx_util as xu
|
||
import genotype as gt
|
||
|
||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||
DEFAULT_STAMMBAEUME = r"C:\Users\gulum\dev\Sttammbäume"
|
||
DEFAULT_WURFCHRONIK = r"C:\Users\gulum\dev\Wurfchronik der Kleine Chaoten Teil1.xlsx"
|
||
OUT = os.path.join(HERE, "output")
|
||
|
||
DOB = re.compile(r"\*\s?(\d{1,2}\.\d{1,2}\.(?:\d{4}|\d{2}))")
|
||
DEATH = re.compile(r"\+\s?(\d{1,2}\.\d{1,2}\.(?:\d{4}|\d{2})|\d{4})")
|
||
|
||
|
||
# ---------------------------------------------------------------- helpers ----
|
||
def gen_of(colnum, offset=0):
|
||
"""Map a column number to a generation band (0=proband ... 5=deepest)."""
|
||
effective_col = colnum - offset
|
||
if effective_col <= 3:
|
||
return 0 # Column B (2) -> proband
|
||
if effective_col <= 6:
|
||
return 1 # Column E (5) -> parents
|
||
if effective_col <= 9:
|
||
return 2 # Column H (8) -> grandparents
|
||
if effective_col <= 12:
|
||
return 3 # Column K (11) -> great-grandparents
|
||
if effective_col <= 15:
|
||
return 4 # Column N (14) -> gg-grandparents
|
||
return 5 # Column Q (17) or deeper -> ggg-grandparents
|
||
|
||
|
||
def norm_name(name):
|
||
if not name:
|
||
return ""
|
||
n = name.lower()
|
||
n = re.sub(r"\[.*?\]", " ", n) # drop [line] tags (Wurfchronik)
|
||
n = re.sub(r"\bgen\.\b", " ", n) # "gen." nickname marker
|
||
n = re.sub(r"\bv\.\s?d\.\b", " von den ", n)
|
||
n = re.sub(r"\b(von der|von den|von|of)\b", " ", n) # cattery/line connectors
|
||
n = unicodedata.normalize("NFKD", n)
|
||
n = re.sub(r"[^a-z0-9äöüß]", "", n)
|
||
return n
|
||
|
||
|
||
def norm_dob(d):
|
||
if not d:
|
||
return ""
|
||
p = d.split(".")
|
||
if len(p) == 3 and len(p[2]) == 2:
|
||
p[2] = "20" + p[2]
|
||
return ".".join(x.zfill(2) if i < 2 else x for i, x in enumerate(p))
|
||
|
||
|
||
def slug(name, dob):
|
||
base = norm_name(name) or "unbekannt"
|
||
d = norm_dob(dob).replace(".", "")
|
||
return (base[:40] + ("-" + d if d else "")) or "unbekannt"
|
||
|
||
|
||
def clean_name(raw):
|
||
"""Strip detail/markers from a name cell, keep the human name + [line]."""
|
||
n = raw.strip().strip(",").strip()
|
||
return n
|
||
|
||
|
||
# --- (name, Zucht) canonicalisation -------------------------------------------
|
||
# Julian (FEAT-8 ruling): the [brackets] in Wurfchronik names ARE the Zucht
|
||
# (breeding line) and are equivalent to the Stammbaum "of/von <line>" suffix.
|
||
# Both fold into one canonical (call-name, zucht) pair; the Zucht acts as a
|
||
# dedup DISCRIMINATOR (same name+DOB but different Zucht = different animal).
|
||
_BRACKET_ZUCHT = re.compile(r"^(.*?)\s*\[([^\]]+)\]\s*$")
|
||
_SUFFIX_ZUCHT = re.compile(
|
||
r"^(.+?)\s+(?:of|von\s+den|von\s+der|v\.\s?d\.|von)\s+(.+)$", re.IGNORECASE)
|
||
# canonical values are post-norm_zucht (word-final 'n' folded: "kleinechaote")
|
||
ZUCHT_ALIASES = {
|
||
"zdkc": "kleinechaote", # "Zucht der kleinen Chaoten" (home cattery shorthand)
|
||
}
|
||
|
||
# A Farbschlag value must NOT contain cattery/line connectors (v.d./von/of/gen.) — when it
|
||
# does, a parent's NAME has bled into the Farbschlag cell (cross-cell chart read, PEDIGREE-LINK
|
||
# bug: e.g. "Victoria Welby gen. Welby v.d. Kleinen Chaoten" became a Farbschlag variant and
|
||
# spawned a phantom conflict). Reject such values so they don't pollute farbschlag/conflicts.
|
||
_NAME_MARKER = re.compile(r"\bv\.\s?d\.|\bvon\b|\bof\b|\bgen\.", re.IGNORECASE)
|
||
|
||
|
||
def looks_like_animal_name(text):
|
||
"""True if a candidate Farbschlag cell actually looks like an animal name (has a
|
||
cattery/line connector). Real Farbschläge are short colour words without these."""
|
||
return bool(_NAME_MARKER.search(text or ""))
|
||
|
||
|
||
def split_name_zucht(raw):
|
||
"""'Luna [ZdkC]' -> ('Luna','ZdkC'); 'Pikachu of Black Forest' ->
|
||
('Pikachu','Black Forest'); plain names -> (name, '')."""
|
||
n = (raw or "").strip()
|
||
m = _BRACKET_ZUCHT.match(n)
|
||
if m:
|
||
return m.group(1).strip(), m.group(2).strip()
|
||
m = _SUFFIX_ZUCHT.match(n)
|
||
if m:
|
||
return m.group(1).strip(), m.group(2).strip()
|
||
return n, ""
|
||
|
||
|
||
def norm_zucht(z):
|
||
"""Canonical Zucht key: drops Zucht/von/der fillers, folds declension
|
||
('kleinen Chaoten' == 'kleine Chaoten'), resolves known shorthands."""
|
||
if not z:
|
||
return ""
|
||
n = z.lower()
|
||
# TOLERANT KC-MATCHER FIX: trailing \b after '.' never fires when next char is ' '
|
||
# (both are non-word chars), so "v.d. kleinen" was NOT stripped. Drop the trailing \b.
|
||
n = re.sub(r"\bv\.\s?d\.", " ", n)
|
||
n = re.sub(r"\b(zucht|privatzucht|der|die|den|des|dem|von|of)\b", " ", n)
|
||
n = re.sub(r"[^a-z0-9äöüß ]", " ", n)
|
||
words = [w[:-1] if len(w) > 4 and w.endswith("n") else w for w in n.split()]
|
||
key = "".join(words)
|
||
return ZUCHT_ALIASES.get(key, key)
|
||
|
||
|
||
def is_clan_zucht(z):
|
||
"""True if the zucht name (raw or canonical) identifies the Kleine Chaoten home cattery.
|
||
|
||
Accepts all known spellings: 'Zucht der Kleinen Chaoten', 'kleinen Chaoten',
|
||
'v.d. Kleinen Chaoten', '[ZdkC]', 'kleinechaote', etc.
|
||
"""
|
||
return norm_zucht(z) == "kleinechaote"
|
||
|
||
|
||
def canon_pair(raw):
|
||
"""Full raw name -> (normalised call-name, canonical zucht)."""
|
||
name, zucht = split_name_zucht(raw)
|
||
return norm_name(name), norm_zucht(zucht)
|
||
|
||
|
||
# --------------------------------------------------- Stammbaum extraction ----
|
||
def parse_detail(text):
|
||
"""From a string that contains *DOB and/or genotype, pull (dob, death, geno_str).
|
||
|
||
For compact lines ("Name,*DOB[/+death], genotype") the genotype is everything
|
||
after the date — we must NOT scan from the first locus-looking letter, or stray
|
||
name words ("den", "of") get swallowed as genotype tokens.
|
||
"""
|
||
dob = DOB.search(text)
|
||
death = DEATH.search(text)
|
||
geno = ""
|
||
if dob:
|
||
tail = text[dob.end():]
|
||
tail = re.sub(r"^\s*/?\+?\s?\d[\d.]*", "", tail) # drop any /+death remnant
|
||
tail = tail.lstrip(" ,").strip()
|
||
# FIX-4 (Skarlett): strip trailing "/ +YEAR" death-year artifacts leaked from compact
|
||
# chart cells (e.g. "… rere / +2018"). The DEATH regex still captures the year from
|
||
# the full cell text, so it appears as a death-date conflict — not a genotype conflict.
|
||
tail = re.sub(r"\s*/\s*\+\d{4}\s*$", "", tail).strip()
|
||
if gt.looks_like_genotype(tail):
|
||
geno = tail
|
||
return (dob.group(1) if dob else "",
|
||
death.group(1) if death else "",
|
||
geno)
|
||
|
||
|
||
def extract_stammbaum(path):
|
||
"""Return list of animal dicts for one chart file."""
|
||
fname = os.path.basename(path)
|
||
z = __import__("zipfile").ZipFile(path)
|
||
ss = xu.shared_strings(z)
|
||
sheets = xu.sheet_paths(z)
|
||
cells = xu.read_cells(z, sheets[0], ss)
|
||
fillsex = xu.cell_fill_sex(z, sheets[0]) # box colour -> sex (blue=male, white=female)
|
||
|
||
has_col2 = any(c == 2 for (c, r) in cells)
|
||
col_offset = 0 if has_col2 else 3
|
||
|
||
# group cells by column for block reconstruction
|
||
by_col = {}
|
||
for (c, r), t in cells.items():
|
||
by_col.setdefault(c, []).append((r, t))
|
||
for c in by_col:
|
||
by_col[c].sort()
|
||
|
||
animals = []
|
||
used = set()
|
||
|
||
for (c, r), t in sorted(cells.items()):
|
||
if (c, r) in used:
|
||
continue
|
||
compact = re.match(r"^(.+?),\s*\*", t) # "Name,*DOB, genotype"
|
||
is_block_dob = bool(re.match(r"^\*\s?\d", t)) # standalone "*DOB"
|
||
if not compact and not is_block_dob:
|
||
continue
|
||
|
||
name_row = r
|
||
if compact:
|
||
name = clean_name(compact.group(1))
|
||
dob, death, geno = parse_detail(t)
|
||
farbschlag = ""
|
||
breeder = ""
|
||
used.add((c, r))
|
||
else:
|
||
# full block: name above, farbschlag/genotype/breeder below
|
||
dob, death, geno0 = parse_detail(t)
|
||
name = ""
|
||
for rr in range(r - 3, r):
|
||
if (c, rr) in cells and not re.match(r"^\*?\s?\d", cells[(c, rr)]) \
|
||
and not gt.looks_like_genotype(cells[(c, rr)]):
|
||
name = clean_name(cells[(c, rr)])
|
||
name_row = rr
|
||
used.add((c, rr))
|
||
break
|
||
farbschlag = ""
|
||
geno = geno0
|
||
breeder = ""
|
||
# BAND-AWARE (Julian-confirmed): early bands (gen 0-1, cols B/E/H) are 5-cell blocks
|
||
# WITH a Farbschlag cell; deep bands (gen >= 2, cols K/N/Q...) are 3-cell blocks
|
||
# (Name/DOB/Genotype) with NO Farbschlag — colour is derived from the genotype. So in
|
||
# deep bands we must NOT grab the next block's name or a stray health note as Farbschlag.
|
||
deep_band = gen_of(c, col_offset) >= 2
|
||
for rr in range(r + 1, r + 4):
|
||
cell = cells.get((c, rr))
|
||
if not cell:
|
||
continue
|
||
if gt.looks_like_genotype(cell):
|
||
geno = cell
|
||
used.add((c, rr))
|
||
elif re.search(r"\b(Zucht|Privatzucht)\b", cell) or cell.startswith("("):
|
||
breeder = cell
|
||
used.add((c, rr))
|
||
elif not deep_band and not farbschlag and not re.match(r"^\*?\s?\d", cell) \
|
||
and not looks_like_animal_name(cell):
|
||
farbschlag = cell
|
||
used.add((c, rr))
|
||
used.add((c, r))
|
||
|
||
g = parse_detail(t) if compact else (dob, death, geno)
|
||
genodict = gt.parse(geno)
|
||
# Zucht: from the name's [tag]/of-von suffix, else from the breeder line
|
||
_, zraw = split_name_zucht(name)
|
||
if not zraw and breeder:
|
||
zraw = breeder
|
||
animals.append({
|
||
"id": None, # assigned in dedup
|
||
"name": name,
|
||
"nameVariants": [],
|
||
"dob": dob,
|
||
"death": death,
|
||
"gender": fillsex.get((c, r)), # box colour: blue=male, white=female
|
||
"farbschlag": farbschlag,
|
||
"genotype": genodict,
|
||
"deaf": genodict.get("deaf"),
|
||
"tags": genodict.get("tags", []),
|
||
"breeder": breeder,
|
||
"zucht": zraw,
|
||
"parentRefs": [],
|
||
"photos": [],
|
||
"sourceFiles": [fname],
|
||
"_gen": gen_of(c, col_offset),
|
||
"_col": c,
|
||
"_row": name_row,
|
||
"_file": fname,
|
||
"_zucht": norm_zucht(zraw),
|
||
})
|
||
|
||
# name-pair cells "X & Y" (deepest generation, names only)
|
||
for (c, r), t in cells.items():
|
||
if (c, r) in used:
|
||
continue
|
||
if " & " in t and not DOB.search(t) and len(t) < 90 and gen_of(c, col_offset) >= 4:
|
||
for part in t.split(" & "):
|
||
part = clean_name(part)
|
||
if part:
|
||
_, zraw = split_name_zucht(part)
|
||
animals.append({
|
||
"id": None, "name": part, "nameVariants": [],
|
||
"dob": "", "death": "", "gender": None, "farbschlag": "",
|
||
"genotype": gt.parse(""), "deaf": None, "tags": [],
|
||
"breeder": "", "zucht": zraw,
|
||
"parentRefs": [], "photos": [], "sourceFiles": [fname],
|
||
"_gen": gen_of(c, col_offset), "_col": c, "_row": r, "_file": fname,
|
||
"_zucht": norm_zucht(zraw),
|
||
})
|
||
|
||
_reconstruct_parents(animals)
|
||
_attach_photos(z, sheets, animals, fname)
|
||
return animals
|
||
|
||
|
||
def _reconstruct_parents(animals):
|
||
"""Positional: an animal's parents are the bracketing blocks one generation
|
||
deeper (father = nearest block above, mother = nearest below). Role guess is
|
||
by vertical position (German charts: Vater oben) — flagged for review; the
|
||
Wurfchronik is authoritative for matched animals (Stage 3)."""
|
||
by_gen = {}
|
||
for a in animals:
|
||
by_gen.setdefault(a["_gen"], []).append(a)
|
||
for g, group in by_gen.items():
|
||
nxt = sorted(by_gen.get(g + 1, []), key=lambda a: a["_row"])
|
||
if not nxt:
|
||
continue
|
||
for a in group:
|
||
r = a["_row"]
|
||
above = [x for x in nxt if x["_row"] <= r]
|
||
below = [x for x in nxt if x["_row"] > r]
|
||
father = above[-1] if above else None
|
||
mother = below[0] if below else None
|
||
for parent, role in ((father, "father"), (mother, "mother")):
|
||
if parent:
|
||
p_name = parent["name"] or "unbekannt"
|
||
a["parentRefs"].append({
|
||
"name": p_name,
|
||
"dob": parent["dob"],
|
||
"roleGuess": role,
|
||
"method": "chart-position",
|
||
"confidence": "medium",
|
||
})
|
||
|
||
|
||
def _attach_photos(z, sheets, animals, fname):
|
||
anchors = [a for a in xu.image_anchors(z)]
|
||
if not anchors:
|
||
return
|
||
|
||
by_gen = {}
|
||
for a in animals:
|
||
by_gen.setdefault(a["_gen"], []).append(a)
|
||
media_dir = os.path.join(OUT, "photos")
|
||
col_offset = 0 if any(a["_col"] == 2 for a in animals) else 3
|
||
|
||
# A photo sits one column either side of (or on) its animal's name cell;
|
||
# the generation is read from the photo's column. Whether photos are LEFT or
|
||
# RIGHT of the name varies, and the old fixed-column heuristic (col 1 / col 4)
|
||
# misclassified sheets that have neither, shifting every photo one generation
|
||
# toward the proband (e.g. „Stammbaum von Kazuya“ / „Picus Son“: Kazuya wore
|
||
# his father's photo, the father his grandfather's). Instead, try BOTH
|
||
# interpretations and keep the one that places photos closest to their
|
||
# assigned animal's name column (minimal horizontal misalignment).
|
||
def get_anchor_gen(colnum, left_style):
|
||
effective_col = colnum - col_offset
|
||
if left_style:
|
||
if effective_col <= 3: return 0
|
||
if effective_col <= 6: return 1
|
||
if effective_col <= 9: return 2
|
||
if effective_col <= 12: return 3
|
||
if effective_col <= 15: return 4
|
||
return 5
|
||
else:
|
||
if effective_col <= 4: return 0
|
||
if effective_col <= 7: return 1
|
||
if effective_col <= 10: return 2
|
||
if effective_col <= 13: return 3
|
||
if effective_col <= 16: return 4
|
||
return 5
|
||
|
||
def targets_for(left_style):
|
||
out = []
|
||
for (sp, col, row, media) in anchors:
|
||
cands = by_gen.get(get_anchor_gen(col, left_style), []) or animals
|
||
out.append(min(cands, key=lambda a: abs((a["_row"] - row) - 10)) if cands else None)
|
||
return out
|
||
|
||
def mean_col_offset(targets):
|
||
vals = [abs(col - t["_col"]) for (sp, col, row, m), t in zip(anchors, targets) if t]
|
||
return sum(vals) / len(vals) if vals else 0.0
|
||
|
||
left_targets = targets_for(True)
|
||
right_targets = targets_for(False)
|
||
targets = (
|
||
left_targets
|
||
if mean_col_offset(left_targets) <= mean_col_offset(right_targets)
|
||
else right_targets
|
||
)
|
||
|
||
for (sp, col, row, media), target in zip(anchors, targets):
|
||
if not target:
|
||
continue
|
||
ext = os.path.splitext(media)[1] or ".img"
|
||
sl = slug(target["name"], target["dob"])
|
||
dest_dir = os.path.join(media_dir, sl)
|
||
os.makedirs(dest_dir, exist_ok=True)
|
||
rel = f"photos/{sl}/{os.path.basename(media)}"
|
||
try:
|
||
with z.open(media) as src, open(os.path.join(OUT, rel), "wb") as dst:
|
||
shutil.copyfileobj(src, dst)
|
||
if rel not in target["photos"]:
|
||
target["photos"].append(rel)
|
||
except KeyError:
|
||
pass
|
||
|
||
|
||
# -------------------------------------------------- Wurfchronik extraction ---
|
||
def extract_wurfchronik(path):
|
||
"""Return list of litter dicts. Parses columns BY HEADER (sheets differ)."""
|
||
fname = os.path.basename(path)
|
||
z = __import__("zipfile").ZipFile(path)
|
||
ss = xu.shared_strings(z)
|
||
litters = []
|
||
for sp in xu.sheet_paths(z):
|
||
cells = xu.read_cells(z, sp, ss)
|
||
if not cells:
|
||
continue
|
||
# build row -> {colnum: text}
|
||
rows = {}
|
||
for (c, r), t in cells.items():
|
||
rows.setdefault(r, {})[c] = t
|
||
hdr = xu.header_row(cells) # colnum -> header label
|
||
hdr_row = min(r for (_, r) in cells) # the header row number, to skip it
|
||
# map header label -> colnum (fuzzy by keyword)
|
||
def find(*keys):
|
||
for c, lbl in hdr.items():
|
||
low = lbl.lower()
|
||
if any(k in low for k in keys):
|
||
return c
|
||
return None
|
||
col_id = find("wurfbuchstabe", "buchstabe")
|
||
col_date = find("geburtsdatum", "datum")
|
||
col_dam = find("mutter")
|
||
col_sire = find("vater")
|
||
col_ws = find("ws", "wurfstärke", "wurfstaerke")
|
||
col_breakdown = find("männchen", "maennchen", "weibchen")
|
||
col_zn = find("zuchtnummer")
|
||
col_note = find("bemerkung", "anmerkung")
|
||
# Julian (authoritative): Tabelle1 has an UNLABELED numeric column between
|
||
# Vater and Wurfstärke = "Überlebende bis zum Abgabedatum" (survivors to
|
||
# go-home). Tabelle2 dropped it. Header-based find() cannot see it, so
|
||
# detect it positionally.
|
||
col_survived = None
|
||
if col_sire and col_ws and col_ws - col_sire > 1:
|
||
mapped = {col_id, col_date, col_dam, col_sire, col_ws,
|
||
col_breakdown, col_zn, col_note}
|
||
for c in range(col_sire + 1, col_ws):
|
||
if c not in mapped:
|
||
col_survived = c
|
||
break
|
||
sheet_name = os.path.basename(sp)
|
||
for r in sorted(rows):
|
||
if r == hdr_row: # skip the header row itself
|
||
continue
|
||
row = rows[r]
|
||
# skip empty-id + "Jahr YYYY" section rows
|
||
txt_b = row.get(col_date, "") if col_date else ""
|
||
if not row.get(col_id):
|
||
continue
|
||
if "jahr" in " ".join(row.values()).lower() and not DOB.search(txt_b):
|
||
continue
|
||
dob = DOB.search(txt_b)
|
||
lid = row.get(col_id, "")
|
||
datestr = dob.group(1) if dob else ""
|
||
dam_raw = row.get(col_dam, "") if col_dam else ""
|
||
sire_raw = row.get(col_sire, "") if col_sire else ""
|
||
_, dam_zucht = split_name_zucht(dam_raw)
|
||
# '&' = multiple sires possible (litter with uncertain/dual sire)
|
||
sire_names = [s.strip() for s in sire_raw.split("&") if s.strip()]
|
||
_, sire_zucht = split_name_zucht(sire_names[0] if sire_names else "")
|
||
|
||
# Numeric layout is read VALUE-ADAPTIVELY per row: a handful of
|
||
# Tabelle2 rows insert an extra numeric column (Überlebende, T1
|
||
# order E,F) before WS and shift the breakdown right of its header.
|
||
warnings = []
|
||
scan_end = (col_note or (col_sire or 4) + 6) + 1
|
||
bd_col = None
|
||
for c in range((col_sire or 4) + 1, scan_end):
|
||
if re.fullmatch(r"\d+(?:[,;]\d+){1,3}", row.get(c, "")):
|
||
bd_col = c
|
||
break
|
||
if bd_col and col_breakdown and bd_col != col_breakdown:
|
||
warnings.append(
|
||
f"Spaltenschema-Abweichung: Geschlechter-Aufschlüsselung in "
|
||
f"Spalte {xu.num_to_col(bd_col)} statt "
|
||
f"{xu.num_to_col(col_breakdown)} gefunden")
|
||
# single numeric cells between Vater and the breakdown: E (Überlebende)
|
||
# and/or F (Wurfstärke), in T1 order
|
||
singles = []
|
||
for c in range((col_sire or 4) + 1, bd_col or scan_end):
|
||
v = row.get(c, "")
|
||
if re.fullmatch(r"\d+", v):
|
||
singles.append(int(v))
|
||
if len(singles) >= 2:
|
||
survived, total_born = singles[0], singles[1]
|
||
if not col_survived:
|
||
warnings.append(
|
||
"Spaltenschema-Abweichung: zusätzliche Zahlenspalte als "
|
||
"„Überlebende“ interpretiert (bitte prüfen)")
|
||
elif len(singles) == 1:
|
||
total_born = singles[0]
|
||
survived = None
|
||
else:
|
||
total_born = _to_int(row.get(col_ws)) if col_ws else None
|
||
survived = _to_int(row.get(col_survived)) if col_survived else None
|
||
|
||
# breakdown "2,0,2,0" = Männchen, Weibchen, Totgeburt, später
|
||
# verstorben ('s' = died after birth, before Abgabe — Julian).
|
||
bd = row.get(bd_col, "") if bd_col else ""
|
||
m = re.findall(r"\d+", bd)
|
||
males = females = stillborn = died_later = None
|
||
if m:
|
||
vals = [int(x) for x in m] + [None] * 4
|
||
males, females, stillborn, died_later = vals[:4]
|
||
# Validation (Julian): Überlebende E should equal F − TG − s.
|
||
# Mismatch = data-quality signal for the review report, NOT a blocker.
|
||
if survived is not None and total_born is not None:
|
||
expected = total_born - (stillborn or 0) - (died_later or 0)
|
||
if survived != expected:
|
||
warnings.append(
|
||
f"Überlebende ({survived}) ≠ Wurfstärke ({total_born}) "
|
||
f"− Totgeburten ({stillborn or 0}) − später verstorben "
|
||
f"({died_later or 0}) = {expected}")
|
||
litters.append({
|
||
"id": f"{sheet_name.replace('.xml','')}-{lid}-{norm_dob(datestr)}",
|
||
"litterId": lid,
|
||
"date": datestr,
|
||
"damName": dam_raw,
|
||
"damZucht": dam_zucht,
|
||
"sireName": sire_raw,
|
||
"sireNames": sire_names,
|
||
"sireZucht": sire_zucht,
|
||
"totalBorn": total_born,
|
||
"survivedToGoHome": survived,
|
||
"males": males,
|
||
"females": females,
|
||
"stillborn": stillborn,
|
||
"diedLater": died_later,
|
||
"breakdownRaw": bd,
|
||
"zuchtnummer": row.get(col_zn, "") if col_zn else "",
|
||
"note": row.get(col_note, "") if col_note else "",
|
||
"warnings": warnings,
|
||
"sourceFile": fname,
|
||
"sheet": sheet_name,
|
||
"row": r,
|
||
})
|
||
return litters
|
||
|
||
|
||
def _to_int(s):
|
||
if not s:
|
||
return None
|
||
m = re.search(r"\d+", s)
|
||
return int(m.group()) if m else None
|
||
|
||
|
||
# ------------------------------------------------------------- stage 2: dedup
|
||
def _geno_key(genodict):
|
||
"""Canonical, order-independent key of a genotype's mapped loci — used for conflict
|
||
detection so Uw==G (and allele ordering) no longer count as a conflict."""
|
||
m = genodict.get("mapped8locus", {})
|
||
return "|".join(f"{locus}:{','.join(sorted(m[locus]))}" for locus in sorted(m))
|
||
|
||
|
||
# --- "presence wins" merge rule (Julian) -------------------------------------
|
||
# When two source variants of the SAME animal differ ONLY by a token PRESENT in one and
|
||
# ABSENT in the other — a whole locus (e.g. spsp recorded in one chart, omitted in another)
|
||
# or a modifier on the same base allele (e^f vs e, i.e. the [f] marker) — keep the present
|
||
# token; that is NOT a conflict. A genuine VALUE contradiction (different filled alleles:
|
||
# E vs e, D vs d, c^h vs c^chm) OR unknown-vs-filled (D- vs DD, the '?' second allele) STILL
|
||
# quarantines for human decision. (Markers/flags WP/DP/WFNZ/hörend are already tags/flags,
|
||
# never part of the genotype, so they never reach here.)
|
||
def _split_allele(a):
|
||
return tuple(a.split("^", 1)) if "^" in a else (a, "")
|
||
|
||
|
||
def _alleles_compatible(a, b):
|
||
if a == b:
|
||
return True
|
||
if a == "?" or b == "?":
|
||
return True # specific-wins: unknown allele is compatible with any
|
||
# specified value (C- vs CC -> CC; G- vs Gg -> Gg)
|
||
(ba, ma), (bb, mb) = _split_allele(a), _split_allele(b)
|
||
if ba != bb:
|
||
return False # different base allele = real value diff (E vs e, D vs d)
|
||
return ma == "" or mb == "" # same base, modifier present-vs-absent -> presence wins
|
||
|
||
|
||
def _pair_compatible(p, q):
|
||
if len(p) != 2 or len(q) != 2:
|
||
return p == q
|
||
return ((_alleles_compatible(p[0], q[0]) and _alleles_compatible(p[1], q[1])) or
|
||
(_alleles_compatible(p[0], q[1]) and _alleles_compatible(p[1], q[0])))
|
||
|
||
|
||
def _genotype_conflict(mapped_list):
|
||
"""True only if two variants GENUINELY contradict at a shared locus. A locus present in
|
||
one variant and absent in another is fine (presence wins); so is a modifier present-vs-
|
||
absent on the same base allele. Replaces the old `len(distinct geno keys) > 1` test."""
|
||
loci = set()
|
||
for m in mapped_list:
|
||
loci.update(m.keys())
|
||
for locus in loci:
|
||
pairs = [m[locus] for m in mapped_list if locus in m]
|
||
for i in range(len(pairs)):
|
||
for j in range(i + 1, len(pairs)):
|
||
if not _pair_compatible(pairs[i], pairs[j]):
|
||
return True
|
||
return False
|
||
|
||
|
||
def dedup(animals):
|
||
"""Merge by normalise(call-name)+DOB, with the canonical Zucht as
|
||
DISCRIMINATOR (Julian: same name+DOB+Zucht = same animal; different Zucht =
|
||
different animal). Returns (merged, conflicts, orphans, zucht_splits)."""
|
||
groups = {}
|
||
orphans = []
|
||
for a in animals:
|
||
call, _ = split_name_zucht(a["name"])
|
||
key = (norm_name(call), norm_dob(a["dob"]))
|
||
if not key[0] or not key[1]:
|
||
orphans.append(a)
|
||
# orphans still get a stable id but are not merged
|
||
key = ("__orphan__", id(a))
|
||
groups.setdefault(key, []).append(a)
|
||
|
||
# split groups whose members carry DIFFERENT (non-empty) canonical Zuchten;
|
||
# members without a Zucht merge into the group only if it is unambiguous.
|
||
final_groups = []
|
||
zucht_splits = []
|
||
for key, grp in groups.items():
|
||
by_zucht = {}
|
||
for a in grp:
|
||
by_zucht.setdefault(a.get("_zucht", ""), []).append(a)
|
||
nonempty = sorted(z for z in by_zucht if z)
|
||
if len(nonempty) <= 1:
|
||
final_groups.append(grp)
|
||
continue
|
||
# genuine split: same call-name+DOB, different Zucht
|
||
for z in nonempty:
|
||
final_groups.append(by_zucht[z])
|
||
if "" in by_zucht:
|
||
# Zucht-less mentions cannot be attributed -> own (flagged) entry
|
||
final_groups.append(by_zucht[""])
|
||
zucht_splits.append({
|
||
"name": grp[0]["name"],
|
||
"dob": norm_dob(grp[0]["dob"]),
|
||
"zuechte": sorted(set(a["zucht"] for a in grp if a.get("zucht"))),
|
||
"files": sorted(set(f for a in grp for f in a["sourceFiles"])),
|
||
})
|
||
|
||
# 2. Merge groups that have different DOBs but same call-name (and Zucht) and matching parents
|
||
def get_parent_keys(a):
|
||
parent_refs = a.get("parentRefs", [])
|
||
f_name = next((p["name"] for p in parent_refs if p.get("roleGuess") == "father"), "")
|
||
m_name = next((p["name"] for p in parent_refs if p.get("roleGuess") == "mother"), "")
|
||
return norm_name(f_name), norm_name(m_name)
|
||
|
||
def groups_parents_match(g1, g2):
|
||
for a1 in g1:
|
||
for a2 in g2:
|
||
f1, m1 = get_parent_keys(a1)
|
||
f2, m2 = get_parent_keys(a2)
|
||
if f1 and f2 and f1 == f2 and m1 and m2 and m1 == m2:
|
||
return True
|
||
return False
|
||
|
||
i = 0
|
||
while i < len(final_groups):
|
||
g1 = final_groups[i]
|
||
call1, _ = split_name_zucht(g1[0]["name"])
|
||
n_call1 = norm_name(call1)
|
||
zucht1 = g1[0].get("_zucht", "")
|
||
|
||
j = i + 1
|
||
merged_any = False
|
||
while j < len(final_groups):
|
||
g2 = final_groups[j]
|
||
call2, _ = split_name_zucht(g2[0]["name"])
|
||
n_call2 = norm_name(call2)
|
||
zucht2 = g2[0].get("_zucht", "")
|
||
|
||
if n_call1 == n_call2:
|
||
if zucht1 == zucht2 or not zucht1 or not zucht2:
|
||
if groups_parents_match(g1, g2):
|
||
g1.extend(g2)
|
||
final_groups.pop(j)
|
||
merged_any = True
|
||
continue
|
||
j += 1
|
||
if merged_any:
|
||
continue
|
||
i += 1
|
||
|
||
merged = []
|
||
conflicts = []
|
||
for grp in final_groups:
|
||
base = dict(grp[0])
|
||
variants = set([base["name"]])
|
||
files = set(base["sourceFiles"])
|
||
photos = list(base["photos"])
|
||
parent_refs = list(base["parentRefs"])
|
||
genos = set()
|
||
geno_keys = set() # GEN-3b: conflict on NORMALIZED genotype (Uw==G) not raw text
|
||
mapped_variants = [] # mapped8locus per variant — for the 'presence wins' conflict test
|
||
farb = set()
|
||
deaths = set()
|
||
deaf_seen = set()
|
||
tags_set = set()
|
||
genders = []
|
||
for a in grp:
|
||
variants.add(a["name"])
|
||
files.update(a["sourceFiles"])
|
||
photos.extend(a["photos"])
|
||
parent_refs.extend(a["parentRefs"])
|
||
if a.get("gender"):
|
||
genders.append(a["gender"])
|
||
if a["genotype"]["mapped8locus"]:
|
||
genos.add(a["genotype"]["rawGenotype"])
|
||
geno_keys.add(_geno_key(a["genotype"]))
|
||
mapped_variants.append(a["genotype"]["mapped8locus"])
|
||
if a["farbschlag"]:
|
||
farb.add(a["farbschlag"])
|
||
if a["death"]:
|
||
deaths.add(norm_dob(a["death"]))
|
||
if a.get("deaf") is not None:
|
||
deaf_seen.add(a["deaf"])
|
||
tags_set.update(a.get("tags", []))
|
||
# pick the richest genotype: most mapped loci, then fewest unknowns ('?' alleles = specific
|
||
# wins, FIX-2), then longest raw string as final tiebreaker.
|
||
def _specificity(gd):
|
||
return sum(1 for pair in gd["mapped8locus"].values() for a in pair if a != "?")
|
||
best = max((a["genotype"] for a in grp),
|
||
key=lambda gd: (len(gd["mapped8locus"]), _specificity(gd), len(gd["rawGenotype"])))
|
||
|
||
# Find best DOB (proband first)
|
||
best_dob = ""
|
||
for a in grp:
|
||
if a.get("_gen") == 0 and a.get("dob"):
|
||
best_dob = a["dob"]
|
||
break
|
||
if not best_dob:
|
||
for a in grp:
|
||
if a.get("dob"):
|
||
best_dob = a["dob"]
|
||
break
|
||
chosen_dob = norm_dob(best_dob or base["dob"])
|
||
|
||
out = {
|
||
"id": slug(base["name"], chosen_dob),
|
||
"name": base["name"],
|
||
"nameVariants": sorted(v for v in variants if v),
|
||
"dob": chosen_dob,
|
||
"death": sorted(deaths)[0] if deaths else "",
|
||
# box-colour sex (blue=male, white=female): majority across mentions, else None.
|
||
"gender": Counter(genders).most_common(1)[0][0] if genders else None,
|
||
"farbschlag": sorted(farb)[0] if farb else "",
|
||
"farbschlagVariants": sorted(farb),
|
||
"genotype": best,
|
||
"breeder": next((a["breeder"] for a in grp if a["breeder"]), ""),
|
||
"zucht": next((a["zucht"] for a in grp if a.get("zucht")), ""),
|
||
"zuchtCanon": next((a["_zucht"] for a in grp if a.get("_zucht")), ""),
|
||
"parentRefs": _dedup_parentrefs(parent_refs),
|
||
"photos": sorted(set(photos)),
|
||
"sourceFiles": sorted(files),
|
||
"mentions": len(grp),
|
||
# GEN-3b: hearing/deaf phenotype flag (deaf wins if any mention says so) + tags.
|
||
"deaf": (True if True in deaf_seen else (False if False in deaf_seen else None)),
|
||
"tags": sorted(tags_set),
|
||
# FEAT-8c: machine-readable quarantine marker so the API loader can skip
|
||
# conflicting records without parsing the German review report.
|
||
"conflict": False,
|
||
# DOB-Remap (manuelle Entscheidung): original/corrected birthdate so the
|
||
# downstream Datenherkunft can record the discarded original value.
|
||
"dobRemap": next((a["dobRemap"] for a in grp if a.get("dobRemap")), None),
|
||
}
|
||
merged.append(out)
|
||
# conflict: same animal, GENUINELY disagreeing genotype (presence-vs-absence is NOT a
|
||
# conflict — Julian's 'presence wins') or >1 distinct farbschlag or >1 distinct death.
|
||
if _genotype_conflict(mapped_variants) or len(farb) > 1 or len(deaths) > 1:
|
||
out["conflict"] = True
|
||
conflicts.append({
|
||
"id": out["id"], "name": base["name"], "dob": out["dob"],
|
||
"genotypes": sorted(genos), "farbschlaege": sorted(farb),
|
||
"deaths": sorted(deaths), "files": sorted(files),
|
||
})
|
||
merged.sort(key=lambda a: (a["dob"], a["name"]))
|
||
# ids must stay unique even when a Zucht split duplicates name+DOB slugs
|
||
seen_ids = {}
|
||
for a in merged:
|
||
if a["id"] in seen_ids:
|
||
seen_ids[a["id"]] += 1
|
||
a["id"] = f"{a['id']}-{seen_ids[a['id']]}"
|
||
else:
|
||
seen_ids[a["id"]] = 1
|
||
return merged, conflicts, orphans, zucht_splits
|
||
|
||
|
||
# ------------------------------------------- animal <-> litter matching ------
|
||
def match_litters(merged, litters):
|
||
"""Attach each animal to its Wurfchronik litter (Pam-validated build order:
|
||
litters are canonical, animals match onto them via DOB + (Vater, Mutter)).
|
||
Sets a['litterRef']; returns match statistics for the report."""
|
||
by_date = {}
|
||
for l in litters:
|
||
d = norm_dob(l["date"])
|
||
if d:
|
||
by_date.setdefault(d, []).append(l)
|
||
|
||
stats = {"parents": 0, "dateOnly": 0, "ambiguous": 0}
|
||
for a in merged:
|
||
if not a["dob"]:
|
||
continue
|
||
cands = by_date.get(a["dob"])
|
||
if not cands:
|
||
continue
|
||
a_parents = set()
|
||
for ref in a["parentRefs"]:
|
||
cn, _ = canon_pair(ref["name"])
|
||
if cn:
|
||
a_parents.add(cn)
|
||
|
||
def score(l):
|
||
s = 0
|
||
for nm in [l["damName"]] + l.get("sireNames", []):
|
||
cn, _ = canon_pair(nm)
|
||
if cn and cn in a_parents:
|
||
s += 1
|
||
return s
|
||
|
||
scored = sorted(((score(l), l["id"]) for l in cands), reverse=True)
|
||
best_score, best_id = scored[0]
|
||
if best_score > 0 and (len(scored) == 1 or scored[1][0] < best_score):
|
||
a["litterRef"] = {"litterId": best_id,
|
||
"method": "geburtsdatum+eltern", "confidence": "hoch"}
|
||
stats["parents"] += 1
|
||
elif len(cands) == 1:
|
||
a["litterRef"] = {"litterId": cands[0]["id"],
|
||
"method": "nur-geburtsdatum", "confidence": "niedrig"}
|
||
stats["dateOnly"] += 1
|
||
else:
|
||
a["litterRef"] = {"litterId": None, "method": "mehrdeutig",
|
||
"candidates": [l["id"] for l in cands]}
|
||
stats["ambiguous"] += 1
|
||
return stats
|
||
|
||
|
||
def _dedup_parentrefs(refs):
|
||
seen = {}
|
||
for r in refs:
|
||
k = (norm_name(r["name"]), r["roleGuess"])
|
||
if k not in seen:
|
||
seen[k] = r
|
||
return list(seen.values())
|
||
|
||
|
||
# ------------------------------------------------------------------ reporting
|
||
def write_report(merged, conflicts, orphans, raw_count, litters, photo_count,
|
||
zucht_splits, match_stats):
|
||
multi = [a for a in merged if a["mentions"] > 1]
|
||
with_dob = [a for a in merged if a["dob"]]
|
||
val_warn = [l for l in litters if any("≠" in w for w in l["warnings"])]
|
||
schema_warn = [l for l in litters if any("Abweichung" in w for w in l["warnings"])]
|
||
matched = match_stats["parents"] + match_stats["dateOnly"]
|
||
|
||
L = []
|
||
L.append("# FEAT-8b — Import-Vorschau & Prüfbericht (Stammbäume + Wurfchronik)\n")
|
||
L.append("_Automatisch erzeugt von `tools/import/extract.py` — **noch nichts in die Datenbank geladen.** "
|
||
"Bitte prüfen, bevor importiert wird._\n")
|
||
L.append("## Überblick\n")
|
||
L.append(f"- Rohe Tier-Einträge aus den Stammbäumen: **{raw_count}**")
|
||
L.append(f"- Nach Zusammenführung (eindeutige Tiere): **{len(merged)}**")
|
||
L.append(f" - davon mit Geburtsdatum: {len(with_dob)}")
|
||
L.append(f" - in mehreren Dateien gefunden (Dubletten zusammengeführt): {len(multi)}")
|
||
L.append(f"- Konflikte zur Klärung: **{len(conflicts)}**")
|
||
L.append(f"- Mehrdeutige / unvollständige Einträge (ohne Name+Datum): **{len(orphans)}**")
|
||
L.append(f"- Fotos zugeordnet: **{photo_count}**")
|
||
L.append(f"- Würfe aus der Wurfchronik: **{len(litters)}**")
|
||
L.append(f" - Tiere mit Wurf verknüpft: **{matched}** "
|
||
f"(davon über Geburtsdatum **und** Eltern: {match_stats['parents']}, "
|
||
f"nur über Geburtsdatum: {match_stats['dateOnly']}; "
|
||
f"mehrdeutig: {match_stats['ambiguous']})")
|
||
L.append(f" - Würfe mit Datenqualitäts-Hinweisen: {len(val_warn)} "
|
||
f"(+ {len(schema_warn)} Zeilen mit abweichendem Spaltenschema)\n")
|
||
|
||
L.append("## Zusammenführungs-Schlüssel\n")
|
||
L.append("Tiere wurden zusammengeführt über **normalisierter Rufname + Geburtsdatum**, "
|
||
"mit der **Zucht als Unterscheidungsmerkmal** (Julians Regel: die `[Klammern]` "
|
||
"in der Wurfchronik und das `of/von <Linie>`-Suffix der Stammbäume bezeichnen "
|
||
"beide die Zucht und werden zusammengeführt — z. B. `[ZdkC]` ≙ "
|
||
"`von den Kleinen Chaoten`). Namensvarianten (z. B. `v.d.` ↔ `von den`, "
|
||
"`gen.`-Spitznamen) werden als `nameVariants` erhalten.\n")
|
||
|
||
if zucht_splits:
|
||
L.append("### Gleicher Name + Geburtsdatum, aber unterschiedliche Zucht "
|
||
"(NICHT zusammengeführt — bitte prüfen)\n")
|
||
L.append("| Tier | Geburtsdatum | Zuchten | Dateien |")
|
||
L.append("|---|---|---|---|")
|
||
for s in zucht_splits[:50]:
|
||
L.append("| {} | {} | {} | {} |".format(
|
||
split_name_zucht(s["name"])[0], s["dob"],
|
||
" // ".join(s["zuechte"]),
|
||
", ".join(os.path.splitext(f)[0] for f in s["files"])))
|
||
L.append("")
|
||
|
||
L.append("## ⚠️ Konflikte (bitte prüfen)\n")
|
||
if conflicts:
|
||
L.append("Gleiches Tier (Name+Datum), aber widersprüchliche Angaben in verschiedenen Dateien:\n")
|
||
L.append("| Tier | Geburtsdatum | abweichende Genotypen | abweichende Farbschläge | Sterbedaten | Dateien |")
|
||
L.append("|---|---|---|---|---|---|")
|
||
for c in conflicts[:200]:
|
||
L.append("| {} | {} | {} | {} | {} | {} |".format(
|
||
c["name"], c["dob"],
|
||
" // ".join(c["genotypes"]) or "—",
|
||
" // ".join(c["farbschlaege"]) or "—",
|
||
" // ".join(c["deaths"]) or "—",
|
||
", ".join(os.path.splitext(f)[0] for f in c["files"])))
|
||
else:
|
||
L.append("_Keine._\n")
|
||
|
||
L.append("\n## Mehrdeutige / unvollständige Einträge\n")
|
||
L.append(f"{len(orphans)} Einträge ohne sichere Name+Datum-Kombination "
|
||
"(z. B. `Name1 & Name2`-Paarzellen der tiefsten Generation, oder Zellen ohne Datum). "
|
||
"Diese werden NICHT automatisch zusammengeführt.\n")
|
||
sample = [o for o in orphans if o["name"]][:40]
|
||
for o in sample:
|
||
L.append(f"- {o['name']} · {o.get('_file','')}")
|
||
|
||
# orphan -> likely same-named full record (soft hint, not auto-merged)
|
||
from collections import Counter
|
||
name_index = {}
|
||
for a in merged:
|
||
if a["dob"]:
|
||
name_index.setdefault(norm_name(a["name"]), []).append(a)
|
||
matchable = []
|
||
for o in orphans:
|
||
if not o["name"]:
|
||
continue
|
||
cands = name_index.get(norm_name(o["name"]))
|
||
if cands:
|
||
matchable.append((o, cands))
|
||
L.append("\n## Wahrscheinliche Zuordnungen unvollständiger Einträge\n")
|
||
L.append(f"{len(matchable)} namenlose/datenlose Einträge tragen denselben Namen wie ein "
|
||
"vollständiges Tier — vermutlich dasselbe Tier (zur Bestätigung):\n")
|
||
for o, cands in matchable[:60]:
|
||
opts = "; ".join(f"{c['name']} (*{c['dob']})" for c in cands[:3])
|
||
L.append(f"- „{o['name']}“ → {opts}")
|
||
|
||
# unmapped-token summary (for Kevin / GEN-2 + the wife)
|
||
tok = Counter()
|
||
for a in merged:
|
||
for t in a["genotype"]["unmappedTokens"]:
|
||
tok[t] += 1
|
||
L.append("\n## Nicht ins 8-Loci-Modell abgebildete Tokens (verbatim erhalten)\n")
|
||
L.append("Diese Tokens stehen weiter in `rawGenotype`/`unmappedTokens` — Entscheidung "
|
||
"(Modell erweitern vs. als Notiz) liegt bei Julian/Kevin:\n")
|
||
L.append("| Token | Vorkommen | Bedeutung (Vermutung) |")
|
||
L.append("|---|---|---|")
|
||
hint = {"Uwuw[d]": "9. Locus Uw (nicht im Modell)", "UwUw": "9. Locus Uw",
|
||
"[WFNZ]": "Marker", "[DP]": "Marker (Dunkelpigment?)", "DP": "Marker",
|
||
"WP": "Marker", "[WP]": "Marker", "C(C)": "Schreibweise (C trägt c)",
|
||
"chmchm": "Schreibweise (c[chm]c[chm])"}
|
||
for t, n in tok.most_common(25):
|
||
L.append(f"| `{t}` | {n} | {hint.get(t, '?')} |")
|
||
|
||
# Wurfchronik data quality (Julian: E sollte = F − TG − s sein)
|
||
L.append("\n## Wurfchronik — Datenqualitäts-Hinweise\n")
|
||
L.append("Julians Spaltenregel: **Überlebende bis Abgabe (E) = Wurfstärke (F) − "
|
||
"Totgeburten (TG) − später verstorben (s)**. Bei diesen Würfen geht die "
|
||
"Rechnung nicht auf — kein Import-Hindernis, aber ein Hinweis auf "
|
||
"Tippfehler oder fehlende Einträge:\n")
|
||
if val_warn:
|
||
L.append("| Wurf | Datum | Mutter × Vater | Hinweis |")
|
||
L.append("|---|---|---|---|")
|
||
for l in val_warn[:120]:
|
||
L.append("| {} | {} | {} × {} | {} |".format(
|
||
l["litterId"], l["date"], l["damName"], l["sireName"],
|
||
"; ".join(w for w in l["warnings"] if "≠" in w)))
|
||
if len(val_warn) > 120:
|
||
L.append(f"\n… und {len(val_warn) - 120} weitere (siehe `litters.json`).")
|
||
else:
|
||
L.append("_Keine — alle Würfe sind rechnerisch konsistent._")
|
||
|
||
L.append("\n### Zeilen mit abweichendem Spaltenschema (automatisch interpretiert)\n")
|
||
L.append(f"{len(schema_warn)} Zeilen (überwiegend Tabelle2 ab 2014) tragen eine "
|
||
"zusätzliche Zahlenspalte vor der Wurfstärke bzw. eine verschobene "
|
||
"Geschlechter-Aufschlüsselung. Sie wurden nach dem Muster von Tabelle1 "
|
||
"gelesen (**Überlebende, Wurfstärke, Aufschlüsselung**) — bei "
|
||
f"{sum(1 for l in schema_warn if not any('≠' in w for w in l['warnings']))} "
|
||
"davon geht die Rechnung E = F − TG − s damit exakt auf, was die Lesart "
|
||
"bestätigt. Alle betroffenen Zeilen sind in `litters.json` mit "
|
||
"`warnings` markiert. Beispiele:\n")
|
||
for l in schema_warn[:8]:
|
||
L.append(f"- Wurf {l['litterId']} ({l['date']}): E={l['survivedToGoHome']}, "
|
||
f"F={l['totalBorn']}, Aufschlüsselung `{l['breakdownRaw']}`")
|
||
|
||
L.append("\n## Hinweise für den Import (Stufe 3, später)\n")
|
||
L.append("- **Wurfchronik = Quelle der Würfe** (Datum, Wurfstärke, Eltern, Zuchtnummer); "
|
||
"**Stammbäume = Abstammung + Genotyp + Fotos**. Verknüpfung über Geburtsdatum + Elternnamen.")
|
||
L.append("- Eltern-Verknüpfungen (`parentRefs`) stammen aus der **Position im Stammbaum** "
|
||
"(Vater oben / Mutter unten, mittlere Konfidenz) — die Wurfchronik korrigiert dies maßgeblich.")
|
||
L.append("- Genotyp: `mapped8locus` (A C D E G P Sp Re), `rawGenotype` (wortgetreu), "
|
||
"`unmappedTokens` (z. B. `Uw`, `Sls`, `Dea`, Marker wie `WFNZ/WP/DP`) — **nichts geht verloren**.")
|
||
L.append("- `-` (unbekanntes zweites Allel) → `?` (Platzhalter; Annahme, bitte bestätigen).")
|
||
with open(os.path.join(OUT, "review-report.md"), "w", encoding="utf-8") as f:
|
||
f.write("\n".join(L) + "\n")
|
||
|
||
|
||
# ------------------------------------------------------------------------ main
|
||
def apply_dob_remaps(raw_animals, path):
|
||
"""PRE-dedup: a conflict-decision carrying `correctDob` marks a record as a DUPLICATE with a
|
||
wrong birthdate — remap that raw record's DOB to correctDob so dedup MERGES it into the
|
||
canonical same-named animal (e.g. Chelsea *15.10.2021 -> *02.04.2021). Match =
|
||
canon_pair(name)+(dob) with same Zucht-aware logic as apply_conflict_decisions (see there).
|
||
Tolerates a missing/garbled file. Returns the remap count.
|
||
Must run BEFORE dedup (it changes the dedup identity). (god/HUMANQUESTION D — Dubletten.)"""
|
||
remaps_full = {} # (nameCanon, zuchtCanon, dob) -> correctDob — decision carries Zucht
|
||
remaps_name = {} # (nameCanon, dob) -> correctDob — no Zucht in decision
|
||
try:
|
||
with open(path, encoding="utf-8") as fh:
|
||
for r in (json.load(fh).get("resolutions") or []):
|
||
if r.get("correctDob"):
|
||
nc, zc = canon_pair(r.get("name", ""))
|
||
dob = norm_dob(r.get("dob", ""))
|
||
if zc:
|
||
remaps_full[(nc, zc, dob)] = r["correctDob"]
|
||
else:
|
||
remaps_name[(nc, dob)] = r["correctDob"]
|
||
except (OSError, ValueError):
|
||
return 0
|
||
if not remaps_full and not remaps_name:
|
||
return 0
|
||
n = 0
|
||
for a in raw_animals:
|
||
nc, zc = canon_pair(a.get("name", ""))
|
||
dob = norm_dob(a.get("dob", ""))
|
||
new = remaps_full.get((nc, zc, dob)) or remaps_name.get((nc, dob))
|
||
if new and a.get("dob") != new:
|
||
# Keep the discarded original DOB so the provenance/Datenherkunft can
|
||
# show „Geburtsdatum X verworfen, per Entscheidung auf Y geändert.“
|
||
a["dobRemap"] = {"original": a.get("dob"), "corrected": new}
|
||
a["dob"] = new
|
||
n += 1
|
||
return n
|
||
|
||
|
||
def apply_conflict_decisions(merged, conflicts, path):
|
||
"""Consume human conflict resolutions (tools/import/conflict-decisions.json) so the wife's
|
||
answers UN-QUARANTINE animals. Schema: {"resolutions":[{name, dob, decision, genotype?,
|
||
farbschlag?, source}]}. Match = canon_pair(name)+(dob):
|
||
- When the decision name CARRIES a Zucht (zuchtCanon != ''), match on the FULL
|
||
(nameCanon, zuchtCanon, dob) triple — preserves the C3 rule that same name+DOB but
|
||
different Zucht = different animal.
|
||
- When the decision has NO Zucht, fall back to (nameCanon, dob) name-only match.
|
||
Both spellings v.d. / von den fold to the same canon. A matching animal: clear its conflict,
|
||
mark resolvedByDecision; an explicit `genotype` (breeder notation) is parsed and becomes
|
||
authoritative, `farbschlag` overrides too. Tolerates a missing/empty/garbled file.
|
||
Returns the number of conflicts resolved. (god/HUMANQUESTION D.)"""
|
||
decisions_full = {} # (nameCanon, zuchtCanon, dob) -> r — when decision carries a Zucht
|
||
decisions_name = {} # (nameCanon, dob) -> r — fallback, decision has no Zucht
|
||
try:
|
||
with open(path, encoding="utf-8") as fh:
|
||
for r in (json.load(fh).get("resolutions") or []):
|
||
nc, zc = canon_pair(r.get("name", ""))
|
||
dob = norm_dob(r.get("dob", ""))
|
||
if zc:
|
||
decisions_full[(nc, zc, dob)] = r
|
||
else:
|
||
decisions_name[(nc, dob)] = r
|
||
except (OSError, ValueError):
|
||
return 0
|
||
if not decisions_full and not decisions_name:
|
||
return 0
|
||
|
||
resolved = 0
|
||
for a in merged:
|
||
nc, zc = canon_pair(a["name"])
|
||
dob = norm_dob(a["dob"])
|
||
d = decisions_full.get((nc, zc, dob)) or decisions_name.get((nc, dob))
|
||
if not d:
|
||
continue
|
||
a["resolvedByDecision"] = True
|
||
if d.get("genotype"):
|
||
# CR-10: validate the parsed genotype — a typo'd decision string yields empty
|
||
# mapped8locus and would silently blank the animal's genotype while marking it
|
||
# 'resolved'. Only apply if the parse produces non-empty loci.
|
||
parsed = gt.parse(d["genotype"])
|
||
if parsed.get("mapped8locus"):
|
||
a["genotype"] = parsed
|
||
else:
|
||
# Keep the existing genotype; flag as a warning in the report.
|
||
a.setdefault("decisionWarnings", []).append(
|
||
f"Ungültiger Override-Genotyp '{d['genotype']}' — "
|
||
"konnte nicht geparst werden (mapped8locus leer). "
|
||
"Bestehender Genotyp behalten; Konflikt wurde trotzdem aufgelöst."
|
||
)
|
||
if d.get("farbschlag"):
|
||
a["farbschlag"] = d["farbschlag"]
|
||
a["farbschlagVariants"] = [d["farbschlag"]]
|
||
if d.get("dateOfDeath"): # D5 death-date resolutions
|
||
a["death"] = norm_dob(d["dateOfDeath"])
|
||
if "father" in d or "mother" in d:
|
||
new_refs = []
|
||
if d.get("father"):
|
||
new_refs.append({
|
||
"name": d["father"],
|
||
"dob": "",
|
||
"roleGuess": "father",
|
||
"method": "decision",
|
||
"confidence": "high"
|
||
})
|
||
if d.get("mother"):
|
||
new_refs.append({
|
||
"name": d["mother"],
|
||
"dob": "",
|
||
"roleGuess": "mother",
|
||
"method": "decision",
|
||
"confidence": "high"
|
||
})
|
||
a["parentRefs"] = new_refs
|
||
if a.get("conflict"):
|
||
a["conflict"] = False
|
||
conflicts[:] = [c for c in conflicts if c.get("id") != a["id"]]
|
||
resolved += 1
|
||
return resolved
|
||
|
||
|
||
def main():
|
||
try:
|
||
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
||
except Exception:
|
||
pass
|
||
ap = argparse.ArgumentParser(description="FEAT-8b extractor (stages 1-2)")
|
||
ap.add_argument("--stammbaeume", default=DEFAULT_STAMMBAEUME)
|
||
ap.add_argument("--wurfchronik", default=DEFAULT_WURFCHRONIK)
|
||
args = ap.parse_args()
|
||
|
||
if os.path.isdir(OUT):
|
||
# keep the report stable across reruns but refresh data/photos
|
||
for sub in ("photos",):
|
||
p = os.path.join(OUT, sub)
|
||
if os.path.isdir(p):
|
||
shutil.rmtree(p)
|
||
os.makedirs(OUT, exist_ok=True)
|
||
|
||
raw_animals = []
|
||
# Skip Excel lock/owner files ("~$...") that appear while a workbook is open.
|
||
files = sorted(f for f in glob.glob(os.path.join(args.stammbaeume, "*.xlsx"))
|
||
if not os.path.basename(f).startswith("~$"))
|
||
print(f"Stammbaum-Dateien: {len(files)}")
|
||
for path in files:
|
||
got = extract_stammbaum(path)
|
||
print(f" {len(got):4d} {os.path.basename(path)}")
|
||
raw_animals.extend(got)
|
||
|
||
# Skip pseudo-animal records (like "DD-Tumor bei Geschwister") that are actually notes
|
||
raw_animals = [a for a in raw_animals if "DD-Tumor" not in a["name"]]
|
||
|
||
litters = []
|
||
if os.path.isfile(args.wurfchronik):
|
||
litters = extract_wurfchronik(args.wurfchronik)
|
||
print(f"Wurfchronik: {len(litters)} Würfe")
|
||
|
||
decisions_path = os.path.join(HERE, "conflict-decisions.json")
|
||
dob_remaps = apply_dob_remaps(raw_animals, decisions_path) # before dedup (changes identity)
|
||
merged, conflicts, orphans, zucht_splits = dedup(raw_animals)
|
||
resolved_by_decision = apply_conflict_decisions(merged, conflicts, decisions_path)
|
||
match_stats = match_litters(merged, litters)
|
||
photo_count = sum(len(a["photos"]) for a in merged)
|
||
|
||
# strip private (_) fields from the JSON output
|
||
def clean(a):
|
||
return {k: v for k, v in a.items() if not k.startswith("_")}
|
||
|
||
with open(os.path.join(OUT, "animals.json"), "w", encoding="utf-8") as f:
|
||
json.dump([clean(a) for a in merged], f, ensure_ascii=False, indent=2)
|
||
with open(os.path.join(OUT, "litters.json"), "w", encoding="utf-8") as f:
|
||
json.dump(litters, f, ensure_ascii=False, indent=2)
|
||
|
||
write_report(merged, conflicts, orphans, len(raw_animals), litters, photo_count,
|
||
zucht_splits, match_stats)
|
||
|
||
print(f"\nRoh: {len(raw_animals)} → eindeutig: {len(merged)} "
|
||
f"| Konflikte: {len(conflicts)} | per Entscheidung gelöst: {resolved_by_decision} "
|
||
f"| DOB-Remaps: {dob_remaps} | Zucht-Splits: {len(zucht_splits)} "
|
||
f"| Orphans: {len(orphans)} | Fotos: {photo_count}")
|
||
print(f"Wurf-Verknüpfung: {match_stats['parents']} (Datum+Eltern), "
|
||
f"{match_stats['dateOnly']} (nur Datum), {match_stats['ambiguous']} mehrdeutig "
|
||
f"| Wurf-Warnungen: {sum(1 for l in litters if l['warnings'])}")
|
||
print(f"Ausgabe in {OUT}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|