FEAT-8b: spreadsheet import tooling (stages 1-2) + review report
tools/import/ (Python, zero-dep migration tooling, not product code): - xlsx_util.py: dependency-free .xlsx reader (shared strings, cells, drawing anchors) - genotype.py: notation -> frozen 8-locus mapping + verbatim rawGenotype + unmappedTokens; '-' -> '?' - extract.py: 10 Stammbaum charts + Wurfchronik -> animals.json/litters.json + anchor-mapped photos; dedup on normalise(name)+DOB -> German review-report.md (no DB load) Run: 889 raw -> 587 unique animals, 24 conflicts, 310 ambiguous, 123 photos, 752 litters. Output gitignored except review-report.md. Re-runnable per file (Wurfchronik Teil2+). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
573
tools/import/extract.py
Normal file
573
tools/import/extract.py
Normal file
@@ -0,0 +1,573 @@
|
||||
#!/usr/bin/env python3
|
||||
"""FEAT-8b stages 1-2 — extract + dedup the GerbilManager source spreadsheets.
|
||||
|
||||
Stage 1: parse the 10 Stammbaum pedigree charts and the Wurfchronik litter
|
||||
chronicle into normalised animals.json + litters.json, and extract
|
||||
embedded photos (anchor-mapped to animals).
|
||||
Stage 2: dedup animals (key = normalise(name)+DOB, corroborated by DOB+genotype)
|
||||
and emit a German-friendly review-report.md for Julian's wife. No DB load.
|
||||
|
||||
Re-runnable per file (later Wurfchronik "Teil2+" / more charts just re-run).
|
||||
Migration tooling — Python, not product code. Zero third-party deps.
|
||||
|
||||
See FEAT-8a-format-spec.md (in Pam's hive workspace) for the format analysis.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import json
|
||||
import glob
|
||||
import shutil
|
||||
import argparse
|
||||
import unicodedata
|
||||
|
||||
import xlsx_util as xu
|
||||
import genotype as gt
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
DEFAULT_STAMMBAEUME = r"C:\Users\gulum\dev\Sttammbäume"
|
||||
DEFAULT_WURFCHRONIK = r"C:\Users\gulum\dev\Wurfchronik der Kleine Chaoten Teil1.xlsx"
|
||||
OUT = os.path.join(HERE, "output")
|
||||
|
||||
DOB = re.compile(r"\*\s?(\d{1,2}\.\d{1,2}\.(?:\d{4}|\d{2}))")
|
||||
DEATH = re.compile(r"\+\s?(\d{1,2}\.\d{1,2}\.(?:\d{4}|\d{2})|\d{4})")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- helpers ----
|
||||
def gen_of(colnum):
|
||||
"""Map a column number to a generation band (0=proband ... 5=deepest)."""
|
||||
if colnum <= 6:
|
||||
return 0 # E band (proband / "Kids")
|
||||
if colnum <= 9:
|
||||
return 1 # H band (parents)
|
||||
if colnum <= 12:
|
||||
return 2 # K band (grandparents)
|
||||
if colnum <= 15:
|
||||
return 3 # N band (great-grandparents)
|
||||
if colnum <= 17:
|
||||
return 4 # Q band (gg-grandparents)
|
||||
return 5 # R/S band (name-pairs)
|
||||
|
||||
|
||||
def norm_name(name):
|
||||
if not name:
|
||||
return ""
|
||||
n = name.lower()
|
||||
n = re.sub(r"\[.*?\]", " ", n) # drop [line] tags (Wurfchronik)
|
||||
n = re.sub(r"\bgen\.\b", " ", n) # "gen." nickname marker
|
||||
n = re.sub(r"\bv\.\s?d\.\b", " von den ", n)
|
||||
n = re.sub(r"\b(von der|von den|von|of)\b", " ", n) # cattery/line connectors
|
||||
n = unicodedata.normalize("NFKD", n)
|
||||
n = re.sub(r"[^a-z0-9äöüß]", "", n)
|
||||
return n
|
||||
|
||||
|
||||
def norm_dob(d):
|
||||
if not d:
|
||||
return ""
|
||||
p = d.split(".")
|
||||
if len(p) == 3 and len(p[2]) == 2:
|
||||
p[2] = "20" + p[2]
|
||||
return ".".join(x.zfill(2) if i < 2 else x for i, x in enumerate(p))
|
||||
|
||||
|
||||
def slug(name, dob):
|
||||
base = norm_name(name) or "unbekannt"
|
||||
d = norm_dob(dob).replace(".", "")
|
||||
return (base[:40] + ("-" + d if d else "")) or "unbekannt"
|
||||
|
||||
|
||||
def clean_name(raw):
|
||||
"""Strip detail/markers from a name cell, keep the human name + [line]."""
|
||||
n = raw.strip().strip(",").strip()
|
||||
return n
|
||||
|
||||
|
||||
# --------------------------------------------------- Stammbaum extraction ----
|
||||
def parse_detail(text):
|
||||
"""From a string that contains *DOB and/or genotype, pull (dob, death, geno_str).
|
||||
|
||||
For compact lines ("Name,*DOB[/+death], genotype") the genotype is everything
|
||||
after the date — we must NOT scan from the first locus-looking letter, or stray
|
||||
name words ("den", "of") get swallowed as genotype tokens.
|
||||
"""
|
||||
dob = DOB.search(text)
|
||||
death = DEATH.search(text)
|
||||
geno = ""
|
||||
if dob:
|
||||
tail = text[dob.end():]
|
||||
tail = re.sub(r"^\s*/?\+?\s?\d[\d.]*", "", tail) # drop any /+death remnant
|
||||
tail = tail.lstrip(" ,").strip()
|
||||
if gt.looks_like_genotype(tail):
|
||||
geno = tail
|
||||
return (dob.group(1) if dob else "",
|
||||
death.group(1) if death else "",
|
||||
geno)
|
||||
|
||||
|
||||
def extract_stammbaum(path):
|
||||
"""Return list of animal dicts for one chart file."""
|
||||
fname = os.path.basename(path)
|
||||
z = __import__("zipfile").ZipFile(path)
|
||||
ss = xu.shared_strings(z)
|
||||
sheets = xu.sheet_paths(z)
|
||||
cells = xu.read_cells(z, sheets[0], ss)
|
||||
|
||||
# group cells by column for block reconstruction
|
||||
by_col = {}
|
||||
for (c, r), t in cells.items():
|
||||
by_col.setdefault(c, []).append((r, t))
|
||||
for c in by_col:
|
||||
by_col[c].sort()
|
||||
|
||||
animals = []
|
||||
used = set()
|
||||
|
||||
for (c, r), t in sorted(cells.items()):
|
||||
if (c, r) in used:
|
||||
continue
|
||||
compact = re.match(r"^(.+?),\s*\*", t) # "Name,*DOB, genotype"
|
||||
is_block_dob = bool(re.match(r"^\*\s?\d", t)) # standalone "*DOB"
|
||||
if not compact and not is_block_dob:
|
||||
continue
|
||||
|
||||
if compact:
|
||||
name = clean_name(compact.group(1))
|
||||
dob, death, geno = parse_detail(t)
|
||||
farbschlag = ""
|
||||
breeder = ""
|
||||
used.add((c, r))
|
||||
else:
|
||||
# full block: name above, farbschlag/genotype/breeder below
|
||||
dob, death, geno0 = parse_detail(t)
|
||||
name = ""
|
||||
for rr in range(r - 1, r - 4, -1):
|
||||
if (c, rr) in cells and not re.match(r"^\*?\s?\d", cells[(c, rr)]) \
|
||||
and not gt.looks_like_genotype(cells[(c, rr)]):
|
||||
name = clean_name(cells[(c, rr)])
|
||||
used.add((c, rr))
|
||||
break
|
||||
farbschlag = ""
|
||||
geno = geno0
|
||||
breeder = ""
|
||||
for rr in range(r + 1, r + 4):
|
||||
cell = cells.get((c, rr))
|
||||
if not cell:
|
||||
continue
|
||||
if gt.looks_like_genotype(cell):
|
||||
geno = cell
|
||||
used.add((c, rr))
|
||||
elif re.search(r"\b(Zucht|Privatzucht)\b", cell) or cell.startswith("("):
|
||||
breeder = cell
|
||||
used.add((c, rr))
|
||||
elif not farbschlag and not re.match(r"^\*?\s?\d", cell):
|
||||
farbschlag = cell
|
||||
used.add((c, rr))
|
||||
used.add((c, r))
|
||||
|
||||
g = parse_detail(t) if compact else (dob, death, geno)
|
||||
genodict = gt.parse(geno)
|
||||
animals.append({
|
||||
"id": None, # assigned in dedup
|
||||
"name": name,
|
||||
"nameVariants": [],
|
||||
"dob": dob,
|
||||
"death": death,
|
||||
"gender": None,
|
||||
"farbschlag": farbschlag,
|
||||
"genotype": genodict,
|
||||
"breeder": breeder,
|
||||
"parentRefs": [],
|
||||
"photos": [],
|
||||
"sourceFiles": [fname],
|
||||
"_gen": gen_of(c),
|
||||
"_col": c,
|
||||
"_row": r,
|
||||
"_file": fname,
|
||||
})
|
||||
|
||||
# name-pair cells "X & Y" (deepest generation, names only)
|
||||
for (c, r), t in cells.items():
|
||||
if (c, r) in used:
|
||||
continue
|
||||
if " & " in t and not DOB.search(t) and len(t) < 90 and gen_of(c) >= 4:
|
||||
for part in t.split(" & "):
|
||||
part = clean_name(part)
|
||||
if part:
|
||||
animals.append({
|
||||
"id": None, "name": part, "nameVariants": [],
|
||||
"dob": "", "death": "", "gender": None, "farbschlag": "",
|
||||
"genotype": gt.parse(""), "breeder": "", "parentRefs": [],
|
||||
"photos": [], "sourceFiles": [fname],
|
||||
"_gen": gen_of(c), "_col": c, "_row": r, "_file": fname,
|
||||
})
|
||||
|
||||
_reconstruct_parents(animals)
|
||||
_attach_photos(z, sheets, animals, fname)
|
||||
return animals
|
||||
|
||||
|
||||
def _reconstruct_parents(animals):
|
||||
"""Positional: an animal's parents are the bracketing blocks one generation
|
||||
deeper (father = nearest block above, mother = nearest below). Role guess is
|
||||
by vertical position (German charts: Vater oben) — flagged for review; the
|
||||
Wurfchronik is authoritative for matched animals (Stage 3)."""
|
||||
by_gen = {}
|
||||
for a in animals:
|
||||
by_gen.setdefault(a["_gen"], []).append(a)
|
||||
for g, group in by_gen.items():
|
||||
nxt = sorted(by_gen.get(g + 1, []), key=lambda a: a["_row"])
|
||||
if not nxt:
|
||||
continue
|
||||
for a in group:
|
||||
r = a["_row"]
|
||||
above = [x for x in nxt if x["_row"] <= r]
|
||||
below = [x for x in nxt if x["_row"] > r]
|
||||
father = above[-1] if above else None
|
||||
mother = below[0] if below else None
|
||||
for parent, role in ((father, "father"), (mother, "mother")):
|
||||
if parent and parent["name"]:
|
||||
a["parentRefs"].append({
|
||||
"name": parent["name"],
|
||||
"dob": parent["dob"],
|
||||
"roleGuess": role,
|
||||
"method": "chart-position",
|
||||
"confidence": "medium",
|
||||
})
|
||||
|
||||
|
||||
def _attach_photos(z, sheets, animals, fname):
|
||||
anchors = [a for a in xu.image_anchors(z)]
|
||||
if not anchors:
|
||||
return
|
||||
by_gen = {}
|
||||
for a in animals:
|
||||
by_gen.setdefault(a["_gen"], []).append(a)
|
||||
media_dir = os.path.join(OUT, "photos")
|
||||
for i, (sp, col, row, media) in enumerate(anchors):
|
||||
g = gen_of(col)
|
||||
cands = by_gen.get(g, [])
|
||||
if not cands:
|
||||
# fall back to nearest animal by row across all gens
|
||||
cands = animals
|
||||
target = min(cands, key=lambda a: abs(a["_row"] - row)) if cands else None
|
||||
if not target:
|
||||
continue
|
||||
ext = os.path.splitext(media)[1] or ".img"
|
||||
sl = slug(target["name"], target["dob"])
|
||||
dest_dir = os.path.join(media_dir, sl)
|
||||
os.makedirs(dest_dir, exist_ok=True)
|
||||
rel = f"photos/{sl}/{os.path.basename(media)}"
|
||||
try:
|
||||
with z.open(media) as src, open(os.path.join(OUT, rel), "wb") as dst:
|
||||
shutil.copyfileobj(src, dst)
|
||||
target["photos"].append(rel)
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
|
||||
# -------------------------------------------------- Wurfchronik extraction ---
|
||||
def extract_wurfchronik(path):
|
||||
"""Return list of litter dicts. Parses columns BY HEADER (sheets differ)."""
|
||||
fname = os.path.basename(path)
|
||||
z = __import__("zipfile").ZipFile(path)
|
||||
ss = xu.shared_strings(z)
|
||||
litters = []
|
||||
for sp in xu.sheet_paths(z):
|
||||
cells = xu.read_cells(z, sp, ss)
|
||||
if not cells:
|
||||
continue
|
||||
# build row -> {colnum: text}
|
||||
rows = {}
|
||||
for (c, r), t in cells.items():
|
||||
rows.setdefault(r, {})[c] = t
|
||||
hdr = xu.header_row(cells) # colnum -> header label
|
||||
hdr_row = min(r for (_, r) in cells) # the header row number, to skip it
|
||||
# map header label -> colnum (fuzzy by keyword)
|
||||
def find(*keys):
|
||||
for c, lbl in hdr.items():
|
||||
low = lbl.lower()
|
||||
if any(k in low for k in keys):
|
||||
return c
|
||||
return None
|
||||
col_id = find("wurfbuchstabe", "buchstabe")
|
||||
col_date = find("geburtsdatum", "datum")
|
||||
col_dam = find("mutter")
|
||||
col_sire = find("vater")
|
||||
col_ws = find("ws", "wurfstärke", "wurfstaerke")
|
||||
col_breakdown = find("männchen", "maennchen", "weibchen")
|
||||
col_zn = find("zuchtnummer")
|
||||
col_note = find("bemerkung")
|
||||
sheet_name = os.path.basename(sp)
|
||||
for r in sorted(rows):
|
||||
if r == hdr_row: # skip the header row itself
|
||||
continue
|
||||
row = rows[r]
|
||||
# skip empty-id + "Jahr YYYY" section rows
|
||||
txt_b = row.get(col_date, "") if col_date else ""
|
||||
if not row.get(col_id):
|
||||
continue
|
||||
if "jahr" in " ".join(row.values()).lower() and not DOB.search(txt_b):
|
||||
continue
|
||||
dob = DOB.search(txt_b)
|
||||
bd = row.get(col_breakdown, "") if col_breakdown else ""
|
||||
m = re.findall(r"\d+", bd)
|
||||
breakdown = {}
|
||||
if len(m) >= 1:
|
||||
keys = ["maennchen", "weibchen", "totgeburt", "s"]
|
||||
for k, val in zip(keys, m):
|
||||
breakdown[k] = int(val)
|
||||
lid = row.get(col_id, "")
|
||||
datestr = dob.group(1) if dob else ""
|
||||
litters.append({
|
||||
"id": f"{sheet_name.replace('.xml','')}-{lid}-{norm_dob(datestr)}",
|
||||
"litterId": lid,
|
||||
"date": datestr,
|
||||
"damName": row.get(col_dam, "") if col_dam else "",
|
||||
"sireName": row.get(col_sire, "") if col_sire else "",
|
||||
"wurfstaerke": _to_int(row.get(col_ws)) if col_ws else None,
|
||||
"sexBreakdown": breakdown,
|
||||
"zuchtnummer": row.get(col_zn, "") if col_zn else "",
|
||||
"note": row.get(col_note, "") if col_note else "",
|
||||
"sourceFile": fname,
|
||||
"sheet": sheet_name,
|
||||
"row": r,
|
||||
})
|
||||
return litters
|
||||
|
||||
|
||||
def _to_int(s):
|
||||
if not s:
|
||||
return None
|
||||
m = re.search(r"\d+", s)
|
||||
return int(m.group()) if m else None
|
||||
|
||||
|
||||
# ------------------------------------------------------------- stage 2: dedup
|
||||
def dedup(animals):
|
||||
"""Merge by normalise(name)+DOB. Returns (merged, conflicts, orphans)."""
|
||||
groups = {}
|
||||
orphans = []
|
||||
for a in animals:
|
||||
key = (norm_name(a["name"]), norm_dob(a["dob"]))
|
||||
if not key[0] or not key[1]:
|
||||
orphans.append(a)
|
||||
# orphans still get a stable id but are not merged
|
||||
key = ("__orphan__", id(a))
|
||||
groups.setdefault(key, []).append(a)
|
||||
|
||||
merged = []
|
||||
conflicts = []
|
||||
for key, grp in groups.items():
|
||||
base = dict(grp[0])
|
||||
variants = set([base["name"]])
|
||||
files = set(base["sourceFiles"])
|
||||
photos = list(base["photos"])
|
||||
parent_refs = list(base["parentRefs"])
|
||||
genos = set()
|
||||
farb = set()
|
||||
deaths = set()
|
||||
for a in grp:
|
||||
variants.add(a["name"])
|
||||
files.update(a["sourceFiles"])
|
||||
photos.extend(a["photos"])
|
||||
parent_refs.extend(a["parentRefs"])
|
||||
if a["genotype"]["rawGenotype"]:
|
||||
genos.add(a["genotype"]["rawGenotype"])
|
||||
if a["farbschlag"]:
|
||||
farb.add(a["farbschlag"])
|
||||
if a["death"]:
|
||||
deaths.add(norm_dob(a["death"]))
|
||||
# pick the richest genotype (most mapped loci, then longest raw)
|
||||
best = max((a["genotype"] for a in grp),
|
||||
key=lambda gd: (len(gd["mapped8locus"]), len(gd["rawGenotype"])))
|
||||
out = {
|
||||
"id": slug(base["name"], base["dob"]),
|
||||
"name": base["name"],
|
||||
"nameVariants": sorted(v for v in variants if v),
|
||||
"dob": norm_dob(base["dob"]),
|
||||
"death": sorted(deaths)[0] if deaths else "",
|
||||
"gender": None,
|
||||
"farbschlag": sorted(farb)[0] if farb else "",
|
||||
"farbschlagVariants": sorted(farb),
|
||||
"genotype": best,
|
||||
"breeder": next((a["breeder"] for a in grp if a["breeder"]), ""),
|
||||
"parentRefs": _dedup_parentrefs(parent_refs),
|
||||
"photos": sorted(set(photos)),
|
||||
"sourceFiles": sorted(files),
|
||||
"mentions": len(grp),
|
||||
}
|
||||
merged.append(out)
|
||||
# conflict: same animal, disagreeing genotype or farbschlag or death
|
||||
if len(genos) > 1 or len(farb) > 1 or len(deaths) > 1:
|
||||
conflicts.append({
|
||||
"id": out["id"], "name": base["name"], "dob": out["dob"],
|
||||
"genotypes": sorted(genos), "farbschlaege": sorted(farb),
|
||||
"deaths": sorted(deaths), "files": sorted(files),
|
||||
})
|
||||
merged.sort(key=lambda a: (a["dob"], a["name"]))
|
||||
return merged, conflicts, orphans
|
||||
|
||||
|
||||
def _dedup_parentrefs(refs):
|
||||
seen = {}
|
||||
for r in refs:
|
||||
k = (norm_name(r["name"]), r["roleGuess"])
|
||||
if k not in seen:
|
||||
seen[k] = r
|
||||
return list(seen.values())
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ reporting
|
||||
def write_report(merged, conflicts, orphans, raw_count, litters, photo_count):
|
||||
keyset = set((a["dob"], norm_name(a["name"])) for a in merged)
|
||||
multi = [a for a in merged if a["mentions"] > 1]
|
||||
with_dob = [a for a in merged if a["dob"]]
|
||||
lit_dates = set(norm_dob(l["date"]) for l in litters if l["date"])
|
||||
joinable = [a for a in merged if a["dob"] and norm_dob(a["dob"]) in lit_dates]
|
||||
|
||||
L = []
|
||||
L.append("# FEAT-8b — Import-Vorschau & Prüfbericht (Stammbäume + Wurfchronik)\n")
|
||||
L.append("_Automatisch erzeugt von `tools/import/extract.py` — **noch nichts in die Datenbank geladen.** "
|
||||
"Bitte prüfen, bevor importiert wird._\n")
|
||||
L.append("## Überblick\n")
|
||||
L.append(f"- Rohe Tier-Einträge aus den Stammbäumen: **{raw_count}**")
|
||||
L.append(f"- Nach Zusammenführung (eindeutige Tiere): **{len(merged)}**")
|
||||
L.append(f" - davon mit Geburtsdatum: {len(with_dob)}")
|
||||
L.append(f" - in mehreren Dateien gefunden (Dubletten zusammengeführt): {len(multi)}")
|
||||
L.append(f"- Konflikte zur Klärung: **{len(conflicts)}**")
|
||||
L.append(f"- Mehrdeutige / unvollständige Einträge (ohne Name+Datum): **{len(orphans)}**")
|
||||
L.append(f"- Fotos zugeordnet: **{photo_count}**")
|
||||
L.append(f"- Würfe aus der Wurfchronik: **{len(litters)}**")
|
||||
L.append(f" - Tiere, deren Geburtsdatum zu einem Wurf passt (verknüpfbar): {len(joinable)}\n")
|
||||
|
||||
L.append("## Zusammenführungs-Schlüssel\n")
|
||||
L.append("Tiere wurden zusammengeführt über **normalisierter Name + Geburtsdatum**. "
|
||||
"Namensvarianten (z. B. `v.d.` ↔ `von den`, `gen.`-Spitznamen, Zuchtsuffixe) "
|
||||
"werden als `nameVariants` erhalten.\n")
|
||||
|
||||
L.append("## ⚠️ Konflikte (bitte prüfen)\n")
|
||||
if conflicts:
|
||||
L.append("Gleiches Tier (Name+Datum), aber widersprüchliche Angaben in verschiedenen Dateien:\n")
|
||||
L.append("| Tier | Geburtsdatum | abweichende Genotypen | abweichende Farbschläge | Sterbedaten | Dateien |")
|
||||
L.append("|---|---|---|---|---|---|")
|
||||
for c in conflicts[:200]:
|
||||
L.append("| {} | {} | {} | {} | {} | {} |".format(
|
||||
c["name"], c["dob"],
|
||||
" // ".join(c["genotypes"]) or "—",
|
||||
" // ".join(c["farbschlaege"]) or "—",
|
||||
" // ".join(c["deaths"]) or "—",
|
||||
", ".join(os.path.splitext(f)[0] for f in c["files"])))
|
||||
else:
|
||||
L.append("_Keine._\n")
|
||||
|
||||
L.append("\n## Mehrdeutige / unvollständige Einträge\n")
|
||||
L.append(f"{len(orphans)} Einträge ohne sichere Name+Datum-Kombination "
|
||||
"(z. B. `Name1 & Name2`-Paarzellen der tiefsten Generation, oder Zellen ohne Datum). "
|
||||
"Diese werden NICHT automatisch zusammengeführt.\n")
|
||||
sample = [o for o in orphans if o["name"]][:40]
|
||||
for o in sample:
|
||||
L.append(f"- {o['name']} · {o.get('_file','')}")
|
||||
|
||||
# orphan -> likely same-named full record (soft hint, not auto-merged)
|
||||
from collections import Counter
|
||||
name_index = {}
|
||||
for a in merged:
|
||||
if a["dob"]:
|
||||
name_index.setdefault(norm_name(a["name"]), []).append(a)
|
||||
matchable = []
|
||||
for o in orphans:
|
||||
if not o["name"]:
|
||||
continue
|
||||
cands = name_index.get(norm_name(o["name"]))
|
||||
if cands:
|
||||
matchable.append((o, cands))
|
||||
L.append("\n## Wahrscheinliche Zuordnungen unvollständiger Einträge\n")
|
||||
L.append(f"{len(matchable)} namenlose/datenlose Einträge tragen denselben Namen wie ein "
|
||||
"vollständiges Tier — vermutlich dasselbe Tier (zur Bestätigung):\n")
|
||||
for o, cands in matchable[:60]:
|
||||
opts = "; ".join(f"{c['name']} (*{c['dob']})" for c in cands[:3])
|
||||
L.append(f"- „{o['name']}“ → {opts}")
|
||||
|
||||
# unmapped-token summary (for Kevin / GEN-2 + the wife)
|
||||
tok = Counter()
|
||||
for a in merged:
|
||||
for t in a["genotype"]["unmappedTokens"]:
|
||||
tok[t] += 1
|
||||
L.append("\n## Nicht ins 8-Loci-Modell abgebildete Tokens (verbatim erhalten)\n")
|
||||
L.append("Diese Tokens stehen weiter in `rawGenotype`/`unmappedTokens` — Entscheidung "
|
||||
"(Modell erweitern vs. als Notiz) liegt bei Julian/Kevin:\n")
|
||||
L.append("| Token | Vorkommen | Bedeutung (Vermutung) |")
|
||||
L.append("|---|---|---|")
|
||||
hint = {"Uwuw[d]": "9. Locus Uw (nicht im Modell)", "UwUw": "9. Locus Uw",
|
||||
"[WFNZ]": "Marker", "[DP]": "Marker (Dunkelpigment?)", "DP": "Marker",
|
||||
"WP": "Marker", "[WP]": "Marker", "C(C)": "Schreibweise (C trägt c)",
|
||||
"chmchm": "Schreibweise (c[chm]c[chm])"}
|
||||
for t, n in tok.most_common(25):
|
||||
L.append(f"| `{t}` | {n} | {hint.get(t, '?')} |")
|
||||
|
||||
L.append("\n## Hinweise für den Import (Stufe 3, später)\n")
|
||||
L.append("- **Wurfchronik = Quelle der Würfe** (Datum, Wurfstärke, Eltern, Zuchtnummer); "
|
||||
"**Stammbäume = Abstammung + Genotyp + Fotos**. Verknüpfung über Geburtsdatum + Elternnamen.")
|
||||
L.append("- Eltern-Verknüpfungen (`parentRefs`) stammen aus der **Position im Stammbaum** "
|
||||
"(Vater oben / Mutter unten, mittlere Konfidenz) — die Wurfchronik korrigiert dies maßgeblich.")
|
||||
L.append("- Genotyp: `mapped8locus` (A C D E G P Sp Re), `rawGenotype` (wortgetreu), "
|
||||
"`unmappedTokens` (z. B. `Uw`, `Sls`, `Dea`, Marker wie `WFNZ/WP/DP`) — **nichts geht verloren**.")
|
||||
L.append("- `-` (unbekanntes zweites Allel) → `?` (Platzhalter; Annahme, bitte bestätigen).")
|
||||
with open(os.path.join(OUT, "review-report.md"), "w", encoding="utf-8") as f:
|
||||
f.write("\n".join(L) + "\n")
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------ main
|
||||
def main():
|
||||
try:
|
||||
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
||||
except Exception:
|
||||
pass
|
||||
ap = argparse.ArgumentParser(description="FEAT-8b extractor (stages 1-2)")
|
||||
ap.add_argument("--stammbaeume", default=DEFAULT_STAMMBAEUME)
|
||||
ap.add_argument("--wurfchronik", default=DEFAULT_WURFCHRONIK)
|
||||
args = ap.parse_args()
|
||||
|
||||
if os.path.isdir(OUT):
|
||||
# keep the report stable across reruns but refresh data/photos
|
||||
for sub in ("photos",):
|
||||
p = os.path.join(OUT, sub)
|
||||
if os.path.isdir(p):
|
||||
shutil.rmtree(p)
|
||||
os.makedirs(OUT, exist_ok=True)
|
||||
|
||||
raw_animals = []
|
||||
files = sorted(glob.glob(os.path.join(args.stammbaeume, "*.xlsx")))
|
||||
print(f"Stammbaum-Dateien: {len(files)}")
|
||||
for path in files:
|
||||
got = extract_stammbaum(path)
|
||||
print(f" {len(got):4d} {os.path.basename(path)}")
|
||||
raw_animals.extend(got)
|
||||
|
||||
litters = []
|
||||
if os.path.isfile(args.wurfchronik):
|
||||
litters = extract_wurfchronik(args.wurfchronik)
|
||||
print(f"Wurfchronik: {len(litters)} Würfe")
|
||||
|
||||
merged, conflicts, orphans = dedup(raw_animals)
|
||||
photo_count = sum(len(a["photos"]) for a in merged)
|
||||
|
||||
# strip private (_) fields from the JSON output
|
||||
def clean(a):
|
||||
return {k: v for k, v in a.items() if not k.startswith("_")}
|
||||
|
||||
with open(os.path.join(OUT, "animals.json"), "w", encoding="utf-8") as f:
|
||||
json.dump([clean(a) for a in merged], f, ensure_ascii=False, indent=2)
|
||||
with open(os.path.join(OUT, "litters.json"), "w", encoding="utf-8") as f:
|
||||
json.dump(litters, f, ensure_ascii=False, indent=2)
|
||||
|
||||
write_report(merged, conflicts, orphans, len(raw_animals), litters, photo_count)
|
||||
|
||||
print(f"\nRoh: {len(raw_animals)} → eindeutig: {len(merged)} "
|
||||
f"| Konflikte: {len(conflicts)} | Orphans: {len(orphans)} | Fotos: {photo_count}")
|
||||
print(f"Ausgabe in {OUT}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user