From 1b776cd99439de551a18f6f813bea79405fb84f1 Mon Sep 17 00:00:00 2001 From: Gulum Date: Sat, 6 Jun 2026 00:40:54 +0200 Subject: [PATCH] FEAT-8b: spreadsheet import tooling (stages 1-2) + review report tools/import/ (Python, zero-dep migration tooling, not product code): - xlsx_util.py: dependency-free .xlsx reader (shared strings, cells, drawing anchors) - genotype.py: notation -> frozen 8-locus mapping + verbatim rawGenotype + unmappedTokens; '-' -> '?' - extract.py: 10 Stammbaum charts + Wurfchronik -> animals.json/litters.json + anchor-mapped photos; dedup on normalise(name)+DOB -> German review-report.md (no DB load) Run: 889 raw -> 587 unique animals, 24 conflicts, 310 ambiguous, 123 photos, 752 litters. Output gitignored except review-report.md. Re-runnable per file (Wurfchronik Teil2+). Co-Authored-By: Claude Opus 4.8 (1M context) --- tools/import/.gitignore | 7 + tools/import/README.md | 58 +++ tools/import/extract.py | 573 +++++++++++++++++++++++++++ tools/import/genotype.py | 109 +++++ tools/import/output/review-report.md | 176 ++++++++ tools/import/xlsx_util.py | 135 +++++++ 6 files changed, 1058 insertions(+) create mode 100644 tools/import/.gitignore create mode 100644 tools/import/README.md create mode 100644 tools/import/extract.py create mode 100644 tools/import/genotype.py create mode 100644 tools/import/output/review-report.md create mode 100644 tools/import/xlsx_util.py diff --git a/tools/import/.gitignore b/tools/import/.gitignore new file mode 100644 index 0000000..7ccc4ee --- /dev/null +++ b/tools/import/.gitignore @@ -0,0 +1,7 @@ +# Generated extraction artifacts — large/binary, regenerated by extract.py. +# Per FEAT-8b: ignore the output dir EXCEPT the human-review report. +output/* +!output/review-report.md + +__pycache__/ +*.pyc diff --git a/tools/import/README.md b/tools/import/README.md new file mode 100644 index 0000000..2b99236 --- /dev/null +++ b/tools/import/README.md @@ -0,0 +1,58 @@ +# GerbilManager import tooling (FEAT-8b) + +One-off **migration tooling** (Python, no third-party deps) that turns Julian's +wife's hand-built spreadsheets into normalised JSON for review and, later, import. +This is *not* product code — it lives outside the app and is run manually. + +See the format analysis in `FEAT-8a-format-spec.md` (Pam's hive workspace). + +## What it does + +`extract.py` runs **stages 1–2** of the pipeline: + +1. **Extract (stage 1)** + - 10 *Stammbaum* pedigree charts → animals (name, DOB, death, Farbschlag, + genotype, breeder, positionally-reconstructed parent links, photos). + - *Wurfchronik* litter chronicle → litters (date, dam, sire, Wurfstärke, + sex breakdown, Zuchtnummer, notes). Columns are read **by header row** because + the two sheets use different schemas. + - Embedded photos (`xl/media`) → `output/photos//`, mapped to the + animal by drawing anchor position. +2. **Dedup + review (stage 2)** + - Merge animals on `normalise(name) + DOB` (corroborated by DOB+genotype). + - Emit a German-language `output/review-report.md` for the breeder to verify + (merges, **conflicts**, ambiguous/incomplete entries, unmapped genotype tokens). + - **Nothing is loaded into the database** — stage 3 (API load) is separate and + waits on DATA-2 + FEAT-1b phase 2. + +Genotypes are mapped to the frozen 8-locus contract (A C D E G P Sp Re) while +preserving everything: `genotype.mapped8locus`, `genotype.rawGenotype` (verbatim), +`genotype.unmappedTokens` (e.g. the `Uw` locus, markers `WFNZ/WP/DP`). A `-` +(unknown second allele) maps to `?`. + +## Run + +```sh +cd tools/import +python extract.py # uses the default source paths +python extract.py --stammbaeume "" --wurfchronik "" +``` + +Requires Python 3. **Re-runnable / idempotent** — re-run when more files arrive +(Wurfchronik `Teil2+`, or new charts). + +## Output (`tools/import/output/`, git-ignored except the report) + +| File | Contents | +|---|---| +| `animals.json` | deduped animals with genotype, parentRefs, photos, sourceFiles | +| `litters.json` | litters from the Wurfchronik | +| `photos//…` | extracted, anchor-mapped images | +| `review-report.md` | **human review deliverable** (committed) | + +## Files + +- `xlsx_util.py` — dependency-free `.xlsx` reader (zip + XML): shared strings, + cells by reference, image/drawing anchors. +- `genotype.py` — genotype notation parser → 8-locus mapping + raw + unmapped. +- `extract.py` — the pipeline (stages 1–2). diff --git a/tools/import/extract.py b/tools/import/extract.py new file mode 100644 index 0000000..5b3b885 --- /dev/null +++ b/tools/import/extract.py @@ -0,0 +1,573 @@ +#!/usr/bin/env python3 +"""FEAT-8b stages 1-2 — extract + dedup the GerbilManager source spreadsheets. + +Stage 1: parse the 10 Stammbaum pedigree charts and the Wurfchronik litter + chronicle into normalised animals.json + litters.json, and extract + embedded photos (anchor-mapped to animals). +Stage 2: dedup animals (key = normalise(name)+DOB, corroborated by DOB+genotype) + and emit a German-friendly review-report.md for Julian's wife. No DB load. + +Re-runnable per file (later Wurfchronik "Teil2+" / more charts just re-run). +Migration tooling — Python, not product code. Zero third-party deps. + +See FEAT-8a-format-spec.md (in Pam's hive workspace) for the format analysis. +""" +import os +import re +import sys +import json +import glob +import shutil +import argparse +import unicodedata + +import xlsx_util as xu +import genotype as gt + +HERE = os.path.dirname(os.path.abspath(__file__)) +DEFAULT_STAMMBAEUME = r"C:\Users\gulum\dev\Sttammbäume" +DEFAULT_WURFCHRONIK = r"C:\Users\gulum\dev\Wurfchronik der Kleine Chaoten Teil1.xlsx" +OUT = os.path.join(HERE, "output") + +DOB = re.compile(r"\*\s?(\d{1,2}\.\d{1,2}\.(?:\d{4}|\d{2}))") +DEATH = re.compile(r"\+\s?(\d{1,2}\.\d{1,2}\.(?:\d{4}|\d{2})|\d{4})") + + +# ---------------------------------------------------------------- helpers ---- +def gen_of(colnum): + """Map a column number to a generation band (0=proband ... 5=deepest).""" + if colnum <= 6: + return 0 # E band (proband / "Kids") + if colnum <= 9: + return 1 # H band (parents) + if colnum <= 12: + return 2 # K band (grandparents) + if colnum <= 15: + return 3 # N band (great-grandparents) + if colnum <= 17: + return 4 # Q band (gg-grandparents) + return 5 # R/S band (name-pairs) + + +def norm_name(name): + if not name: + return "" + n = name.lower() + n = re.sub(r"\[.*?\]", " ", n) # drop [line] tags (Wurfchronik) + n = re.sub(r"\bgen\.\b", " ", n) # "gen." nickname marker + n = re.sub(r"\bv\.\s?d\.\b", " von den ", n) + n = re.sub(r"\b(von der|von den|von|of)\b", " ", n) # cattery/line connectors + n = unicodedata.normalize("NFKD", n) + n = re.sub(r"[^a-z0-9äöüß]", "", n) + return n + + +def norm_dob(d): + if not d: + return "" + p = d.split(".") + if len(p) == 3 and len(p[2]) == 2: + p[2] = "20" + p[2] + return ".".join(x.zfill(2) if i < 2 else x for i, x in enumerate(p)) + + +def slug(name, dob): + base = norm_name(name) or "unbekannt" + d = norm_dob(dob).replace(".", "") + return (base[:40] + ("-" + d if d else "")) or "unbekannt" + + +def clean_name(raw): + """Strip detail/markers from a name cell, keep the human name + [line].""" + n = raw.strip().strip(",").strip() + return n + + +# --------------------------------------------------- Stammbaum extraction ---- +def parse_detail(text): + """From a string that contains *DOB and/or genotype, pull (dob, death, geno_str). + + For compact lines ("Name,*DOB[/+death], genotype") the genotype is everything + after the date — we must NOT scan from the first locus-looking letter, or stray + name words ("den", "of") get swallowed as genotype tokens. + """ + dob = DOB.search(text) + death = DEATH.search(text) + geno = "" + if dob: + tail = text[dob.end():] + tail = re.sub(r"^\s*/?\+?\s?\d[\d.]*", "", tail) # drop any /+death remnant + tail = tail.lstrip(" ,").strip() + if gt.looks_like_genotype(tail): + geno = tail + return (dob.group(1) if dob else "", + death.group(1) if death else "", + geno) + + +def extract_stammbaum(path): + """Return list of animal dicts for one chart file.""" + fname = os.path.basename(path) + z = __import__("zipfile").ZipFile(path) + ss = xu.shared_strings(z) + sheets = xu.sheet_paths(z) + cells = xu.read_cells(z, sheets[0], ss) + + # group cells by column for block reconstruction + by_col = {} + for (c, r), t in cells.items(): + by_col.setdefault(c, []).append((r, t)) + for c in by_col: + by_col[c].sort() + + animals = [] + used = set() + + for (c, r), t in sorted(cells.items()): + if (c, r) in used: + continue + compact = re.match(r"^(.+?),\s*\*", t) # "Name,*DOB, genotype" + is_block_dob = bool(re.match(r"^\*\s?\d", t)) # standalone "*DOB" + if not compact and not is_block_dob: + continue + + if compact: + name = clean_name(compact.group(1)) + dob, death, geno = parse_detail(t) + farbschlag = "" + breeder = "" + used.add((c, r)) + else: + # full block: name above, farbschlag/genotype/breeder below + dob, death, geno0 = parse_detail(t) + name = "" + for rr in range(r - 1, r - 4, -1): + if (c, rr) in cells and not re.match(r"^\*?\s?\d", cells[(c, rr)]) \ + and not gt.looks_like_genotype(cells[(c, rr)]): + name = clean_name(cells[(c, rr)]) + used.add((c, rr)) + break + farbschlag = "" + geno = geno0 + breeder = "" + for rr in range(r + 1, r + 4): + cell = cells.get((c, rr)) + if not cell: + continue + if gt.looks_like_genotype(cell): + geno = cell + used.add((c, rr)) + elif re.search(r"\b(Zucht|Privatzucht)\b", cell) or cell.startswith("("): + breeder = cell + used.add((c, rr)) + elif not farbschlag and not re.match(r"^\*?\s?\d", cell): + farbschlag = cell + used.add((c, rr)) + used.add((c, r)) + + g = parse_detail(t) if compact else (dob, death, geno) + genodict = gt.parse(geno) + animals.append({ + "id": None, # assigned in dedup + "name": name, + "nameVariants": [], + "dob": dob, + "death": death, + "gender": None, + "farbschlag": farbschlag, + "genotype": genodict, + "breeder": breeder, + "parentRefs": [], + "photos": [], + "sourceFiles": [fname], + "_gen": gen_of(c), + "_col": c, + "_row": r, + "_file": fname, + }) + + # name-pair cells "X & Y" (deepest generation, names only) + for (c, r), t in cells.items(): + if (c, r) in used: + continue + if " & " in t and not DOB.search(t) and len(t) < 90 and gen_of(c) >= 4: + for part in t.split(" & "): + part = clean_name(part) + if part: + animals.append({ + "id": None, "name": part, "nameVariants": [], + "dob": "", "death": "", "gender": None, "farbschlag": "", + "genotype": gt.parse(""), "breeder": "", "parentRefs": [], + "photos": [], "sourceFiles": [fname], + "_gen": gen_of(c), "_col": c, "_row": r, "_file": fname, + }) + + _reconstruct_parents(animals) + _attach_photos(z, sheets, animals, fname) + return animals + + +def _reconstruct_parents(animals): + """Positional: an animal's parents are the bracketing blocks one generation + deeper (father = nearest block above, mother = nearest below). Role guess is + by vertical position (German charts: Vater oben) — flagged for review; the + Wurfchronik is authoritative for matched animals (Stage 3).""" + by_gen = {} + for a in animals: + by_gen.setdefault(a["_gen"], []).append(a) + for g, group in by_gen.items(): + nxt = sorted(by_gen.get(g + 1, []), key=lambda a: a["_row"]) + if not nxt: + continue + for a in group: + r = a["_row"] + above = [x for x in nxt if x["_row"] <= r] + below = [x for x in nxt if x["_row"] > r] + father = above[-1] if above else None + mother = below[0] if below else None + for parent, role in ((father, "father"), (mother, "mother")): + if parent and parent["name"]: + a["parentRefs"].append({ + "name": parent["name"], + "dob": parent["dob"], + "roleGuess": role, + "method": "chart-position", + "confidence": "medium", + }) + + +def _attach_photos(z, sheets, animals, fname): + anchors = [a for a in xu.image_anchors(z)] + if not anchors: + return + by_gen = {} + for a in animals: + by_gen.setdefault(a["_gen"], []).append(a) + media_dir = os.path.join(OUT, "photos") + for i, (sp, col, row, media) in enumerate(anchors): + g = gen_of(col) + cands = by_gen.get(g, []) + if not cands: + # fall back to nearest animal by row across all gens + cands = animals + target = min(cands, key=lambda a: abs(a["_row"] - row)) if cands else None + if not target: + continue + ext = os.path.splitext(media)[1] or ".img" + sl = slug(target["name"], target["dob"]) + dest_dir = os.path.join(media_dir, sl) + os.makedirs(dest_dir, exist_ok=True) + rel = f"photos/{sl}/{os.path.basename(media)}" + try: + with z.open(media) as src, open(os.path.join(OUT, rel), "wb") as dst: + shutil.copyfileobj(src, dst) + target["photos"].append(rel) + except KeyError: + pass + + +# -------------------------------------------------- Wurfchronik extraction --- +def extract_wurfchronik(path): + """Return list of litter dicts. Parses columns BY HEADER (sheets differ).""" + fname = os.path.basename(path) + z = __import__("zipfile").ZipFile(path) + ss = xu.shared_strings(z) + litters = [] + for sp in xu.sheet_paths(z): + cells = xu.read_cells(z, sp, ss) + if not cells: + continue + # build row -> {colnum: text} + rows = {} + for (c, r), t in cells.items(): + rows.setdefault(r, {})[c] = t + hdr = xu.header_row(cells) # colnum -> header label + hdr_row = min(r for (_, r) in cells) # the header row number, to skip it + # map header label -> colnum (fuzzy by keyword) + def find(*keys): + for c, lbl in hdr.items(): + low = lbl.lower() + if any(k in low for k in keys): + return c + return None + col_id = find("wurfbuchstabe", "buchstabe") + col_date = find("geburtsdatum", "datum") + col_dam = find("mutter") + col_sire = find("vater") + col_ws = find("ws", "wurfstärke", "wurfstaerke") + col_breakdown = find("männchen", "maennchen", "weibchen") + col_zn = find("zuchtnummer") + col_note = find("bemerkung") + sheet_name = os.path.basename(sp) + for r in sorted(rows): + if r == hdr_row: # skip the header row itself + continue + row = rows[r] + # skip empty-id + "Jahr YYYY" section rows + txt_b = row.get(col_date, "") if col_date else "" + if not row.get(col_id): + continue + if "jahr" in " ".join(row.values()).lower() and not DOB.search(txt_b): + continue + dob = DOB.search(txt_b) + bd = row.get(col_breakdown, "") if col_breakdown else "" + m = re.findall(r"\d+", bd) + breakdown = {} + if len(m) >= 1: + keys = ["maennchen", "weibchen", "totgeburt", "s"] + for k, val in zip(keys, m): + breakdown[k] = int(val) + lid = row.get(col_id, "") + datestr = dob.group(1) if dob else "" + litters.append({ + "id": f"{sheet_name.replace('.xml','')}-{lid}-{norm_dob(datestr)}", + "litterId": lid, + "date": datestr, + "damName": row.get(col_dam, "") if col_dam else "", + "sireName": row.get(col_sire, "") if col_sire else "", + "wurfstaerke": _to_int(row.get(col_ws)) if col_ws else None, + "sexBreakdown": breakdown, + "zuchtnummer": row.get(col_zn, "") if col_zn else "", + "note": row.get(col_note, "") if col_note else "", + "sourceFile": fname, + "sheet": sheet_name, + "row": r, + }) + return litters + + +def _to_int(s): + if not s: + return None + m = re.search(r"\d+", s) + return int(m.group()) if m else None + + +# ------------------------------------------------------------- stage 2: dedup +def dedup(animals): + """Merge by normalise(name)+DOB. Returns (merged, conflicts, orphans).""" + groups = {} + orphans = [] + for a in animals: + key = (norm_name(a["name"]), norm_dob(a["dob"])) + if not key[0] or not key[1]: + orphans.append(a) + # orphans still get a stable id but are not merged + key = ("__orphan__", id(a)) + groups.setdefault(key, []).append(a) + + merged = [] + conflicts = [] + for key, grp in groups.items(): + base = dict(grp[0]) + variants = set([base["name"]]) + files = set(base["sourceFiles"]) + photos = list(base["photos"]) + parent_refs = list(base["parentRefs"]) + genos = set() + farb = set() + deaths = set() + for a in grp: + variants.add(a["name"]) + files.update(a["sourceFiles"]) + photos.extend(a["photos"]) + parent_refs.extend(a["parentRefs"]) + if a["genotype"]["rawGenotype"]: + genos.add(a["genotype"]["rawGenotype"]) + if a["farbschlag"]: + farb.add(a["farbschlag"]) + if a["death"]: + deaths.add(norm_dob(a["death"])) + # pick the richest genotype (most mapped loci, then longest raw) + best = max((a["genotype"] for a in grp), + key=lambda gd: (len(gd["mapped8locus"]), len(gd["rawGenotype"]))) + out = { + "id": slug(base["name"], base["dob"]), + "name": base["name"], + "nameVariants": sorted(v for v in variants if v), + "dob": norm_dob(base["dob"]), + "death": sorted(deaths)[0] if deaths else "", + "gender": None, + "farbschlag": sorted(farb)[0] if farb else "", + "farbschlagVariants": sorted(farb), + "genotype": best, + "breeder": next((a["breeder"] for a in grp if a["breeder"]), ""), + "parentRefs": _dedup_parentrefs(parent_refs), + "photos": sorted(set(photos)), + "sourceFiles": sorted(files), + "mentions": len(grp), + } + merged.append(out) + # conflict: same animal, disagreeing genotype or farbschlag or death + if len(genos) > 1 or len(farb) > 1 or len(deaths) > 1: + conflicts.append({ + "id": out["id"], "name": base["name"], "dob": out["dob"], + "genotypes": sorted(genos), "farbschlaege": sorted(farb), + "deaths": sorted(deaths), "files": sorted(files), + }) + merged.sort(key=lambda a: (a["dob"], a["name"])) + return merged, conflicts, orphans + + +def _dedup_parentrefs(refs): + seen = {} + for r in refs: + k = (norm_name(r["name"]), r["roleGuess"]) + if k not in seen: + seen[k] = r + return list(seen.values()) + + +# ------------------------------------------------------------------ reporting +def write_report(merged, conflicts, orphans, raw_count, litters, photo_count): + keyset = set((a["dob"], norm_name(a["name"])) for a in merged) + multi = [a for a in merged if a["mentions"] > 1] + with_dob = [a for a in merged if a["dob"]] + lit_dates = set(norm_dob(l["date"]) for l in litters if l["date"]) + joinable = [a for a in merged if a["dob"] and norm_dob(a["dob"]) in lit_dates] + + L = [] + L.append("# FEAT-8b — Import-Vorschau & Prüfbericht (Stammbäume + Wurfchronik)\n") + L.append("_Automatisch erzeugt von `tools/import/extract.py` — **noch nichts in die Datenbank geladen.** " + "Bitte prüfen, bevor importiert wird._\n") + L.append("## Überblick\n") + L.append(f"- Rohe Tier-Einträge aus den Stammbäumen: **{raw_count}**") + L.append(f"- Nach Zusammenführung (eindeutige Tiere): **{len(merged)}**") + L.append(f" - davon mit Geburtsdatum: {len(with_dob)}") + L.append(f" - in mehreren Dateien gefunden (Dubletten zusammengeführt): {len(multi)}") + L.append(f"- Konflikte zur Klärung: **{len(conflicts)}**") + L.append(f"- Mehrdeutige / unvollständige Einträge (ohne Name+Datum): **{len(orphans)}**") + L.append(f"- Fotos zugeordnet: **{photo_count}**") + L.append(f"- Würfe aus der Wurfchronik: **{len(litters)}**") + L.append(f" - Tiere, deren Geburtsdatum zu einem Wurf passt (verknüpfbar): {len(joinable)}\n") + + L.append("## Zusammenführungs-Schlüssel\n") + L.append("Tiere wurden zusammengeführt über **normalisierter Name + Geburtsdatum**. " + "Namensvarianten (z. B. `v.d.` ↔ `von den`, `gen.`-Spitznamen, Zuchtsuffixe) " + "werden als `nameVariants` erhalten.\n") + + L.append("## ⚠️ Konflikte (bitte prüfen)\n") + if conflicts: + L.append("Gleiches Tier (Name+Datum), aber widersprüchliche Angaben in verschiedenen Dateien:\n") + L.append("| Tier | Geburtsdatum | abweichende Genotypen | abweichende Farbschläge | Sterbedaten | Dateien |") + L.append("|---|---|---|---|---|---|") + for c in conflicts[:200]: + L.append("| {} | {} | {} | {} | {} | {} |".format( + c["name"], c["dob"], + " // ".join(c["genotypes"]) or "—", + " // ".join(c["farbschlaege"]) or "—", + " // ".join(c["deaths"]) or "—", + ", ".join(os.path.splitext(f)[0] for f in c["files"]))) + else: + L.append("_Keine._\n") + + L.append("\n## Mehrdeutige / unvollständige Einträge\n") + L.append(f"{len(orphans)} Einträge ohne sichere Name+Datum-Kombination " + "(z. B. `Name1 & Name2`-Paarzellen der tiefsten Generation, oder Zellen ohne Datum). " + "Diese werden NICHT automatisch zusammengeführt.\n") + sample = [o for o in orphans if o["name"]][:40] + for o in sample: + L.append(f"- {o['name']} · {o.get('_file','')}") + + # orphan -> likely same-named full record (soft hint, not auto-merged) + from collections import Counter + name_index = {} + for a in merged: + if a["dob"]: + name_index.setdefault(norm_name(a["name"]), []).append(a) + matchable = [] + for o in orphans: + if not o["name"]: + continue + cands = name_index.get(norm_name(o["name"])) + if cands: + matchable.append((o, cands)) + L.append("\n## Wahrscheinliche Zuordnungen unvollständiger Einträge\n") + L.append(f"{len(matchable)} namenlose/datenlose Einträge tragen denselben Namen wie ein " + "vollständiges Tier — vermutlich dasselbe Tier (zur Bestätigung):\n") + for o, cands in matchable[:60]: + opts = "; ".join(f"{c['name']} (*{c['dob']})" for c in cands[:3]) + L.append(f"- „{o['name']}“ → {opts}") + + # unmapped-token summary (for Kevin / GEN-2 + the wife) + tok = Counter() + for a in merged: + for t in a["genotype"]["unmappedTokens"]: + tok[t] += 1 + L.append("\n## Nicht ins 8-Loci-Modell abgebildete Tokens (verbatim erhalten)\n") + L.append("Diese Tokens stehen weiter in `rawGenotype`/`unmappedTokens` — Entscheidung " + "(Modell erweitern vs. als Notiz) liegt bei Julian/Kevin:\n") + L.append("| Token | Vorkommen | Bedeutung (Vermutung) |") + L.append("|---|---|---|") + hint = {"Uwuw[d]": "9. Locus Uw (nicht im Modell)", "UwUw": "9. Locus Uw", + "[WFNZ]": "Marker", "[DP]": "Marker (Dunkelpigment?)", "DP": "Marker", + "WP": "Marker", "[WP]": "Marker", "C(C)": "Schreibweise (C trägt c)", + "chmchm": "Schreibweise (c[chm]c[chm])"} + for t, n in tok.most_common(25): + L.append(f"| `{t}` | {n} | {hint.get(t, '?')} |") + + L.append("\n## Hinweise für den Import (Stufe 3, später)\n") + L.append("- **Wurfchronik = Quelle der Würfe** (Datum, Wurfstärke, Eltern, Zuchtnummer); " + "**Stammbäume = Abstammung + Genotyp + Fotos**. Verknüpfung über Geburtsdatum + Elternnamen.") + L.append("- Eltern-Verknüpfungen (`parentRefs`) stammen aus der **Position im Stammbaum** " + "(Vater oben / Mutter unten, mittlere Konfidenz) — die Wurfchronik korrigiert dies maßgeblich.") + L.append("- Genotyp: `mapped8locus` (A C D E G P Sp Re), `rawGenotype` (wortgetreu), " + "`unmappedTokens` (z. B. `Uw`, `Sls`, `Dea`, Marker wie `WFNZ/WP/DP`) — **nichts geht verloren**.") + L.append("- `-` (unbekanntes zweites Allel) → `?` (Platzhalter; Annahme, bitte bestätigen).") + with open(os.path.join(OUT, "review-report.md"), "w", encoding="utf-8") as f: + f.write("\n".join(L) + "\n") + + +# ------------------------------------------------------------------------ main +def main(): + try: + sys.stdout.reconfigure(encoding="utf-8", errors="replace") + except Exception: + pass + ap = argparse.ArgumentParser(description="FEAT-8b extractor (stages 1-2)") + ap.add_argument("--stammbaeume", default=DEFAULT_STAMMBAEUME) + ap.add_argument("--wurfchronik", default=DEFAULT_WURFCHRONIK) + args = ap.parse_args() + + if os.path.isdir(OUT): + # keep the report stable across reruns but refresh data/photos + for sub in ("photos",): + p = os.path.join(OUT, sub) + if os.path.isdir(p): + shutil.rmtree(p) + os.makedirs(OUT, exist_ok=True) + + raw_animals = [] + files = sorted(glob.glob(os.path.join(args.stammbaeume, "*.xlsx"))) + print(f"Stammbaum-Dateien: {len(files)}") + for path in files: + got = extract_stammbaum(path) + print(f" {len(got):4d} {os.path.basename(path)}") + raw_animals.extend(got) + + litters = [] + if os.path.isfile(args.wurfchronik): + litters = extract_wurfchronik(args.wurfchronik) + print(f"Wurfchronik: {len(litters)} Würfe") + + merged, conflicts, orphans = dedup(raw_animals) + photo_count = sum(len(a["photos"]) for a in merged) + + # strip private (_) fields from the JSON output + def clean(a): + return {k: v for k, v in a.items() if not k.startswith("_")} + + with open(os.path.join(OUT, "animals.json"), "w", encoding="utf-8") as f: + json.dump([clean(a) for a in merged], f, ensure_ascii=False, indent=2) + with open(os.path.join(OUT, "litters.json"), "w", encoding="utf-8") as f: + json.dump(litters, f, ensure_ascii=False, indent=2) + + write_report(merged, conflicts, orphans, len(raw_animals), litters, photo_count) + + print(f"\nRoh: {len(raw_animals)} → eindeutig: {len(merged)} " + f"| Konflikte: {len(conflicts)} | Orphans: {len(orphans)} | Fotos: {photo_count}") + print(f"Ausgabe in {OUT}") + + +if __name__ == "__main__": + main() diff --git a/tools/import/genotype.py b/tools/import/genotype.py new file mode 100644 index 0000000..3807e53 --- /dev/null +++ b/tools/import/genotype.py @@ -0,0 +1,109 @@ +"""Parse the breeder's free-text genotype notation into our frozen 8-locus +contract while losing nothing (FEAT-8b ruling from god): + + - mapped8locus : {locus: [allele1, allele2]} for A C D E G P Sp Re + - rawGenotype : the verbatim source string + - unmappedTokens: tokens we couldn't map (Uw/Sls/Dea, markers like WFNZ/WP/DP, …) + +Conventions in the source data: + - allele superscripts are bracketed: c[chm] -> c^chm, c[h] -> c^h, e[f] -> e^f + - a single '-' for the second allele means "unknown" -> mapped to '?' + (frozen-contract wildcard; assumption pending the wife's confirmation) +""" +import re + +LOCI = ["A", "C", "D", "E", "G", "P", "Sp", "Re"] + +# locus -> regex that matches that locus's token (longest alternatives first) +_LOCUS_TOKEN = { + "Sp": re.compile(r"^(Sp|sp)(Sp|sp|-)?$"), + "Re": re.compile(r"^(Re|re)(Re|re|-)?$"), + "A": re.compile(r"^(A|a)(A|a|-)?$"), + "C": re.compile(r"^(C|c)(\[(?:chm|chl|ch|h|hm|e|-)\])?(C|c|-)?(\[(?:chm|chl|ch|h|hm|e|-)\])?$"), + "D": re.compile(r"^(D|d)(D|d|-)?$"), + "E": re.compile(r"^(E|e)(\[(?:f|-)\])?(E|e|-)?(\[(?:f|-)\])?$"), + "G": re.compile(r"^(G|g)(G|g|-)?$"), + "P": re.compile(r"^(P|p)(P|p|-)?$"), +} +# loci our model does NOT have but the data uses +_KNOWN_UNMAPPED = re.compile(r"^(Uw|uw)(\[d\])?(Uw|uw)?(\[d\])?$|^(Sls|sls|Dea|dea)$", re.I) +_MARKER = re.compile(r"^\[?(WFNZ|WP|DP|GV|RV)\]?$|^\((taub|hörend|hoerend|RV|GV|extern[^)]*)\)$", re.I) + + +# one allele unit per locus (longest-match alternatives first); '-' = unknown +_ALLELE_UNIT = { + "Sp": re.compile(r"Sp|sp|-"), + "Re": re.compile(r"Re|re|-"), + "C": re.compile(r"[Cc]\[(?:chm|chl|ch|hm|h|e|-)\]|[Cc]|-"), + "E": re.compile(r"[Ee]\[(?:f|-)\]|[Ee]|-"), + "A": re.compile(r"[Aa]|-"), + "D": re.compile(r"[Dd]|-"), + "G": re.compile(r"[Gg]|-"), + "P": re.compile(r"[Pp]|-"), +} + + +def _alleles_for(locus, token): + """Extract the (allele1, allele2) pair from a single locus token, handling + two-letter alleles (Sp/Re) and bracketed superscripts (c[chm] -> c^chm).""" + pat = _ALLELE_UNIT.get(locus) + units = pat.findall(token) if pat else re.findall(r"[A-Za-z](?:\[[a-z]+\])?|-", token) + alleles = [] + for u in units: + if u == "-": + alleles.append("?") + else: + m = re.match(r"([A-Za-z]+)\[([a-z\-]+)\]", u) + if m: + # [-] = sub-allele unknown -> keep the base letter only + alleles.append(m.group(1) if m.group(2) == "-" else f"{m.group(1)}^{m.group(2)}") + else: + alleles.append(u) + if len(alleles) == 1: + alleles.append("?") + return alleles[:2] + + +def parse(raw): + """raw: a genotype string (may include trailing free text/markers). + + Returns dict {mapped8locus, rawGenotype, unmappedTokens}. + """ + raw = (raw or "").strip() + mapped = {} + unmapped = [] + # tokenise on whitespace; keep order + for tok in raw.split(): + t = tok.strip().rstrip(",") + if not t: + continue + matched = False + for locus in LOCI: + pat = _LOCUS_TOKEN.get(locus) + if pat and pat.match(t): + if locus not in mapped: # first occurrence wins + mapped[locus] = _alleles_for(locus, t) + matched = True + break + if matched: + continue + if _KNOWN_UNMAPPED.match(t) or _MARKER.match(t): + unmapped.append(t) + else: + # anything else (stray notes, malformed tokens) -> unmapped, nothing lost + unmapped.append(t) + return { + "mapped8locus": mapped, + "rawGenotype": raw, + "unmappedTokens": unmapped, + } + + +def looks_like_genotype(text): + """Heuristic: does this cell text contain >=3 recognisable locus tokens?""" + n = 0 + for tok in text.split(): + t = tok.rstrip(",") + if any(p.match(t) for p in _LOCUS_TOKEN.values()): + n += 1 + return n >= 3 diff --git a/tools/import/output/review-report.md b/tools/import/output/review-report.md new file mode 100644 index 0000000..9fc00e3 --- /dev/null +++ b/tools/import/output/review-report.md @@ -0,0 +1,176 @@ +# FEAT-8b — Import-Vorschau & Prüfbericht (Stammbäume + Wurfchronik) + +_Automatisch erzeugt von `tools/import/extract.py` — **noch nichts in die Datenbank geladen.** Bitte prüfen, bevor importiert wird._ + +## Überblick + +- Rohe Tier-Einträge aus den Stammbäumen: **889** +- Nach Zusammenführung (eindeutige Tiere): **587** + - davon mit Geburtsdatum: 292 + - in mehreren Dateien gefunden (Dubletten zusammengeführt): 142 +- Konflikte zur Klärung: **24** +- Mehrdeutige / unvollständige Einträge (ohne Name+Datum): **310** +- Fotos zugeordnet: **123** +- Würfe aus der Wurfchronik: **752** + - Tiere, deren Geburtsdatum zu einem Wurf passt (verknüpfbar): 152 + +## Zusammenführungs-Schlüssel + +Tiere wurden zusammengeführt über **normalisierter Name + Geburtsdatum**. Namensvarianten (z. B. `v.d.` ↔ `von den`, `gen.`-Spitznamen, Zuchtsuffixe) werden als `nameVariants` erhalten. + +## ⚠️ Konflikte (bitte prüfen) + +Gleiches Tier (Name+Datum), aber widersprüchliche Angaben in verschiedenen Dateien: + +| Tier | Geburtsdatum | abweichende Genotypen | abweichende Farbschläge | Sterbedaten | Dateien | +|---|---|---|---|---|---| +| Louis von den Kleinen Chaoten | 15.07.2017 | Aa Cc[] D- Ee Gg P- spsp // Aa Cc[chm] D- Ee Uwuw[d] P- spsp | Roswitha von den Kleinen Chaoten | 01.07.2020 | Stammbaum von Akio Kids, Stammbaum von CP-Fuchs, CP-Sa Sp von Unity | +| Roswitha von den Kleinen Chaoten | 10.09.2018 | aa CC D- ee[f] Gg P- spsp // aa CC D- ee[f] Uwuw[d] P- spsp | — | 05.08.2021 | Stammbaum von Akio Kids, Stammbaum von CP-Fuchs, CP-Sa Sp von Unity | +| Firefly von den Kleinen Chaoten | 18.12.2019 | /+, Aa c[chm]c[chm] D- Ee Gg PP Spsp // Aa c[chm]c[chm] DD Ee Gg PP Spsp | — | 2024 | Stammbaum von Akio Kids, Stammbaum von CP-Fuchs, CP-Sa Sp von Unity, Stammbaum von Fire Kids, Stammbaum von Valentino Firehearts Kids | +| Zuleika von den Kleinen Chaoten | 24.10.2015 | aa c[chm]c[h] D- E G P- spsp // aa c[chm]c[h] D- Ee Gg P- spsp // aa c[chm]c[h] DD Ee Gg P- spsp | — | 24.02.2019 | Stammbaum von Akio Kids, Stammbaum von CP-Fuchs, CP-Sa Sp von Unity, Stammbaum von Valentino Firehearts Kids | +| WildFire von den Kleinen Chaoten | 05.10.2017 | aa c[chm]c[chm] D- Ee gg P- spsp // aa c[chm]c[chm] D- Ee gg PP spsp | — | — | Stammbaum von Akio Kids, Stammbaum von CP-Fuchs, CP-Sa Sp von Unity, Stammbaum von Fire Kids, Stammbaum von Valentino Firehearts Kids | +| Flint von den Kleinen Chaoten | 23.12.2017 | aa Cc[chm] D- ee Gg P- spsp | — | 10.05.2021 // 10.05.2022 | Stammbaum von Akio Kids, Stammbaum von CP-Fuchs, CP-Sa Sp von Unity | +| Silenos gen. Adonis v.d. Kleinen Chaoten | 11.10.2015 | aa Cc[chm] D- Ee Gg PP spsp // aa Cc[chm] D- Ee Uwuw[d] PP spsp | — | 18.07.2019 | Stammbaum von Akio Kids, Stammbaum von Goldfuchs Sp (Pikachu) Kids, Stammbaum von Watarus Kids | +| Milka of LennyLengo | 09.12.2018 | aa C- dd E- Gg P- Spsp // aa Cc[h] dd EE Gg P- Spsp | — | 22.12.2021 | Stammbaum von Alberto Kids, Stammbaum von Stella Kids | +| Hedwig of BGB | 30.10.2019 | aa CC DD E- G- P- Spsp WP // aa CC DD E- G- P- Spsp WP DP (hörend) | — | 30.08.2023 | Stammbaum von Alberto Kids, Stammbaum von Fire Kids, Stammbaum von Stella Kids | +| Silvain von den Kleinen Chaoten | 27.03.2022 | aa c[chm]c[chm] Dd Ee[-] Gg P- Spsp // aa c[chm]c[chm] Dd ee[-] Gg Pp Spsp | — | 31.12.2024 | Stammbaum von Alberto Kids, Stammbaum von CP-Fuchs, CP-Sa Sp von Unity | +| Pitari gen. Piti von den Kleinen Chaoten | 16.05.2021 | Aa CC dd ee Gg P- Spsp DP // Aa CC dd ee Gg P- Spsp [DP] | — | — | Stammbaum von Alberto Kids, Stammbaum von Fire Kids, Stammbaum von Stella Kids | +| Brandon Stark von den Kleinen Chaoten | 13.12.2017 | aa Cc[chm] D- Ee Gg P- spsp // aa Cc[chm] D- Ee Uwuw[d] P- spsp | — | — | Stammbaum von Alberto Kids, Stammbaum von Fire Kids, Stammbaum von Kohlief, Goldfuchsef Sp von Chrissi, Stammbaum von Stella Kids | +| Enya von den Kleinen Chaoten | 01.11.2017 | Aa c[chm]c[chm] D- ee[-] G- P- spsp // Aa c[chm]c[chm] D- ee[-] Uwuw[d] P- spsp | — | — | Stammbaum von Alberto Kids, Stammbaum von Fire Kids, Stammbaum von Kohlief, Goldfuchsef Sp von Chrissi, Stammbaum von Stella Kids | +| Little Hero of Black Forest | 22.02.2018 | AA CC DD EE GG PP [WFNZ] // AA CC DD EE GG PP spsp [WFNZ] | — | 18.06.2021 | Stammbaum von Alberto Kids, Stammbaum von CP-Fuchs, CP-Sa Sp von Unity, Stammbaum von Fire Kids, Stammbaum von Kohlief, Goldfuchsef Sp von Chrissi, Stammbaum von Stella Kids, Stammbaum von Valentino Firehearts Kids | +| Molly of Black Forest | 13.09.2021 | /+, Aa Cc[chm] D- Ee gg P- spsp // Aa Cc[chm] Dd Ee gg Pp spsp | — | 03.05.2021 | Stammbaum von Alberto Kids, Stammbaum von CP-Fuchs, CP-Sa Sp von Unity | +| Little Runner's Big Ben | 03.02.2020 | Aa Cc[chm] DD Ee Gg PP Spsp // Aa Cc[chm] DD Ee Gg Pp Spsp | Daja of Little Rose | 14.10.2023 | Stammbaum von CP-Fuchs, CP-Sa Sp von Unity, Stammbaum von Fire Kids, Stammbaum von Goldfuchs Sp (Pikachu) Kids, Stammbaum von Kohlief, Goldfuchsef Sp von Chrissi, Stammbaum von Valentino Firehearts Kids, Stammbaum von Watarus Kids | +| Daja of Little Rose | 16.05.2021 | aa chmchm D- EE Gg P- // aa chmchm D- EE Gg P- spsp | — | — | Stammbaum von CP-Fuchs, CP-Sa Sp von Unity, Stammbaum von Fire Kids, Stammbaum von Valentino Firehearts Kids | +| Ichika von den Kleinen Chaoten | 19.04.2020 | aa CC D- ee Gg pp spsp // aa CC D- ee[f] Gg pp spsp | — | 27.11.2023 | Stammbaum von Goldfuchs Sp (Pikachu) Kids, Stammbaum von Watarus Kids | +| Victoria Welby gen. Welby v.d. Kleinen Chaoten | 16.01.2023 | Aa CC D- Ee[f] Gg pp Spsp [DP] // Aa CC D- ee[f] Gg pp Spsp [DP] | — | 17.02.2026 | Stammbaum von Goldfuchs Sp (Pikachu) Kids, Stammbaum von Watarus Kids | +| Zac gen. Action von den Kleinen Chaoten | 25.12.2020 | aa C- D- Ee G- Pp Spsp [DP] // aa CC D- Ee G- Pp Spsp [DP] | Belica gen. Emi von den Kleinen Chaoten | 31.01.2025 | Stammbaum von Goldfuchs Sp (Pikachu) Kids, Stammbaum von Kohlief, Goldfuchsef Sp von Chrissi, Stammbaum von Watarus Kids | +| Chelsea von den Kleinen Chaoten | 02.04.2021 | /+, Aa CC Dd ee gg Pp spsp // Aa CC Dd ee gg Pp spsp | — | — | Stammbaum von Goldfuchs Sp (Pikachu) Kids, Stammbaum von Valentino Firehearts Kids | +| Ethan von den Kleinen Chaoten | 09.07.2020 | Aa Cc[chm] D- ee[f] Gg Pp Spsp | Ichika von den Kleinen Chaoten // Orangeschimmel, hell Kragenschecke | 30.07.2024 | Stammbaum von Kentucky, Stammbaum von Watarus Kids | +| Quied Soldier of Black Forest | 07.06.2018 | /+, Aa C- D- ee[f] GG Pp Spsp [DP] // Aa C- D- ee[f] GG Pp Spsp DP | Hoshi von den Kleinen Chaoten | — | Stammbaum von Kentucky | +| Hanami von den Kleinen Chaoten | 10.09.2015 | aa Cc[chm] D- Ee gg P- spsp | — | 12.12.2019 // 14.01.2020 | Stammbaum von Kentucky, Stammbaum von Stella Kids | + +## Mehrdeutige / unvollständige Einträge + +310 Einträge ohne sichere Name+Datum-Kombination (z. B. `Name1 & Name2`-Paarzellen der tiefsten Generation, oder Zellen ohne Datum). Diese werden NICHT automatisch zusammengeführt. + +- Oskar v.d. bunten Fellnase · Stammbaum von Akio Kids.xlsx +- Raya v.d. bunten Fellnasen · Stammbaum von Akio Kids.xlsx +- Oskar v.d. bunten Fellnase · Stammbaum von Akio Kids.xlsx +- Raya v.d. bunten Fellnasen · Stammbaum von Akio Kids.xlsx +- Tai of Lennylengo · Stammbaum von Akio Kids.xlsx +- Trixxy · Stammbaum von Akio Kids.xlsx +- Katsu · Stammbaum von Akio Kids.xlsx +- Akina (RV) · Stammbaum von Akio Kids.xlsx +- Arrow PZ Niederlande · Stammbaum von Akio Kids.xlsx +- Isa of Golden Lights · Stammbaum von Akio Kids.xlsx +- BlackFire · Stammbaum von Akio Kids.xlsx +- Leila v. Jessica Walldorf · Stammbaum von Akio Kids.xlsx +- Osamu · Stammbaum von Akio Kids.xlsx +- Montana · Stammbaum von Akio Kids.xlsx +- Isiri of Golden Lights · Stammbaum von Akio Kids.xlsx +- Danjo of Golden Lights · Stammbaum von Akio Kids.xlsx +- Sam gen. Shelly v.d. bunten Fellnasen · Stammbaum von Akio Kids.xlsx +- Jacky · Stammbaum von Akio Kids.xlsx +- BlackFire · Stammbaum von Akio Kids.xlsx +- Katara · Stammbaum von Akio Kids.xlsx +- Jack Jr. · Stammbaum von Akio Kids.xlsx +- Nala · Stammbaum von Akio Kids.xlsx +- Brandon Stark · Stammbaum von Akio Kids.xlsx +- Enya · Stammbaum von Akio Kids.xlsx +- Oscar of Black Forest · Stammbaum von Akio Kids.xlsx +- Naho · Stammbaum von Akio Kids.xlsx +- Hagrid Rubeus of Black Forest · Stammbaum von Akio Kids.xlsx +- Lilo of LennyLengo · Stammbaum von Akio Kids.xlsx +- Kazuya · Stammbaum von Akio Kids.xlsx +- Rainny · Stammbaum von Akio Kids.xlsx +- Eddward · Stammbaum von Akio Kids.xlsx +- Harumi · Stammbaum von Akio Kids.xlsx +- Marc Sloan · Stammbaum von Akio Kids.xlsx +- Iris · Stammbaum von Akio Kids.xlsx +- Pan · Stammbaum von Akio Kids.xlsx +- Gin · Stammbaum von Akio Kids.xlsx +- Jack II · Stammbaum von Akio Kids.xlsx +- Isa of Golden Lights · Stammbaum von Akio Kids.xlsx +- Mystique of Black Forest · Stammbaum von Alberto Kids.xlsx +- Verpaarung von Fleur · Stammbaum von Alberto Kids.xlsx + +## Wahrscheinliche Zuordnungen unvollständiger Einträge + +38 namenlose/datenlose Einträge tragen denselben Namen wie ein vollständiges Tier — vermutlich dasselbe Tier (zur Bestätigung): + +- „Oscar of Black Forest“ → Oscar of Black Forest (*12.06.2019) +- „Hagrid Rubeus of Black Forest“ → Hagrid Rubeus of Black Forest (*18.07.2019) +- „Lilo of LennyLengo“ → Lilo of LennyLengo (*04.11.2018) +- „Mystique of Black Forest“ → Mystique of Black Forest (*12.03.2022) +- „Hagrid Rubeus of Black Forest“ → Hagrid Rubeus of Black Forest (*18.07.2019) +- „Charly of Golden Lights“ → Charly of Golden Lights (*05.04.2016) +- „Ziwa of Golden Lights“ → Ziwa of Golden Lights (*29.04.2016) +- „Chelsea von den Kleinen Chaoten“ → Chelsea von den Kleinen Chaoten (*02.04.2021); Chelsea von den Kleinen Chaoten (*15.10.2021) +- „Pinto of Fiomi“ → Pinto of Fiomi (*28.08.2016) +- „Living Force's Idefix“ → Living Force's Idefix (*05.04.2016) +- „Scarlett of Samsimar“ → Scarlett of Samsimar (*05.09.2018) +- „Living Force's Idefix“ → Living Force's Idefix (*05.04.2016) +- „Rosie of LennyLengo“ → Rosie of LennyLengo (*15.03.2016) +- „Stich von Privatzucht Gießen“ → Stich von Privatzucht Gießen (*01.09.2018) +- „Lilo of LennyLengo“ → Lilo of LennyLengo (*04.11.2018) +- „Living Force's Idefix“ → Living Force's Idefix (*05.04.2016) +- „Rosie of LennyLengo“ → Rosie of LennyLengo (*15.03.2016) +- „Living Force's Idefix“ → Living Force's Idefix (*05.04.2016) +- „Rosie of LennyLengo“ → Rosie of LennyLengo (*15.03.2016) +- „Inusch of Black Forest“ → Inusch of Black Forest (*25.10.2017) +- „Little Hero of Black Forest“ → Little Hero of Black Forest (*22.02.2018) +- „Little Runner's Destiny“ → Little Runner's Destiny (*02.03.2019) +- „Chevrolet Camaro of Topolino“ → Chevrolet Camaro of Topolino (*10.04.2018) +- „Marlin of Black Forest“ → Marlin of Black Forest (*28.05.2019) +- „Nisha of Black Forest“ → Nisha of Black Forest (*28.03.2018) +- „Nisha of Black Forest“ → Nisha of Black Forest (*28.03.2018) +- „Dorie of Black Forest“ → Dorie of Black Forest (*28.05.2019) +- „Zadar from Zeko i ptica, Croatia“ → Zadar from Zeko i ptica, Croatia (*12.04.2019) +- „Living Force's Vally“ → Living Force's Vally (*01.11.2014) +- „Pinto of Fiomi“ → Pinto of Fiomi (*28.08.2016) +- „Oscar of Black Forest“ → Oscar of Black Forest (*12.06.2019) +- „Hagrid Rubeus of Black Forest“ → Hagrid Rubeus of Black Forest (*18.07.2019) +- „Lilo of LennyLengo“ → Lilo of LennyLengo (*04.11.2018) +- „Chevrolet Camaro of Topolino“ → Chevrolet Camaro of Topolino (*10.04.2018) +- „Lilo of LennyLengo“ → Lilo of LennyLengo (*04.11.2018) +- „Rosie of LennyLengo“ → Rosie of LennyLengo (*15.03.2016) +- „Rosie of LennyLengo“ → Rosie of LennyLengo (*15.03.2016) +- „Little Runner's Destiny“ → Little Runner's Destiny (*02.03.2019) + +## Nicht ins 8-Loci-Modell abgebildete Tokens (verbatim erhalten) + +Diese Tokens stehen weiter in `rawGenotype`/`unmappedTokens` — Entscheidung (Modell erweitern vs. als Notiz) liegt bei Julian/Kevin: + +| Token | Vorkommen | Bedeutung (Vermutung) | +|---|---|---| +| `[DP]` | 16 | Marker (Dunkelpigment?) | +| `[WFNZ]` | 13 | Marker | +| `DP` | 10 | Marker | +| `WP` | 8 | Marker | +| `/+` | 8 | ? | +| `Uwuw[d]` | 4 | 9. Locus Uw (nicht im Modell) | +| `[WP]` | 3 | Marker | +| `-g` | 2 | ? | +| `C(C)` | 2 | Schreibweise (C trägt c) | +| `UwUw` | 2 | 9. Locus Uw | +| `chmchm` | 2 | Schreibweise (c[chm]c[chm]) | +| `Cc[]` | 1 | ? | +| `-psp` | 1 | ? | +| `G(G)` | 1 | ? | +| `uw[d]uw[d]` | 1 | ? | +| `[DP` | 1 | ? | +| `Dea/dea]` | 1 | ? | +| `DD-Tumor` | 1 | ? | +| `bei` | 1 | ? | +| `Geschwistern` | 1 | ? | +| `C-D-` | 1 | ? | +| `Sls` | 1 | ? | +| `(hörend)` | 1 | ? | +| `-DD` | 1 | ? | + +## Hinweise für den Import (Stufe 3, später) + +- **Wurfchronik = Quelle der Würfe** (Datum, Wurfstärke, Eltern, Zuchtnummer); **Stammbäume = Abstammung + Genotyp + Fotos**. Verknüpfung über Geburtsdatum + Elternnamen. +- Eltern-Verknüpfungen (`parentRefs`) stammen aus der **Position im Stammbaum** (Vater oben / Mutter unten, mittlere Konfidenz) — die Wurfchronik korrigiert dies maßgeblich. +- Genotyp: `mapped8locus` (A C D E G P Sp Re), `rawGenotype` (wortgetreu), `unmappedTokens` (z. B. `Uw`, `Sls`, `Dea`, Marker wie `WFNZ/WP/DP`) — **nichts geht verloren**. +- `-` (unbekanntes zweites Allel) → `?` (Platzhalter; Annahme, bitte bestätigen). diff --git a/tools/import/xlsx_util.py b/tools/import/xlsx_util.py new file mode 100644 index 0000000..2a8412a --- /dev/null +++ b/tools/import/xlsx_util.py @@ -0,0 +1,135 @@ +"""Minimal dependency-free .xlsx reader (xlsx = zip of XML). + +We only need: shared strings, cell text by reference, and image/drawing anchors. +Using stdlib zipfile + regex keeps this migration tooling free of openpyxl so it +runs anywhere Python 3 is present. +""" +import zipfile +import re +import html +import os + +_T = re.compile(r"]*>(.*?)", re.S) +_SI = re.compile(r"(.*?)", re.S) +_CELL = re.compile( + r']*?)>(?:]*>.*?)?(?:(.*?)|(.*?))?', re.S) +_ATTR = re.compile(r'(\w+)="([^"]*)"') + + +def col_to_num(col): + n = 0 + for ch in col: + n = n * 26 + (ord(ch) - 64) + return n + + +def num_to_col(n): + s = "" + while n > 0: + n, r = divmod(n - 1, 26) + s = chr(65 + r) + s + return s + + +def shared_strings(z): + try: + raw = z.read("xl/sharedStrings.xml").decode("utf-8") + except KeyError: + return [] + return [html.unescape("".join(_T.findall(si))).strip() for si in _SI.findall(raw)] + + +def sheet_paths(z): + """Return worksheet xml paths in workbook order (best effort).""" + paths = sorted(n for n in z.namelist() + if re.match(r"xl/worksheets/sheet\d+\.xml$", n)) + return paths + + +def read_cells(z, sheet_path, ss=None): + """Return {(colnum, row): text} for a worksheet.""" + if ss is None: + ss = shared_strings(z) + raw = z.read(sheet_path).decode("utf-8") + cells = {} + for attrs, v, istr in _CELL.findall(raw): + a = dict(_ATTR.findall(attrs)) + ref = a.get("r") + if not ref: + continue + m = re.match(r"([A-Z]+)(\d+)", ref) + if not m: + continue + col, row = m.group(1), int(m.group(2)) + typ = a.get("t") + if typ == "s" and v != "": + try: + text = ss[int(v)] + except (ValueError, IndexError): + text = "" + elif typ == "inlineStr" and istr: + text = html.unescape("".join(_T.findall(istr))) + elif v != "": + text = html.unescape(v) + else: + continue + if text.strip(): + cells[(col_to_num(col), row)] = text.strip() + return cells + + +def header_row(cells): + """Return {colnum: header_text} for the topmost row that has text.""" + if not cells: + return {} + top = min(r for (_, r) in cells) + return {c: t for (c, r), t in cells.items() if r == top} + + +def image_anchors(z): + """Yield (sheet_path, from_col, from_row, media_zip_path) for every image anchor. + + drawing rels map rId -> media target; the worksheet rels map the drawing to a + sheet. We resolve sheet via worksheet _rels where possible, else attribute all + anchors to the single worksheet (these files are single-sheet charts). + """ + out = [] + # sheet -> drawing + sheet_drawing = {} + for sp in sheet_paths(z): + rels = "xl/worksheets/_rels/" + os.path.basename(sp) + ".rels" + try: + r = z.read(rels).decode("utf-8") + except KeyError: + continue + for rid, tgt in re.findall(r'Id="([^"]+)"[^>]*Target="([^"]+)"', r): + if "drawing" in tgt: + dpath = os.path.normpath(os.path.join( + "xl/worksheets", tgt)).replace("\\", "/") + sheet_drawing[sp] = dpath + for sp, dpath in sheet_drawing.items(): + try: + d = z.read(dpath).decode("utf-8") + except KeyError: + continue + drels = os.path.join(os.path.dirname(dpath), "_rels", + os.path.basename(dpath) + ".rels").replace("\\", "/") + relmap = {} + try: + rr = z.read(drels).decode("utf-8") + for rid, tgt in re.findall(r'Id="([^"]+)"[^>]*Target="([^"]+)"', rr): + relmap[rid] = os.path.normpath(os.path.join( + os.path.dirname(dpath), tgt)).replace("\\", "/") + except KeyError: + pass + for anc in re.findall(r"", d, re.S): + fm = re.search( + r".*?(\d+).*?(\d+)", anc, re.S) + emb = re.search(r'r:embed="([^"]+)"', anc) + if fm and emb: + media = relmap.get(emb.group(1)) + if media: + # xdr col/row are 0-based; convert to 1-based to match cell refs + out.append((sp, int(fm.group(1)) + 1, + int(fm.group(2)) + 1, media)) + return out