diff --git a/tools/import/conflict-decisions.json b/tools/import/conflict-decisions.json index 187a71b..f936de0 100644 --- a/tools/import/conflict-decisions.json +++ b/tools/import/conflict-decisions.json @@ -1,5 +1,5 @@ { - "_doc": "Human conflict resolutions for the import quarantine (HUMANQUESTION section D / C6). The importer consumes this to UN-QUARANTINE an animal: for a matching (name + dob) it accepts the given authoritative field(s) — `genotype`, `farbschlag`, and/or `dateOfDeath` (DD.MM.YYYY) — and skips the conflict. Special field `correctDob` (DD.MM.YYYY): the matched (name + dob) record is a DUPLICATE with a WRONG birthdate — remap its DOB to `correctDob` BEFORE dedup so it merges into the canonical same-named animal. Other override fields per resolution: `gender` (male|female|m|w) — fix a misread box-colour gender (applies to stammbaum AND Wurfchronik/docx animals via merge_and_resolve.apply_decision_overrides); `father`/`mother` — authoritative parent NAMES; optional `fatherDob`/`motherDob` (DD.MM.YYYY) disambiguate a parent when several same-named animals exist. Special top-level array `addAnimals` [{name, gender, zucht?, dob?}] materialises a non-resident stub gerbil for a KNOWN parent that has no own source record (e.g. a mother named only on a Wurfchronik litter), so the litter's parent link resolves. Key match = normalize(call-name) + dob, same identity as dedup. Maintained by god (Michael) as Julian/his wife answer the D-conflicts; originals (xlsx) stay read-only.", + "_doc": "Human conflict resolutions for the import quarantine (HUMANQUESTION section D / C6). The importer consumes this to UN-QUARANTINE an animal: for a matching (name + dob) it accepts the given authoritative field(s) — `genotype`, `farbschlag`, and/or `dateOfDeath` (DD.MM.YYYY) — and skips the conflict. Special field `externalRef` (the dedup slug / animals.json id, e.g. \"unbekannt-13082025-3\"): matches ONE specific record even when several NAMELESS animals share the same (name=\"\" + dob) key — externalRef wins over the name/dob keys. An externalRef-only resolution (no `name`) does NOT register a name/dob key. Special field `correctDob` (DD.MM.YYYY): the matched (name + dob) record is a DUPLICATE with a WRONG birthdate — remap its DOB to `correctDob` BEFORE dedup so it merges into the canonical same-named animal. Other override fields per resolution: `gender` (male|female|m|w) — fix a misread box-colour gender (applies to stammbaum AND Wurfchronik/docx animals via merge_and_resolve.apply_decision_overrides); `father`/`mother` — authoritative parent NAMES; optional `fatherDob`/`motherDob` (DD.MM.YYYY) disambiguate a parent when several same-named animals exist. Special top-level array `addAnimals` [{name, gender, zucht?, dob?}] materialises a non-resident stub gerbil for a KNOWN parent that has no own source record (e.g. a mother named only on a Wurfchronik litter), so the litter's parent link resolves. Key match = normalize(call-name) + dob, same identity as dedup. Maintained by god (Michael) as Julian/his wife answer the D-conflicts; originals (xlsx) stay read-only.", "resolutions": [ { "name": "Firefly von den Kleinen Chaoten", @@ -368,6 +368,24 @@ ["stammbaum-unbekannt-13112023", "stammbaum-unbekannt-13112022"] ], "source": "Züchterin 2026-06-22 — Ticket f618dcc3 (doppelter namenloser Bock)" + }, + { + "name": "Mamta Mini v.d. Kleinen Chaoten", + "dob": "11.11.2023", + "decision": "E-Locus = Ee (das unbekannte zweite Allel ist erzwungen 'e', weil Vater Geely von den Kleinen Chaoten am E-Locus reinerbig ee=Fuchs ist und nur 'e' vererben kann). Eltern Geely (Vater) × Gaida (Mutter) verbindlich am Geburtswurf verankert (die chart-position-Heuristik lieferte sie bereits) — als Entscheidung/high gesetzt, damit der Wurf die Eltern sicher verknuepft.", + "genotype": "AA CC D- Ee Gg PP spsp", + "father": "Geely von den Kleinen Chaoten", + "mother": "Gaida von den Kleinen Chaoten", + "fatherDob": "04.03.2023", + "motherDob": "02.08.2022", + "source": "Züchterin 2026-06-22 — Tickets cc9ea3fe / 1a508c04 (Mamta Mini Ee[-]→Ee, Eltern Geely×Gaida)" + }, + { + "name": "", + "externalRef": "unbekannt-13082025-3", + "decision": "Sp-Locus = spsp (KEINE Schecke). Das namenlose Weibchen (*13.08.2025, Quelle 'Stammbaum von Martin.xlsx') war im Quell-Stammbaum als Spsp notiert, ist aber ungescheckt — der Sp-Locus muss spsp sein. Der Farbschlag bleibt der genotyp-berechnete Kohlfuchsschimmel (ee[f] = Fuchsschimmel). externalRef pinnt genau dieses Tier (mehrere namenlose Tiere teilen das Datum 13.08.2025).", + "genotype": "aa Cc[chm] D- ee[f] Gg Pp spsp", + "source": "Züchterin 2026-06-22 — Ticket e09d6f22 (faelschlich Schecke, soll spsp)" } ], "addAnimals": [ diff --git a/tools/import/extract.py b/tools/import/extract.py index 2ca792b..3fe70cf 100644 --- a/tools/import/extract.py +++ b/tools/import/extract.py @@ -1099,9 +1099,19 @@ def apply_conflict_decisions(merged, conflicts, path): Returns the number of conflicts resolved. (god/HUMANQUESTION D.)""" decisions_full = {} # (nameCanon, zuchtCanon, dob) -> r — when decision carries a Zucht decisions_name = {} # (nameCanon, dob) -> r — fallback, decision has no Zucht + decisions_ref = {} # externalRef (merged-animal id) -> r — for NAMELESS animals whose + # (name="" + dob) key is shared by several records: the externalRef + # (the dedup slug, e.g. „unbekannt-13082025-3") pins exactly one. try: with open(path, encoding="utf-8") as fh: for r in (json.load(fh).get("resolutions") or []): + ref = r.get("externalRef") + if ref: + decisions_ref[ref] = r + # An externalRef-only decision (no name) must NOT register a name/dob + # key — a ("", "") key would match every nameless, dateless animal. + if not r.get("name"): + continue nc, zc = canon_pair(r.get("name", "")) dob = norm_dob(r.get("dob", "")) if zc: @@ -1110,14 +1120,17 @@ def apply_conflict_decisions(merged, conflicts, path): decisions_name[(nc, dob)] = r except (OSError, ValueError): return 0 - if not decisions_full and not decisions_name: + if not decisions_full and not decisions_name and not decisions_ref: return 0 resolved = 0 for a in merged: nc, zc = canon_pair(a["name"]) dob = norm_dob(a["dob"]) - d = decisions_full.get((nc, zc, dob)) or decisions_name.get((nc, dob)) + # externalRef (the dedup id) wins — it is the most specific key and the only + # way to address one of several same-(name,dob) nameless animals. + d = decisions_ref.get(a.get("id")) or decisions_full.get((nc, zc, dob)) \ + or decisions_name.get((nc, dob)) if not d: continue a["resolvedByDecision"] = True diff --git a/tools/import/genotype.py b/tools/import/genotype.py index 53a05d3..7b28f6f 100644 --- a/tools/import/genotype.py +++ b/tools/import/genotype.py @@ -176,3 +176,261 @@ def looks_like_genotype(text): if any(p.match(t) for p in _LOCUS_TOKEN.values()): n += 1 return n >= 3 + + +# ──────────────────────────────────────────────────────────────────────────── +# Genotype → Farbschlag (German variety name) +# +# A faithful Python port of gerbil-manager-web/src/genetics/catalog.ts +# (`genotypeToFarbschlag` + the loci/genotype helpers it relies on). The engine +# is the single source of truth for the colour names; the IMPORT mirrors it here +# so the stored colorVarietyId can be DERIVED from a known genotype instead of a +# fragile free-text colour label (ticket cluster genetics-farbschlag). +# +# Allele symbols here use the CATALOG form (cchm / ch / ef), so we normalise the +# parser's '^'-form ('c^chm' -> 'cchm', 'e^f' -> 'ef') and treat '?' as unknown. +# Keep this in lockstep with catalog.ts — when the TS catalog changes, change here +# too (the round-trip tests in test_genotype.py guard the mapping). +# ──────────────────────────────────────────────────────────────────────────── + +# Alleles per locus, MOST-DOMINANT FIRST (mirror of loci.ts LOCI). +_FARB_LOCI = { + "A": ["A", "a"], + "C": ["C", "cchm", "ch"], + "D": ["D", "d"], + "E": ["E", "ef", "e"], + "G": ["G", "g"], + "P": ["P", "p"], + "Sp": ["Sp", "sp"], + "Re": ["Re", "re"], +} +_MARKER_LOCI = ("Sp", "Re", "Sls") + +UNKNOWN_FARBSCHLAG = "Unbekannter Farbschlag" + +# BASE_COLORS — order matters (first match wins). Mirror of catalog.ts BASE_COLORS. +# Each entry: (name, {locus: token, ...}); omitted loci are wildcards. +_BASE_COLORS = [ + # ── Frozen names (DB-key contract) ── + ("REW", {"C": "ch", "P": "p"}), + ("Hermelin", {"A": "a", "C": "ch", "D": "D", "P": "P"}), + ("Himalaya", {"A": "A", "C": "ch", "D": "D", "P": "P"}), + ("Zobel", {"A": "a", "C": "cchm", "D": "D", "E": "E", "G": "g", "P": "P"}), + ("Rotaugenschimmel", {"C": "C", "D": "D", "E": "ef", "G": "G", "P": "p"}), + ("Agouti", {"A": "A", "C": "C", "D": "D", "E": "E", "G": "G", "P": "P"}), + ("Schwarz", {"A": "a", "C": "C", "D": "D", "E": "E", "G": "G", "P": "P"}), + ("Silberagouti", {"A": "A", "C": "C", "D": "D", "E": "E", "G": "g", "P": "P"}), + ("Anthrazit", {"A": "a", "C": "C", "D": "D", "E": "E", "G": "g", "P": "P"}), + ("Algierfuchs", {"A": "A", "C": "C", "D": "D", "E": "e", "G": "G", "P": "P"}), + ("Blau", {"A": "a", "C": "C", "D": "d", "E": "E", "G": "G", "P": "P"}), + ("Gold", {"A": "A", "C": "C", "D": "D", "E": "E", "G": "G", "P": "p"}), + ("Platin", {"A": "a", "C": "C", "D": "D", "E": "E", "G": "G", "P": "p"}), + ("Goldfuchs", {"A": "A", "C": "C", "D": "D", "E": "e", "G": "G", "P": "p"}), + ("Rotfuchs", {"A": "a", "C": "C", "D": "D", "E": "e", "G": "G", "P": "p"}), + ("Dilute Gold", {"A": "A", "C": "C", "D": "d", "E": "E", "G": "G", "P": "p"}), + ("Dilute Platin", {"A": "a", "C": "C", "D": "d", "E": "E", "G": "G", "P": "p"}), + # ── baseportal.de varieties ── + ("Altweiss (REW)", {"A": "a", "C": "C", "D": "D", "E": "E", "G": "g", "P": "p"}), + ("Apricot (Blassfuchs)", {"A": "A", "C": "C", "D": "D", "E": "e", "G": "g", "P": "p"}), + ("Blaufuchs", {"A": "a", "C": "C", "D": "D", "E": "e", "G": "g", "P": "P"}), + ("C-Separator", {"A": "a", "C": "C", "D": "D", "E": "e", "G": "g", "P": "p"}), + ("Elfenbein", {"A": "A", "C": "C", "D": "D", "E": "E", "G": "g", "P": "p"}), + ("Kohlfuchs", {"A": "a", "C": "C", "D": "D", "E": "e", "G": "G", "P": "P"}), + ("Polarfuchs", {"A": "A", "C": "C", "D": "D", "E": "e", "G": "g", "P": "P"}), + ("Saphir", {"A": "a", "C": "C", "D": "D", "E": "E", "G": "G", "P": "p"}), + ("Orangeschimmel", {"A": "A", "C": "C", "D": "D", "E": "ef", "G": "G", "P": "P"}), + ("Topas", {"A": "A", "C": "C", "D": "D", "E": "E", "G": "G", "P": "p"}), + ("Platin-Hell", {"A": "a", "C": "C", "D": "D", "E": "E", "G": "G", "P": "p"}), + ("Dilute Agouti", {"A": "A", "C": "C", "D": "d", "E": "E", "G": "G", "P": "P"}), + ("Dilute Silberagouti", {"A": "A", "C": "C", "D": "d", "E": "E", "G": "g", "P": "P"}), + ("Dilute Kohlfuchs", {"A": "a", "C": "C", "D": "d", "E": "e", "G": "G", "P": "P"}), + ("Dilute Anthrazit", {"A": "a", "C": "C", "D": "d", "E": "E", "G": "g", "P": "P"}), + ("Dilute Algierfuchs", {"A": "A", "C": "C", "D": "d", "E": "e", "G": "G", "P": "P"}), + ("Dilute Goldfuchs", {"A": "A", "C": "C", "D": "d", "E": "e", "G": "G", "P": "p"}), + ("Dilute Rotfuchs", {"A": "a", "C": "C", "D": "d", "E": "e", "G": "G", "P": "p"}), + ("Dilute Polarfuchs", {"A": "A", "C": "C", "D": "d", "E": "e", "G": "g", "P": "P"}), + ("Silberschimmel", {"C": "C", "D": "D", "E": "ef", "G": "g", "P": "P"}), + ("Polarfuchsschimmel", {"A": "A", "C": "C", "D": "D", "E": "ef", "G": "g", "P": "P"}), + ("Algierfuchsschimmel", {"A": "A", "C": "C", "D": "D", "E": "ef", "G": "G", "P": "P"}), + ("Kohlfuchsschimmel", {"A": "a", "C": "C", "D": "D", "E": "ef", "G": "G", "P": "P"}), + ("Blaufuchsschimmel", {"A": "a", "C": "C", "D": "D", "E": "ef", "G": "g", "P": "P"}), + ("Kohlfuchs, hell", {"A": "a", "C": "C", "D": "D", "E": "e", "G": "G", "P": "P"}), + ("Goldfuchs, hell", {"A": "A", "C": "C", "D": "D", "E": "e", "G": "G", "P": "p"}), + ("Goldfuchsschimmel", {"A": "A", "C": "C", "D": "D", "E": "ef", "G": "G", "P": "p"}), + ("Gold-Hell", {"A": "A", "C": "C", "D": "D", "E": "E", "G": "G", "P": "p"}), + ("Blaufuchs, hell", {"A": "a", "C": "C", "D": "D", "E": "e", "G": "g", "P": "P"}), + ("Rotfuchsschimmel", {"A": "a", "C": "C", "D": "D", "E": "ef", "G": "G", "P": "p"}), + ("Polarfuchs, hell", {"A": "A", "C": "C", "D": "D", "E": "e", "G": "g", "P": "P"}), + ("Kohlfuchsschimmel, hell", {"A": "a", "C": "C", "D": "D", "E": "ef", "G": "G", "P": "P"}), + ("Rotfuchs, hell", {"A": "a", "C": "C", "D": "D", "E": "e", "G": "G", "P": "p"}), + ("Kohlfuchs-Hell", {"A": "a", "C": "C", "D": "D", "E": "e", "G": "G", "P": "P"}), + ("Algierfuchs, hell", {"A": "A", "C": "C", "D": "D", "E": "e", "G": "G", "P": "P"}), + ("Dilute Topas", {"A": "A", "C": "C", "D": "d", "E": "E", "G": "G", "P": "p"}), + ("Dilute Blaufuchs", {"A": "a", "C": "C", "D": "d", "E": "e", "G": "g", "P": "P"}), + # ── c^chm colourpoint varieties ── + ("Marder", {"A": "a", "C": "cchm", "D": "D", "E": "E", "G": "G", "P": "P"}), + ("Siam", {"A": "a", "C": "cchm/ch", "D": "D", "E": "E", "G": "G", "P": "P"}), + ("Zobel-Hell", {"A": "a", "C": "cchm/ch", "D": "D", "E": "E", "G": "g", "P": "P"}), + ("CP-Agouti", {"A": "A", "C": "cchm", "D": "D", "E": "E", "G": "G", "P": "P"}), + ("CP-Agouti-Hell", {"A": "A", "C": "cchm/ch", "D": "D", "E": "E", "G": "G", "P": "P"}), + ("CP-Silberagouti", {"A": "A", "C": "cchm", "D": "D", "E": "E", "G": "g", "P": "P"}), + ("CP-Silberagouti-Hell", {"A": "A", "C": "cchm/ch", "D": "D", "E": "E", "G": "g", "P": "P"}), + ("CP-Algierfuchs", {"A": "A", "C": "cchm", "D": "D", "E": "e", "G": "G", "P": "P"}), + ("CP-Algierfuchs-Hell", {"A": "A", "C": "cchm/ch", "D": "D", "E": "e", "G": "G", "P": "P"}), + ("CP-Polarfuchs", {"A": "A", "C": "cchm", "D": "D", "E": "e", "G": "g", "P": "P"}), + ("CP-Polarfuchs-Hell", {"A": "A", "C": "cchm/ch", "D": "D", "E": "e", "G": "g", "P": "P"}), + ("CP-Fuchs", {"A": "A", "C": "cchm", "D": "d", "E": "e", "G": "G", "P": "P"}), + ("CP-Fuchs-Hell", {"A": "A", "C": "cchm/ch", "D": "d", "E": "e", "G": "G", "P": "P"}), + ("CP-Blaufuchs", {"A": "A", "C": "cchm", "D": "d", "E": "e", "G": "g", "P": "P"}), + ("CP-Orangeschimmel", {"C": "cchm", "D": "D", "E": "ef", "G": "G", "P": "P"}), + ("CP-Orangeschimmel-Hell", {"C": "cchm/ch", "D": "D", "E": "ef", "G": "G", "P": "P"}), +] + + +def _normalize_allele(a): + """Parser allele form -> catalog form. 'c^chm'->'cchm', 'e^f'->'ef', '-'/None->'?'.""" + if a is None or a == "-": + return "?" + return a.replace("^", "") + + +def _resolve_allele_pair(locus, pair): + """GEN-5 unknown-allele rule (mirror of genotype.ts resolveAllelePair). + An unknown '?' is a COPY of the known partner; both unknown -> wild-type + (markers default to the unmarked recessive).""" + a = _normalize_allele(pair[0] if len(pair) > 0 else "?") + b = _normalize_allele(pair[1] if len(pair) > 1 else "?") + a_unknown = a == "?" + b_unknown = b == "?" + if not a_unknown and not b_unknown: + return [a, b] + if a_unknown and b_unknown: + alleles = _FARB_LOCI.get(locus, ["?"]) + fb = alleles[-1] if locus in _MARKER_LOCI else alleles[0] + return [fb, fb] + known = b if a_unknown else a + return [known, known] + + +def _dominance_rank(locus, allele): + alleles = _FARB_LOCI.get(locus, []) + return alleles.index(allele) if allele in alleles else len(alleles) + + +def _dominant_allele(locus, a, b): + return a if _dominance_rank(locus, a) <= _dominance_rank(locus, b) else b + + +def _locus_token(mapped, locus): + """Expressed token at a locus (mirror of catalog.ts locusToken).""" + x, y = _resolve_allele_pair(locus, mapped.get(locus, ["?", "?"])) + if locus == "E": + if x == y: + return x + if (x == "e" and y == "ef") or (x == "ef" and y == "e"): + return "ef" + return _dominant_allele("E", x, y) + return _dominant_allele(locus, x, y) + + +def _e_family(mapped): + """E-locus family tag ('Fuchs'/'Fuchsschimmel'/'Schimmel') or None.""" + x, y = _resolve_allele_pair("E", mapped.get("E", ["?", "?"])) + if x == "e" and y == "e": + return "Fuchs" + if (x == "e" and y == "ef") or (x == "ef" and y == "e"): + return "Fuchsschimmel" + if x == "ef" and y == "ef": + return "Schimmel" + return None + + +def _entry_in_e_family(name, tokens, family): + if tokens.get("E") is None: + return False + n = name.lower() + if family == "Fuchsschimmel": + return "fuchsschimmel" in n + if family == "Schimmel": + return "schimmel" in n and "fuchsschimmel" not in n + return "schimmel" not in n + + +def _matches(mapped, tokens): + return all(_locus_token(mapped, locus) == tok for locus, tok in tokens.items()) + + +def _base_colour_for(mapped): + family = _e_family(mapped) + if family: + for name, tokens in _BASE_COLORS: + if _entry_in_e_family(name, tokens, family) and _matches(mapped, tokens): + return name + return None + for name, tokens in _BASE_COLORS: + if _matches(mapped, tokens): + return name + return None + + +def _colourpoint_name(mapped): + """C-locus colourpoint NAMING transform (mirror of catalog.ts colourpointName).""" + c = _resolve_allele_pair("C", mapped.get("C", ["?", "?"])) + if "C" in c: + return None + if c[0] == "ch" and c[1] == "ch": + return None + both_cchm = c[0] == "cchm" and c[1] == "cchm" + agouti = "A" in _resolve_allele_pair("A", mapped.get("A", ["?", "?"])) + if not agouti and _e_family(mapped) is None: + g1, g2 = _resolve_allele_pair("G", mapped.get("G", ["?", "?"])) + grey = g1 == "g" and g2 == "g" + if grey: + return "Zobel" if both_cchm else "Zobel-Hell" + return "Marder" if both_cchm else "Siam" + # Name the colour as if C were full, then prefix 'CP-'. + forced = dict(mapped) + forced["C"] = ["C", "C"] + base = _base_colour_for(forced) + if not base: + return None + DILUTE = "Dilute " + if base.startswith(DILUTE): + return f"{DILUTE}CP-{base[len(DILUTE):]}{'' if both_cchm else '-Hell'}" + return f"CP-{base}{'' if both_cchm else '-Hell'}" + + +_CATEGORY_NAMES = { + "Standard", "Colourpoint", "Dilute", + "Fuchs", "Fuchsschimmel", "Schimmel", + "Colourpoint Dilute", +} + + +def genotype_to_farbschlag(mapped): + """Resolve a parsed `mapped8locus` dict to its German Farbschlag (BASE name, + WITHOUT Schecke/Rex modifiers), or `UNKNOWN_FARBSCHLAG`. + + `mapped` is genotype.parse(...)['mapped8locus'] (allele '^'-form / '-' / missing + loci tolerated). Modifiers (Schecke/Rex) are intentionally OMITTED — the import + stores them as the genotype's Sp/Re loci and via the colour label, not in the + catalog colorVarietyId. Mirror of catalog.ts genotypeToFarbschlag (minus the + appended modifiers). + """ + if not mapped: + return UNKNOWN_FARBSCHLAG + # REW check: both C reduced (no full 'C') AND pink-eyed (pp). + c0, c1 = _resolve_allele_pair("C", mapped.get("C", ["?", "?"])) + p0, p1 = _resolve_allele_pair("P", mapped.get("P", ["?", "?"])) + c_reduced = lambda c: c in ("cchm", "ch") + if c_reduced(c0) and c_reduced(c1) and p0 == "p" and p1 == "p": + return "REW" + base_name = _colourpoint_name(mapped) or _base_colour_for(mapped) + if not base_name or base_name in _CATEGORY_NAMES: + return UNKNOWN_FARBSCHLAG + return base_name + + +def farbschlag_from_genotype_string(raw): + """Convenience: raw genotype string -> Farbschlag base name (or UNKNOWN).""" + return genotype_to_farbschlag(parse(raw).get("mapped8locus") or {}) diff --git a/tools/import/merge_and_resolve.py b/tools/import/merge_and_resolve.py index 7423cd0..03511ee 100644 --- a/tools/import/merge_and_resolve.py +++ b/tools/import/merge_and_resolve.py @@ -5,6 +5,8 @@ import uuid import sys from datetime import datetime +import genotype as gt + # Prevent encoding crashes on Windows consoles when printing unicode if sys.platform.startswith('win'): try: @@ -422,33 +424,39 @@ def get_dedup_name_key(name): return "".join(c for c in n if c.isalnum()) def clean_color_name(c_desc): + """Normalise a free-text colour label to a catalog key + Schecke flag. + + Returns (clean_name, is_schecke). A PARENTHETICAL „(schimmel)" is NOT a + definitive Schimmel — the breeder writes it to mean „könnte sich später als + Schimmel entpuppen" (ticket e22764aa). So we STRIP the „(…)" instead of + folding it into the name (which used to turn „Blaufuchs(schimmel)" into the + wrong „blaufuchsschimmel"); the still-uncertain Schimmel-modifier is carried + by the genotype (ee[-] = Fuchs, Schimmel unknown), not the colour label. + """ if not c_desc: return "", False - + # Lowercase and strip c = c_desc.lower().strip() - + # Check for Schecke is_schecke = False if re.search(r'\bsp\b|\bsp\d|\bsp[*(²³]|\bspotted|\bschecke|[- ]sp\b|\w+sp\b', c): is_schecke = True - - # Standardize parentheticals for schimmel - c = c.replace("(schimmel)", "schimmel") - c = c.replace("(schimmel-hell)", "schimmel hell") - c = c.replace("(schimmel hell)", "schimmel hell") - + # Strip schecke/sp markers and any trailing text starting from sp c = re.sub(r'\([- ]?sp(otted)?\)', '', c) # handles (-sp) c = re.sub(r'[- ]?sp(otted)?\b.*', '', c) # handles -sp(k), -sp*(k), -sp, etc. c = re.sub(r'[- ]?schecke\b.*', '', c) c = re.sub(r'[- ]?spotted\b.*', '', c) - - # Strip any other parentheticals, symbols, or trailing stars/numbers + + # Strip any other parentheticals (incl. „(schimmel)" = „möglich/unbestimmt"), + # symbols, or trailing stars/numbers. The parenthetical is deliberately NOT + # promoted to a definitive part of the colour name (ticket e22764aa). c = re.sub(r'\s*\(.*?\)\s*', ' ', c) c = re.sub(r'[²³*]', '', c) c = c.strip() - + # Mapping table for abbreviations, typos, and specific combinations mapping = { "antra": "anthrazit", @@ -476,27 +484,87 @@ def clean_color_name(c_desc): return c, is_schecke -def resolve_color_and_genotype(color_val, existing_genotype, variety_map, variety_genotypes): - if not color_val: - return None, existing_genotype - color_str = str(color_val).strip() - clean_name, is_schecke = clean_color_name(color_str) - # Match color in variety_map - color_variety_id = None +def _match_color_label(clean_name, variety_map): + """Map a cleaned colour label to a ColorVariety id (text-only path). + + Exact name wins; otherwise pick the LONGEST/most-specific substring match + (ticket 3f5942a2 — the old code broke on the FIRST substring hit, so „Goldfuchs" + matched the shorter „Gold" first). Among substring candidates the longest seed + name wins, then the longest clean_name overlap; ties broken deterministically. + """ + if not clean_name: + return None if clean_name in variety_map: - color_variety_id = variety_map[clean_name] - else: - for seed_name, seed_id in variety_map.items(): - if seed_name in clean_name or clean_name in seed_name: - color_variety_id = seed_id - break - # Update genotype if it's a Schecke + return variety_map[clean_name] + candidates = [] + for seed_name, seed_id in variety_map.items(): + if not seed_name: + continue + if seed_name in clean_name or clean_name in seed_name: + # Specificity score: prefer the longer seed name (more specific), + # then the closeness of lengths so „goldfuchs" beats „gold" for the + # label „goldfuchs". + candidates.append((len(seed_name), -abs(len(seed_name) - len(clean_name)), + seed_name, seed_id)) + if not candidates: + return None + candidates.sort(reverse=True) + return candidates[0][3] + + +def resolve_color_and_genotype(color_val, existing_genotype, variety_map, variety_genotypes): + """Resolve a gerbil's stored ColorVariety id + genotype. + + GENOTYPE WINS (ticket cluster genetics-farbschlag): when a parseable genotype + is present and the genetics engine (genotype.genotype_to_farbschlag — a faithful + Python mirror of catalog.ts) computes a KNOWN catalog variety, that variety is + authoritative for colorVarietyId. The free-text colour label is only a fallback + (no genotype, or genotype resolves to „Unbekannt"). This fixes the imports where + the source label ignored a locus (dd → „Agouti" instead of „Dilute Agouti", + ee → „Gold" instead of „Goldfuchs", parenthetical „(schimmel)", …). + + Returns (color_variety_id, genotype). `genotype` is the (possibly Schecke- + annotated) genotype STRING — never silently flips an explicit spsp to Spsp. + """ + if not color_val and not existing_genotype: + return None, existing_genotype + + clean_name, is_schecke = clean_color_name(str(color_val).strip()) if color_val else ("", False) + + # 1) Genotype-derived variety (authoritative when it resolves to a known name). + # GUARD (VORSICHTIG): only trust the genotype when it parsed CLEANLY enough to + # decide a colour — both the C and E loci must be mapped. The breeder sometimes + # writes the genotype in the COMPACT catalog notation („cchmcchm", „efef", + # „chch") which this parser leaves UNMAPPED (it expects the bracketed „c[chm]" + # form); a dropped C/E locus would silently read as wild-type and mis-recolour + # an otherwise-correct animal (e.g. Marder→Schwarz, Orangeschimmel→Agouti). When + # the parse is incomplete we keep the source text label instead. + color_variety_id = None + geno_name = None + if existing_genotype: + try: + mapped = gt.parse(existing_genotype).get("mapped8locus") or {} + except Exception: + mapped = {} + if mapped.get("C") and mapped.get("E"): + fs = gt.genotype_to_farbschlag(mapped) + if fs and fs != gt.UNKNOWN_FARBSCHLAG: + geno_name = fs + color_variety_id = variety_map.get(fs.strip().lower()) + + # 2) Fall back to the text label when the genotype gave nothing usable. + if not color_variety_id: + color_variety_id = _match_color_label(clean_name, variety_map) + + # Update genotype if the LABEL says Schecke — but never override an explicit + # Sp-locus already present in the source genotype (ticket e09d6f22: a source + # „spsp" must NOT be flipped to „Spsp" just because the label looked scheckig; + # the source genotype is authoritative for the Sp-locus). Only ADD Spsp when + # the genotype carries no Sp token at all. genotype = existing_genotype if is_schecke: if genotype: - if "spsp" in genotype: - genotype = genotype.replace("spsp", "Spsp") - elif "Spsp" not in genotype and "Sp" not in genotype: + if "Sp" not in genotype and "sp" not in genotype: genotype = f"{genotype} Spsp".strip() else: canonical = variety_genotypes.get(color_variety_id) diff --git a/tools/import/test_genotype.py b/tools/import/test_genotype.py index d614ad7..d4a0766 100644 --- a/tools/import/test_genotype.py +++ b/tools/import/test_genotype.py @@ -91,6 +91,42 @@ check("Algierfuchs genotype: no unmapped tokens", r["unmappedTokens"] == []) check("looks_like_genotype sees Uw as G", g.looks_like_genotype("aa Cc Uwuw") is True) + +# ── genotype_to_farbschlag — Python mirror of catalog.ts genotypeToFarbschlag ── +# Genetik/Farbschlag ist korrektheitskritisch: jeder Ticket-Fall bekommt einen +# Regressionstest (Konvention). The base name is returned WITHOUT the Schecke/Rex +# modifier (the import carries those via the Sp/Re loci, not the variety id). +def fs(s): + return g.genotype_to_farbschlag(g.parse(s)["mapped8locus"]) + +# Ticket 3f5942a2 — Goldfuchs (ee) NOT Gold (EE): a fox genotype must resolve to +# a Fuchs variety, never the substring-shorter „Gold". +check("3f5942a2: ee Fuchs -> Goldfuchs (not Gold)", fs("AA CC DD ee GG pp spsp") == "Goldfuchs") +check("3f5942a2: ee[f] Fuchsschimmel -> Goldfuchsschimmel", + fs("Aa C- D- ee[f] G- pp Spsp") == "Goldfuchsschimmel") +# Ticket 1aac054f — namenloses Weibchen *13.08.2025: Kohlfuchsschimmel (not Gold). +check("1aac054f: aa ee[f] -> Kohlfuchsschimmel", + fs("aa Cc[chm] D- ee[f] Gg Pp Spsp") == "Kohlfuchsschimmel") +# Ticket 998087e2 — dd must NOT be ignored: Dilute Agouti (not Agouti). +check("998087e2: AA dd EE -> Dilute Agouti", fs("AA CC dd EE GG PP spsp") == "Dilute Agouti") +check("998087e2 counter: AA DD EE -> Agouti (no dilute)", fs("AA CC DD EE GG PP spsp") == "Agouti") +# Ticket 06217eb3 — dd Anthrazit: Dilute Anthrazit (not Anthrazit). +check("06217eb3: aa dd gg -> Dilute Anthrazit", fs("aa CC dd Ee gg P- spsp") == "Dilute Anthrazit") +# Ticket e22764aa — ee[-] = Fuchs (Schimmel-Modifier unbekannt) -> Blaufuchs, +# NEVER Blaufuchsschimmel (the „(schimmel)" parenthetical is „möglich", not definitiv). +check("e22764aa: aa ee[-] gg -> Blaufuchs (not …schimmel)", + fs("aa C- D- ee[-] gg P- spsp") == "Blaufuchs") +# Ticket cc9ea3fe / 1a508c04 — Mamta Mini Ee resolves to Agouti (AA, E_). +check("Mamta Mini: AA Ee -> Agouti", fs("AA CC D- Ee Gg PP spsp") == "Agouti") +# E-locus phenotype rules (breeder): ef/ef = Schimmel family, ef/e = Fuchsschimmel. +check("efef agouti base -> Orangeschimmel", fs("AA CC DD e[f]e[f] GG PP spsp") == "Orangeschimmel") +check("ef/e het -> Fuchsschimmel family (Kohlfuchsschimmel)", + fs("aa CC DD ee[f] GG PP spsp") == "Kohlfuchsschimmel") +# Unknown allele = copy of the visible partner (GEN-5): A? -> AA, D? -> DD. +check("unknown copies known: AA C? DD ee GG pp -> Goldfuchs", fs("AA C- DD ee GG pp spsp") == "Goldfuchs") +# An incomplete parse must NOT throw and must not be invented as a real colour. +check("empty mapping -> Unbekannt", g.genotype_to_farbschlag({}) == g.UNKNOWN_FARBSCHLAG) + if check.failed: print(f"\n{check.failed} test(s) FAILED") sys.exit(1) diff --git a/tools/import/test_merge_resolve.py b/tools/import/test_merge_resolve.py index 65578fc..fc2bd1f 100644 --- a/tools/import/test_merge_resolve.py +++ b/tools/import/test_merge_resolve.py @@ -432,6 +432,61 @@ check("contracts: animal-less record has empty Animals list", _sale3 and _sale3[0]["Animals"] == []) +# ── resolve_color_and_genotype + clean_color_name (genetics-farbschlag cluster) ── +# A tiny synthetic variety_map (name->id) with the keys these cases need. +_VM = { + "gold": "ID-gold", "goldfuchs": "ID-goldfuchs", "goldfuchsschimmel": "ID-gfs", + "agouti": "ID-agouti", "dilute agouti": "ID-dagouti", + "anthrazit": "ID-anthrazit", "dilute anthrazit": "ID-danthrazit", + "blaufuchs": "ID-blaufuchs", "blaufuchsschimmel": "ID-bfs", + "kohlfuchsschimmel": "ID-kfs", "marder": "ID-marder", "schwarz": "ID-schwarz", + "orangeschimmel": "ID-orange", +} +_VG = {} + + +def _rc(color, geno): + return m.resolve_color_and_genotype(color, geno, _VM, _VG)[0] + + +# Ticket 3f5942a2 — specificity: „Goldfuchs"-label must NOT collapse to „Gold". +check("3f5942a2 label: 'Goldfuchs' -> goldfuchs (not gold)", + m._match_color_label("goldfuchs", _VM) == "ID-goldfuchs") +# Genotype wins: ee fox genotype overrides a stale „Gold" label. +check("3f5942a2 genotype wins: ee -> Goldfuchs over 'Gold' label", + _rc("Gold", "AA CC DD ee GG pp spsp") == "ID-goldfuchs") +# Ticket 998087e2 — dd ignored by label: genotype gives Dilute Agouti. +check("998087e2: dd genotype -> Dilute Agouti over 'Agouti' label", + _rc("Agouti", "AA CC dd EE GG PP spsp") == "ID-dagouti") +# Ticket 06217eb3 — Dilute Anthrazit. +check("06217eb3: dd genotype -> Dilute Anthrazit over 'Anthrazit'", + _rc("Anthrazit", "aa CC dd Ee gg P- spsp") == "ID-danthrazit") +# Ticket 1aac054f — Kohlfuchsschimmel over a stale 'Gold' label. +check("1aac054f: ee[f] genotype -> Kohlfuchsschimmel over 'Gold'", + _rc("Gold", "aa Cc[chm] D- ee[f] Gg Pp Spsp") == "ID-kfs") +# Ticket e22764aa — „Blaufuchs(schimmel)" parenthetical is NOT definitive; the +# cleaned label is „blaufuchs" and the ee[-] genotype confirms Blaufuchs. +_cn, _sc = m.clean_color_name("Blaufuchs(schimmel)") +check("e22764aa: '(schimmel)' stripped, not promoted -> 'blaufuchs'", _cn == "blaufuchs") +check("e22764aa: ee[-] genotype -> Blaufuchs (not Blaufuchsschimmel)", + _rc("Blaufuchs(schimmel)", "aa C- D- ee[-] gg P- spsp") == "ID-blaufuchs") +# Ticket e09d6f22 — a Schecke-looking LABEL must not flip an explicit source spsp +# to Spsp (the source genotype is authoritative for the Sp-locus). +_, _g_spsp = m.resolve_color_and_genotype("Kohlfuchsschimmel, hell", + "aa Cc[chm] D- ee[f] Gg Pp spsp", _VM, _VG) +check("e09d6f22: explicit spsp kept (label-Schecke does not force Spsp)", + "Spsp" not in _g_spsp and "spsp" in _g_spsp) +# VORSICHTIG guard: a COMPACT-notation genotype (cchmcchm/efef) the parser can't +# read must fall back to the text label, NOT mis-recolour (e.g. Marder->Schwarz). +check("guard: compact 'cchmcchm' unparsable -> keep label 'Marder'", + _rc("Marder", "aa cchmcchm DD EE GG PP spsp rere") == "ID-marder") +check("guard: compact 'efef' unparsable -> keep label 'Orangeschimmel'", + _rc("Orangeschimmel", "AA CC DD efef GG PP spsp rere") == "ID-orange") +# A genuinely Schecke label with no Sp in the genotype still appends Spsp. +_, _g_add = m.resolve_color_and_genotype("Agouti Schecke", "AA CC DD EE GG PP", _VM, _VG) +check("schecke label + no Sp token -> appends Spsp", "Spsp" in _g_add) + + # ── Integration: assert the resolved_import.json output reflects the ticket fixes ── # (Only when the pipeline has already been run; tolerant if the file is absent.) import os as _os, json as _json @@ -534,6 +589,59 @@ if _os.path.exists(_resolved): and "unbekannt" in (g.get("ExternalRef") or "").lower()] check("Duplicate-merge: nameless buck *15.02.2024 deduped to one record", len(_bucks) == 1) + + # ── genetics-farbschlag cluster: the STORED colorVarietyId is now genotype- + # correct for the ticket animals. Build the id→name map from the authoritative + # ApplicationContext.cs catalog (same source the pipeline uses for the ids). + import re as _re + _app = _os.path.abspath(_os.path.join(_os.path.dirname(__file__), + "../../GerbilManagerWebAPI/ApplicationContext.cs")) + _idname = {} + if _os.path.exists(_app): + _cm = _re.search(r"catalog\s*=\s*\{(.*?)\};", open(_app, encoding="utf-8").read(), _re.DOTALL) + if _cm: + for _i, (_n, _g, _so) in enumerate(_re.findall( + r'\(\s*"([^"]+)"\s*,\s*"([^"]+)"\s*,\s*(\d+)\s*\)', _cm.group(1))): + _idname[f"00000000-0000-0000-0000-{_i + 1:012d}"] = _n + + def _by_ref(ref): + return next((g for g in _d["gerbils"] if g.get("ExternalRef") == ref), None) + + def _cv_name(g): + return _idname.get(g.get("ColorVarietyId")) if g else None + + if _idname: + # Ticket 1aac054f — namenloses Weibchen *13.08.2025 -> Kohlfuchsschimmel. + _t1 = _by_ref("stammbaum-unbekannt-13082025-2") + check("1aac054f: nameless *13.08.2025 stored as Kohlfuchsschimmel", + _cv_name(_t1) == "Kohlfuchsschimmel") + # Ticket e09d6f22 — same litter, *-3: spsp (NOT Schecke) + Kohlfuchsschimmel. + _t2 = _by_ref("stammbaum-unbekannt-13082025-3") + check("e09d6f22: Sp-locus is spsp (no Schecke)", + _t2 is not None and "Spsp" not in (_t2.get("Genotype") or "") + and "spsp" in (_t2.get("Genotype") or "")) + check("e09d6f22: stored as Kohlfuchsschimmel", _cv_name(_t2) == "Kohlfuchsschimmel") + # Ticket 06217eb3 — Dilute Anthrazit (dd). + _t3 = _by_ref("stammbaum-unbekannt-27052025") + check("06217eb3: nameless dd-Weibchen stored as Dilute Anthrazit", + _cv_name(_t3) == "Dilute Anthrazit") + # Ticket e22764aa — Blaufuchs (NOT Blaufuchsschimmel). + _t4 = _by_ref("stammbaum-unbekannt-16012026") + check("e22764aa: '(schimmel)' animal stored as Blaufuchs", + _cv_name(_t4) == "Blaufuchs") + # Ticket 3f5942a2 — named fox animals are Goldfuchs (ee), not Gold (EE). + _banjo = _find("Banjo of Fiomi") + check("3f5942a2: Banjo of Fiomi stored as Goldfuchs", + _cv_name(_banjo) == "Goldfuchs") + + # ── Mamta Mini (cc9ea3fe / 1a508c04): Ee[-] resolved to Ee + parents linked. ── + _mamta = _find("Mamta Mini") + check("Mamta Mini: E-locus resolved to Ee (no unknown [-])", + _mamta is not None and "Ee[-]" not in (_mamta.get("Genotype") or "") + and "Ee" in (_mamta.get("Genotype") or "")) + _mf, _mm = _parents(_mamta) + check("Mamta Mini: father Geely, mother Gaida linked at the litter", + (_mf or "").startswith("Geely") and (_mm or "").startswith("Gaida")) else: print("note: output/resolved_import.json not present — skipped integration assertions")