From 0c94cfcbf1f82d3029c3106bafecfbe78c0aedf9 Mon Sep 17 00:00:00 2001 From: Gulum Date: Sat, 6 Jun 2026 14:53:27 +0200 Subject: [PATCH 1/2] IMPORT-POLISH: 4 Importer-Fixes nach Re-Import #2 FIX-1 decision-matching: apply_conflict_decisions/apply_dob_remaps nutzen jetzt canon_pair(name)[0] als Match-Key (Dedup-Identitaet: call-name ohne Zucht, v.d.<->von den gefaltet). Workaround-Spelling v.d. in Victoria Welbys Decision bleibt erhalten; beide Formen matchen jetzt. Kommentar im decision-Eintrag aktualisiert. FIX-2 specific-wins: _alleles_compatible aendert '? vs x = False' -> '? vs x = True' (spezifischer Wert gewinnt). C- vs CC, G- vs Gg, P? vs PP sind kein Konflikt mehr. Echte Wert-Widersprueche (DD vs Dd, Ee vs ee, PP vs Pp) bleiben Konflikte. Loest Enya, Ella, Zac automatisch (Konflikte 8->5 erwartet). 2 bestehende Tests angepasst, 7 neue Tests. FIX-3 parent-FK backfill: nach dem Wurfchronik-Rueckverknuepfungs- Block iteriert ImportService.RunAsync ueber bereits importierte Wuerfe mit null Father/MotherId und setzt fehlende FKs wenn das Elterntier jetzt ladbar ist. Trockenlauf zaehlt, Execute schreibt. LitterSummary.ParentFksBackfilled + 2 neue C#-Tests (SQLite). FIX-4 Skarlett-Artefakt: parse_detail() strippt trailing / +YEAR aus dem Genotyp-Tail (re.sub). Sterbejahr bleibt als death-Date erhalten -> Skarlett erscheint als reiner Sterbedatum-Konflikt. 2 neue Python-Tests. Gate: 124/124 C#-Tests, Python test_extract/test_genotype ALL PASS, has-pending-model-changes = No. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- GerbilManager.Tests/ImportServiceTests.cs | 110 ++++++++++++++++++++ GerbilManagerWebAPI/Import/ImportModels.cs | 3 +- GerbilManagerWebAPI/Import/ImportService.cs | 47 ++++++++- tools/import/conflict-decisions.json | 2 +- tools/import/extract.py | 31 ++++-- tools/import/test_extract.py | 76 ++++++++++++-- 6 files changed, 248 insertions(+), 21 deletions(-) diff --git a/GerbilManager.Tests/ImportServiceTests.cs b/GerbilManager.Tests/ImportServiceTests.cs index 16f1bda..450d9fc 100644 --- a/GerbilManager.Tests/ImportServiceTests.cs +++ b/GerbilManager.Tests/ImportServiceTests.cs @@ -321,6 +321,116 @@ namespace GerbilManager.Tests Assert.Equal("aa Ccchm ?? eef ?? ?? ?? ??", ImportService.ComposeGenotype(g)); } + [Fact] + public async Task ParentFkBackfill_fills_null_litter_parent_on_reimport() + { + // Run 1: litter "Wurf A" has sire "Vater" (conflict=true — not loaded) and dam "Mutter" + // (conflict=false — loaded). After run 1: litter.FatherId = null. + // Run 2: sire "Vater" is no longer in conflict. Backfill must set litter.FatherId. + var dir = Path.Combine(Path.GetTempPath(), "backfill-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(dir); + using var conn = new SqliteConnection("DataSource=:memory:"); + conn.Open(); + try + { + var littersJson = """ + [{"id":"L-A","litterId":"A","date":"01.05.2023","damName":"Mutter [ZdkC]","sireName":"Vater [ZdkC]","totalBorn":3,"zuchtnummer":"","note":""}] + """; + // Run 1: Vater is in conflict -> not loaded + var animals1 = """ + [ + {"id":"mutter","name":"Mutter [ZdkC]","dob":"01.01.2021","death":"","farbschlag":"","gender":"female","zuchtCanon":"kleinechaote", + "genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":false}, + {"id":"vater","name":"Vater [ZdkC]","dob":"02.02.2021","death":"","farbschlag":"","gender":"male","zuchtCanon":"kleinechaote", + "genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":true}, + {"id":"kind","name":"Kind [ZdkC]","dob":"01.05.2023","death":"","farbschlag":"","gender":null,"zuchtCanon":"kleinechaote", + "genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":false, + "litterRef":{"litterId":"L-A","method":"geburtsdatum+eltern","confidence":"hoch"}} + ] + """; + File.WriteAllText(Path.Combine(dir, "litters.json"), littersJson); + File.WriteAllText(Path.Combine(dir, "animals.json"), animals1); + + var opts = new DbContextOptionsBuilder().UseSqlite(conn).Options; + using var db = new ApplicationContext(opts); + await db.Database.EnsureCreatedAsync(); + + var report1 = await new ImportService(db, dir, dir).RunAsync(execute: true); + Assert.Equal(0, report1.Litters.ParentFksBackfilled); + var litter1 = await db.Litters.SingleAsync(l => l.Name == "Wurf A"); + Assert.Null(litter1.FatherId); // Vater was quarantined -> null FK + Assert.NotNull(litter1.MotherId); // Mutter was loaded -> set + + // Run 2: Vater is now conflict=false + var animals2 = """ + [ + {"id":"mutter","name":"Mutter [ZdkC]","dob":"01.01.2021","death":"","farbschlag":"","gender":"female","zuchtCanon":"kleinechaote", + "genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":false}, + {"id":"vater","name":"Vater [ZdkC]","dob":"02.02.2021","death":"","farbschlag":"","gender":"male","zuchtCanon":"kleinechaote", + "genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":false}, + {"id":"kind","name":"Kind [ZdkC]","dob":"01.05.2023","death":"","farbschlag":"","gender":null,"zuchtCanon":"kleinechaote", + "genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":false, + "litterRef":{"litterId":"L-A","method":"geburtsdatum+eltern","confidence":"hoch"}} + ] + """; + File.WriteAllText(Path.Combine(dir, "animals.json"), animals2); + + var report2 = await new ImportService(db, dir, dir).RunAsync(execute: true); + Assert.Equal(1, report2.Litters.ParentFksBackfilled); // backfill happened + var vater = await db.Gerbils.SingleAsync(g => g.ExternalRef == "vater"); + var litter2 = await db.Litters.SingleAsync(l => l.Name == "Wurf A"); + Assert.Equal(vater.Id, litter2.FatherId); // FK now set + } + finally + { + try { Directory.Delete(dir, recursive: true); } catch { } + } + } + + [Fact] + public async Task ParentFkBackfill_dry_run_counts_without_writing() + { + // Dry-run on a DB with an existing null-parent litter should predict the backfill count. + var dir = Path.Combine(Path.GetTempPath(), "backfill-dr-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(dir); + using var conn = new SqliteConnection("DataSource=:memory:"); + conn.Open(); + try + { + var littersJson = """ + [{"id":"L-B","litterId":"B","date":"15.06.2023","damName":"Mami [ZdkC]","sireName":"Papi [ZdkC]","totalBorn":2,"zuchtnummer":"","note":""}] + """; + var animals1 = """ + [ + {"id":"mami","name":"Mami [ZdkC]","dob":"03.03.2021","death":"","farbschlag":"","gender":"female","zuchtCanon":"kleinechaote", + "genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":false}, + {"id":"papi","name":"Papi [ZdkC]","dob":"04.04.2021","death":"","farbschlag":"","gender":"male","zuchtCanon":"kleinechaote", + "genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":true} + ] + """; + File.WriteAllText(Path.Combine(dir, "litters.json"), littersJson); + File.WriteAllText(Path.Combine(dir, "animals.json"), animals1); + + var opts = new DbContextOptionsBuilder().UseSqlite(conn).Options; + using var db = new ApplicationContext(opts); + await db.Database.EnsureCreatedAsync(); + await new ImportService(db, dir, dir).RunAsync(execute: true); // run 1 + + // Run 2 dry-run with papi un-quarantined + var animals2 = animals1.Replace("\"conflict\":true", "\"conflict\":false"); + File.WriteAllText(Path.Combine(dir, "animals.json"), animals2); + var dry = await new ImportService(db, dir, dir).RunAsync(execute: false); + + Assert.Equal(1, dry.Litters.ParentFksBackfilled); // predicted but not written + var litter = await db.Litters.SingleAsync(l => l.Name == "Wurf B"); + Assert.Null(litter.FatherId); // not written in dry-run + } + finally + { + try { Directory.Delete(dir, recursive: true); } catch { } + } + } + [Theory] [InlineData("01.02.2020", 2020, 2, 1)] [InlineData("5.3.21", 2021, 3, 5)] diff --git a/GerbilManagerWebAPI/Import/ImportModels.cs b/GerbilManagerWebAPI/Import/ImportModels.cs index 51a76cf..449c875 100644 --- a/GerbilManagerWebAPI/Import/ImportModels.cs +++ b/GerbilManagerWebAPI/Import/ImportModels.cs @@ -86,7 +86,8 @@ namespace GerbilManagerWebAPI.Import public sealed record ResidencySummary(int Resident, int External, int FlippedByParentRule); public sealed record LitterSummary(int InSource, int Created, int AlreadyImported, - int DerivedFromChart = 0, int DerivedSkipped = 0, int ParentFksDropped = 0); + int DerivedFromChart = 0, int DerivedSkipped = 0, int ParentFksDropped = 0, + int ParentFksBackfilled = 0); public sealed record AnimalSummary( int InSource, diff --git a/GerbilManagerWebAPI/Import/ImportService.cs b/GerbilManagerWebAPI/Import/ImportService.cs index f664126..ae62868 100644 --- a/GerbilManagerWebAPI/Import/ImportService.cs +++ b/GerbilManagerWebAPI/Import/ImportService.cs @@ -399,10 +399,55 @@ namespace GerbilManagerWebAPI.Import await _db.SaveChangesAsync(); } + // PARENT-FK BACKFILL (idempotent re-run): already-imported Wurfchronik litters that + // have null Father/MotherId because the parent was previously quarantined may now be + // resolvable if that parent is loadable in this run. Counted for dry-run too. + int parentFksBackfilled = 0; + { + var existingWithNullParent = await _db.Litters + .Where(l => l.FatherId == null || l.MotherId == null) + .Select(l => new { l.Id, l.Name, l.FatherId, l.MotherId }) + .ToListAsync(); + var sourceByName = litters + .GroupBy(sl => $"Wurf {sl.LitterId}".Trim()) + .ToDictionary(g => g.Key, g => g.First()); + foreach (var el in existingWithNullParent) + { + if (!sourceByName.TryGetValue(el.Name, out var sl)) continue; + Guid? newF = null, newM = null; + if (el.FatherId == null) + { + var n = Normalize(StripZucht(sl.SireName)); + if (n.Length > 0 && createdAnimalByName.TryGetValue(n, out var fid) && persisted.Contains(fid)) + newF = fid; + } + if (el.MotherId == null) + { + var n = Normalize(StripZucht(sl.DamName)); + if (n.Length > 0 && createdAnimalByName.TryGetValue(n, out var mid) && persisted.Contains(mid)) + newM = mid; + } + if (newF is null && newM is null) continue; + parentFksBackfilled++; + if (execute) + { + var row = await _db.Litters.FirstOrDefaultAsync(l => l.Id == el.Id); + if (row is not null) + { + if (newF is not null) row.FatherId = newF; + if (newM is not null) row.MotherId = newM; + } + } + } + if (execute && parentFksBackfilled > 0) await _db.SaveChangesAsync(); + } + notes.Add("Quarantäne (kein Import): Konflikte + Stubs ohne Geburtsdatum + unsichere Wurf-Zuordnungen — warten auf die Prüfung durch die Züchterin."); if (parentLinksAdded > 0) notes.Add($"Stammbaum-Diagramm: {parentLinksAdded} Tiere über Eltern-Verknüpfung einem (abgeleiteten) Wurf zugeordnet ({derivedLitters} abgeleitete Würfe)."); notes.Add($"FK-Integrität: {litterParentFksDropped} Eltern-Verknüpfung(en) verworfen (Elternteil nicht ladbar), {derivedLittersSkipped} abgeleitete Würfe übersprungen (kein ladbares Elternteil). Bei 0/0 ist /import/execute FK-sicher."); + if (parentFksBackfilled > 0) + notes.Add($"Parent-FK-Backfill: {parentFksBackfilled} bereits importierte Würfe haben jetzt eine Eltern-Verknüpfung (Elternteil war zuvor in Quarantäne, jetzt geladen)."); notes.Add($"Bestand/Herkunft: {residentTotal} im Bestand (Clan Kleine Chaoten), {externalTotal} externe Ahnen ({flippedByParentRule} davon über die Eltern-Regel als Bestand erkannt)."); int conflictsResolvedByDecision = loadable.Count(a => a.ResolvedByDecision); if (conflictsResolvedByDecision > 0) @@ -411,7 +456,7 @@ namespace GerbilManagerWebAPI.Import return new ImportReport( Executed: execute, - Litters: new LitterSummary(litters.Count, littersCreated, littersExisting, derivedLitters, derivedLittersSkipped, litterParentFksDropped), + Litters: new LitterSummary(litters.Count, littersCreated, littersExisting, derivedLitters, derivedLittersSkipped, litterParentFksDropped, parentFksBackfilled), Animals: new AnimalSummary( animals.Count, animalsCreated, linked, fbMatched, fbUnmatched, animalsExisting, new QuarantineSummary(conflicts, stubs, dateOnly, ambiguous, conflicts + stubs), diff --git a/tools/import/conflict-decisions.json b/tools/import/conflict-decisions.json index 40d6df2..d21e166 100644 --- a/tools/import/conflict-decisions.json +++ b/tools/import/conflict-decisions.json @@ -81,7 +81,7 @@ { "name": "Victoria Welby gen. Welby v.d. Kleinen Chaoten", "dob": "16.01.2023", - "decision": "E-locus = ee[f] (Fuchs). NOTE: this is the mother of animal 'C' (c-29042024) — un-quarantining her links C's second parent. Name kept in the merged record's v.d. spelling: extract.py decision matching uses norm_name (no v.d.<->von den fold) — workaround until the canon_pair matching fix lands.", + "decision": "E-locus = ee[f] (Fuchs). This is the mother of animal 'C' (c-29042024) — un-quarantining her links C's second parent. Name in v.d. spelling (workaround from Re-Import #2); both spellings now match after FIX-1 (canon_pair identity).", "genotype": "Aa CC D- ee[f] Gg pp Spsp [DP]", "source": "Julian 2026-06-06 — HUMANQUESTION D4" } diff --git a/tools/import/extract.py b/tools/import/extract.py index d892bde..263af89 100644 --- a/tools/import/extract.py +++ b/tools/import/extract.py @@ -158,6 +158,10 @@ def parse_detail(text): tail = text[dob.end():] tail = re.sub(r"^\s*/?\+?\s?\d[\d.]*", "", tail) # drop any /+death remnant tail = tail.lstrip(" ,").strip() + # FIX-4 (Skarlett): strip trailing "/ +YEAR" death-year artifacts leaked from compact + # chart cells (e.g. "… rere / +2018"). The DEATH regex still captures the year from + # the full cell text, so it appears as a death-date conflict — not a genotype conflict. + tail = re.sub(r"\s*/\s*\+\d{4}\s*$", "", tail).strip() if gt.looks_like_genotype(tail): geno = tail return (dob.group(1) if dob else "", @@ -518,10 +522,11 @@ def _alleles_compatible(a, b): if a == b: return True if a == "?" or b == "?": - return False # unknown vs filled = contradiction (D- vs DD) + return True # specific-wins: unknown allele is compatible with any + # specified value (C- vs CC -> CC; G- vs Gg -> Gg) (ba, ma), (bb, mb) = _split_allele(a), _split_allele(b) if ba != bb: - return False # different base allele = real value diff (E vs e) + return False # different base allele = real value diff (E vs e, D vs d) return ma == "" or mb == "" # same base, modifier present-vs-absent -> presence wins @@ -887,21 +892,22 @@ def apply_dob_remaps(raw_animals, path): """PRE-dedup: a conflict-decision carrying `correctDob` marks a record as a DUPLICATE with a wrong birthdate — remap that raw record's DOB to correctDob so dedup MERGES it into the canonical same-named animal (e.g. Chelsea *15.10.2021 -> *02.04.2021). Match = - norm_name(name)+norm_dob(dob). Tolerates a missing/garbled file. Returns the remap count. + canon_pair(name)[0]+norm_dob(dob) (same identity as dedup — strips zucht suffix, folds + v.d.<->von den). Tolerates a missing/garbled file. Returns the remap count. Must run BEFORE dedup (it changes the dedup identity). (god/HUMANQUESTION D — Dubletten.)""" remaps = {} try: with open(path, encoding="utf-8") as fh: for r in (json.load(fh).get("resolutions") or []): if r.get("correctDob"): - remaps[(norm_name(r.get("name", "")), norm_dob(r.get("dob", "")))] = r["correctDob"] + remaps[(canon_pair(r.get("name", ""))[0], norm_dob(r.get("dob", "")))] = r["correctDob"] except (OSError, ValueError): return 0 if not remaps: return 0 n = 0 for a in raw_animals: - new = remaps.get((norm_name(a.get("name", "")), norm_dob(a.get("dob", "")))) + new = remaps.get((canon_pair(a.get("name", ""))[0], norm_dob(a.get("dob", "")))) if new and a.get("dob") != new: a["dob"] = new n += 1 @@ -911,15 +917,18 @@ def apply_dob_remaps(raw_animals, path): def apply_conflict_decisions(merged, conflicts, path): """Consume human conflict resolutions (tools/import/conflict-decisions.json) so the wife's answers UN-QUARANTINE animals. Schema: {"resolutions":[{name, dob, decision, genotype?, - farbschlag?, source}]}. Match = norm_name(name)+norm_dob(dob) (same identity as dedup). A - matching animal: clear its conflict, mark resolvedByDecision; an explicit `genotype` - (breeder notation) is parsed and becomes authoritative, `farbschlag` overrides too. Tolerates - a missing/empty/garbled file. Returns the number of conflicts resolved. (god/HUMANQUESTION D.)""" + farbschlag?, source}]}. Match = canon_pair(name)[0]+norm_dob(dob) — the same dedup identity + (call-name only, zucht stripped, v.d.<->von den folded). A matching animal: clear its + conflict, mark resolvedByDecision; an explicit `genotype` (breeder notation) is parsed and + becomes authoritative, `farbschlag` overrides too. Tolerates a missing/empty/garbled file. + Returns the number of conflicts resolved. (god/HUMANQUESTION D.)""" decisions = {} try: with open(path, encoding="utf-8") as fh: for r in (json.load(fh).get("resolutions") or []): - decisions[(norm_name(r.get("name", "")), norm_dob(r.get("dob", "")))] = r + # FIX-1: use dedup identity (call-name only, zucht stripped) so that e.g. + # a decision written as "von den" matches a merged record with "v.d." spelling. + decisions[(canon_pair(r.get("name", ""))[0], norm_dob(r.get("dob", "")))] = r except (OSError, ValueError): return 0 if not decisions: @@ -927,7 +936,7 @@ def apply_conflict_decisions(merged, conflicts, path): resolved = 0 for a in merged: - d = decisions.get((norm_name(a["name"]), norm_dob(a["dob"]))) + d = decisions.get((canon_pair(a["name"])[0], norm_dob(a["dob"]))) if not d: continue a["resolvedByDecision"] = True diff --git a/tools/import/test_extract.py b/tools/import/test_extract.py index ab388c2..f8643ec 100644 --- a/tools/import/test_extract.py +++ b/tools/import/test_extract.py @@ -102,6 +102,39 @@ check("apply_conflict_decisions returns resolved count", n == 2) check("missing decisions file tolerated (returns 0)", e.apply_conflict_decisions([], [], os.path.join(tempfile.gettempdir(), "does-not-exist.json")) == 0) +# FIX-1: decision matching uses canon_pair identity -> 'von den' decision matches 'v.d.' record +dec_vd = os.path.join(tempfile.gettempdir(), "decisions-vd.json") +_json.dump({"resolutions": [ + {"name": "Victoria Welby gen. Welby von den Kleinen Chaoten", # written with 'von den' + "dob": "16.01.2023", "decision": "E-locus = ee[f]", + "genotype": "Aa CC D- ee[f] Gg pp Spsp", "source": "test"}, +]}, open(dec_vd, "w", encoding="utf-8")) +merged_vd = [ + {"id": "vw", "name": "Victoria Welby gen. Welby v.d. Kleinen Chaoten", # record has 'v.d.' + "dob": "16.01.2023", "conflict": True, "farbschlag": "", "death": "", + "genotype": {"mapped8locus": {}, "rawGenotype": "", "unmappedTokens": []}}, +] +conflicts_vd = [{"id": "vw"}] +n_vd = e.apply_conflict_decisions(merged_vd, conflicts_vd, dec_vd) +check("FIX-1: 'von den' decision matches 'v.d.' record (canon_pair identity)", n_vd == 1) +check("FIX-1: conflict cleared for v.d. record", merged_vd[0]["conflict"] is False) +# Also verify the workaround spelling (v.d. in decision) matches a 'von den' record +_json.dump({"resolutions": [ + {"name": "Victoria Welby gen. Welby v.d. Kleinen Chaoten", # workaround: v.d. in decision + "dob": "16.01.2023", "decision": "E-locus = ee[f]", + "genotype": "Aa CC D- ee[f] Gg pp Spsp", "source": "test"}, +]}, open(dec_vd, "w", encoding="utf-8")) +merged_vd2 = [ + {"id": "vw2", "name": "Victoria Welby gen. Welby von den Kleinen Chaoten", # record 'von den' + "dob": "16.01.2023", "conflict": True, "farbschlag": "", "death": "", + "genotype": {"mapped8locus": {}, "rawGenotype": "", "unmappedTokens": []}}, +] +conflicts_vd2 = [{"id": "vw2"}] +n_vd2 = e.apply_conflict_decisions(merged_vd2, conflicts_vd2, dec_vd) +check("FIX-1: v.d. decision also matches 'von den' record (both spellings match)", n_vd2 == 1) +try: os.remove(dec_vd) +except OSError: pass + # --- correctDob: a wrong-birthdate duplicate is remapped BEFORE dedup so it merges --- dec2 = os.path.join(tempfile.gettempdir(), "decisions-dob.json") _json.dump({"resolutions": [ @@ -128,23 +161,52 @@ except OSError: pass try: os.remove(dec_path) except OSError: pass -# --- "presence wins" conflict rule (Julian) --- -# present-vs-absent (whole locus or [f] modifier) is NOT a conflict; differing filled values are. +# --- "presence wins" + "specific wins" conflict rules (Julian) --- +# present-vs-absent (whole locus or [f] modifier) is NOT a conflict; differing FILLED values are. +# FIX-2 (specific-wins): unknown allele '?' vs any specified value is also NOT a conflict — +# the specific value wins (C- vs CC -> CC; G- vs Gg -> Gg; P? vs PP -> PP). check("spsp present vs locus absent -> no conflict", not e._genotype_conflict([{"Sp": ["sp", "sp"]}, {}])) check("ee[f] vs ee ([f] modifier present/absent) -> no conflict", not e._genotype_conflict([{"E": ["e", "e^f"]}, {"E": ["e", "e"]}])) -check("DD vs D- (unknown vs filled) -> conflict", - e._genotype_conflict([{"D": ["D", "D"]}, {"D": ["D", "?"]}])) +# FIX-2: '?' vs specified = specific wins (was: contradiction) +check("FIX-2: DD vs D- (specific wins: DD wins) -> NOT conflict", + not e._genotype_conflict([{"D": ["D", "D"]}, {"D": ["D", "?"]}])) +check("FIX-2: C- vs Cc[h] (specific wins: c^h wins) -> NOT conflict", + not e._genotype_conflict([{"C": ["C", "?"]}, {"C": ["C", "c^h"]}])) +check("FIX-2: C- vs CC (specific wins: CC) -> NOT conflict", + not e._genotype_conflict([{"C": ["C", "?"]}, {"C": ["C", "C"]}])) +check("FIX-2: G- vs Gg (specific wins) -> NOT conflict", + not e._genotype_conflict([{"G": ["G", "?"]}, {"G": ["G", "g"]}])) +check("FIX-2: PP vs P? (specific wins: PP) -> NOT conflict", + not e._genotype_conflict([{"P": ["P", "P"]}, {"P": ["P", "?"]}])) +# Genuine value contradictions (both alleles specified but different) still quarantine check("Ee vs ee (different base allele) -> conflict", e._genotype_conflict([{"E": ["E", "e"]}, {"E": ["e", "e"]}])) -check("C- vs Cc[h] -> conflict", - e._genotype_conflict([{"C": ["C", "?"]}, {"C": ["C", "c^h"]}])) -check("c[h] vs c[chm] (different modifiers) -> conflict", +check("DD vs Dd (both specified, D vs d) -> conflict", + e._genotype_conflict([{"D": ["D", "D"]}, {"D": ["D", "d"]}])) +check("PP vs Pp (both specified) -> conflict", + e._genotype_conflict([{"P": ["P", "P"]}, {"P": ["P", "p"]}])) +check("c[h] vs c[chm] (different modifiers, both specified) -> conflict", not e._alleles_compatible("c^h", "c^chm")) check("identical genotypes -> no conflict", not e._genotype_conflict([{"A": ["A", "a"]}, {"A": ["A", "a"]}])) +# --- FIX-4: Skarlett parse artifact — trailing "/ +YEAR" stripped from geno, death captured --- +dob4, death4, geno4 = e.parse_detail("Skarlett,*17.04.2016, aa C- DD ee Gg PP spsp rere / +2018") +check("FIX-4: '/ +YEAR' artifact stripped from geno tail", + geno4 == "aa C- DD ee Gg PP spsp rere") +check("FIX-4: death year still captured from full cell text", + death4 == "2018") +check("FIX-4: DOB still correct", + dob4 == "17.04.2016") +# Without artifact — must be unchanged +dob5, death5, geno5 = e.parse_detail("*01.01.2020, aa C- DD ee Gg PP spsp rere") +check("FIX-4: no artifact -> geno unchanged", + geno5 == "aa C- DD ee Gg PP spsp rere") +check("FIX-4: no artifact -> no spurious death", + death5 == "") + # --- name-bleed guard (a parent name is not a Farbschlag) --- check("v.d. name rejected", e.looks_like_animal_name("Tennessee von den Kleinen Chaoten")) check("gen.+v.d. name rejected", e.looks_like_animal_name("Victoria Welby gen. Welby v.d. Kleinen Chaoten")) From dfcd296119182602dccb01d48493ae073ab70953 Mon Sep 17 00:00:00 2001 From: Gulum Date: Sat, 6 Jun 2026 15:03:17 +0200 Subject: [PATCH 2/2] IMPORT-POLISH v2: 3 Korrekturen nach god-Review FIX-1 ZUCHT: apply_conflict_decisions/apply_dob_remaps matchen jetzt auf das VOLLE canon_pair-Tupel (nameCanon, zuchtCanon, dob) wenn die Decision eine Zucht traegt; Fallback name-only wenn keine Zucht. C3-Regel gewahrt: gleicher Name+DOB, andere Zucht -> kein Hit. Neuer Regression-Test: Luna ZdkC-Decision trifft nur luna-kc, nicht luna-bf (andere Zucht). FIX-2 MERGE: dedup() waehlt das spezifischste Genotyp (fewest '?' alleles) als sekundaeren Tiebreaker nach locus-count. CC schlaegt C-, Gg schlaegt G- unabhaengig von der Reihenfolge. 3 neue Merge-Tests (C- first/CC first/G-vsGg). FIX-3 BACKFILL allDbNormToGid: ResolveParentForBackfill prueft jetzt BEIDE Quellen: (a) createdAnimalByName (aktiver Lauf) und (b) allDbNormToGid (alle DB-Tiere). Decktt den kritischen Fall: Elterntier in fruehrem Lauf geladen, in diesem Lauf absent vom Extract. Neuer 3-Lauf-SQLite-Test: Lauf 1 null-Vater, Lauf 2 laedt Vater, Lauf 3 backfillt via allDbNormToGid. Gate: 125/125 C#-Tests, Python ALL PASS, has-pending-model-changes=No. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- GerbilManager.Tests/ImportServiceTests.cs | 80 ++++++++++++++++++++- GerbilManagerWebAPI/Import/ImportService.cs | 40 +++++++---- tools/import/extract.py | 57 ++++++++++----- tools/import/test_extract.py | 67 +++++++++++++++++ 4 files changed, 210 insertions(+), 34 deletions(-) diff --git a/GerbilManager.Tests/ImportServiceTests.cs b/GerbilManager.Tests/ImportServiceTests.cs index 450d9fc..f40fd14 100644 --- a/GerbilManager.Tests/ImportServiceTests.cs +++ b/GerbilManager.Tests/ImportServiceTests.cs @@ -326,7 +326,8 @@ namespace GerbilManager.Tests { // Run 1: litter "Wurf A" has sire "Vater" (conflict=true — not loaded) and dam "Mutter" // (conflict=false — loaded). After run 1: litter.FatherId = null. - // Run 2: sire "Vater" is no longer in conflict. Backfill must set litter.FatherId. + // Run 2: sire "Vater" now conflict=false → loaded as NEW in run 2. Backfill via + // createdAnimalByName sets FatherId. (god steering point 3: run-2 path.) var dir = Path.Combine(Path.GetTempPath(), "backfill-" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(dir); using var conn = new SqliteConnection("DataSource=:memory:"); @@ -361,7 +362,7 @@ namespace GerbilManager.Tests Assert.Null(litter1.FatherId); // Vater was quarantined -> null FK Assert.NotNull(litter1.MotherId); // Mutter was loaded -> set - // Run 2: Vater is now conflict=false + // Run 2: Vater now conflict=false -> loaded as NEW animal in this run var animals2 = """ [ {"id":"mutter","name":"Mutter [ZdkC]","dob":"01.01.2021","death":"","farbschlag":"","gender":"female","zuchtCanon":"kleinechaote", @@ -387,6 +388,81 @@ namespace GerbilManager.Tests } } + [Fact] + public async Task ParentFkBackfill_uses_allDb_lookup_when_parent_not_in_current_loadable() + { + // god steering point 3: the main case — parent was loaded in a PREVIOUS run (not in + // the current run's animals.json at all). Backfill must find them via allDbNormToGid. + // + // Run 1: litter "Wurf C" + dam loaded, sire quarantined -> FatherId null. + // Run 2: sire loaded (new animal). + // Run 3: animals.json has ONLY the kind (sire absent from extract). Sire is in DB + // from run 2 but NOT in the current run's loadable/createdAnimalByName. + // Backfill must use allDbNormToGid to find him. + var dir = Path.Combine(Path.GetTempPath(), "backfill-db-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(dir); + using var conn = new SqliteConnection("DataSource=:memory:"); + conn.Open(); + try + { + var littersJson = """ + [{"id":"L-C","litterId":"C","date":"10.06.2023","damName":"Dame [ZdkC]","sireName":"Herr [ZdkC]","totalBorn":2,"zuchtnummer":"","note":""}] + """; + // Run 1: sire quarantined + File.WriteAllText(Path.Combine(dir, "litters.json"), littersJson); + File.WriteAllText(Path.Combine(dir, "animals.json"), """ + [ + {"id":"dame","name":"Dame [ZdkC]","dob":"05.05.2021","death":"","farbschlag":"","gender":"female","zuchtCanon":"kleinechaote", + "genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":false}, + {"id":"herr","name":"Herr [ZdkC]","dob":"06.06.2021","death":"","farbschlag":"","gender":"male","zuchtCanon":"kleinechaote", + "genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":true} + ] + """); + var opts = new DbContextOptionsBuilder().UseSqlite(conn).Options; + using var db = new ApplicationContext(opts); + await db.Database.EnsureCreatedAsync(); + await new ImportService(db, dir, dir).RunAsync(execute: true); + Assert.Null((await db.Litters.SingleAsync(l => l.Name == "Wurf C")).FatherId); + + // Run 2: sire now loaded + File.WriteAllText(Path.Combine(dir, "animals.json"), """ + [ + {"id":"dame","name":"Dame [ZdkC]","dob":"05.05.2021","death":"","farbschlag":"","gender":"female","zuchtCanon":"kleinechaote", + "genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":false}, + {"id":"herr","name":"Herr [ZdkC]","dob":"06.06.2021","death":"","farbschlag":"","gender":"male","zuchtCanon":"kleinechaote", + "genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":false} + ] + """); + await new ImportService(db, dir, dir).RunAsync(execute: true); + var herrId = (await db.Gerbils.SingleAsync(g => g.ExternalRef == "herr")).Id; + // Run 2 itself may or may not backfill (depends on name normalization alignment). + // For the test we care about run 3. + + // Run 3: sire NOT in animals.json at all (absent from new extract). + // litter still has FatherId=null if run 2 didn't backfill; if it did, we simulate + // by manually resetting FatherId to null so run 3 must fix it. + var litter3 = await db.Litters.SingleAsync(l => l.Name == "Wurf C"); + litter3.FatherId = null; + await db.SaveChangesAsync(); + + File.WriteAllText(Path.Combine(dir, "animals.json"), """ + [ + {"id":"dame","name":"Dame [ZdkC]","dob":"05.05.2021","death":"","farbschlag":"","gender":"female","zuchtCanon":"kleinechaote", + "genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":false} + ] + """); + // Run 3: sire absent from loadable (NOT in createdAnimalByName), but IS in DB. + var report3 = await new ImportService(db, dir, dir).RunAsync(execute: true); + Assert.Equal(1, report3.Litters.ParentFksBackfilled); // allDbNormToGid path + var litter3After = await db.Litters.SingleAsync(l => l.Name == "Wurf C"); + Assert.Equal(herrId, litter3After.FatherId); // FK set from DB lookup + } + finally + { + try { Directory.Delete(dir, recursive: true); } catch { } + } + } + [Fact] public async Task ParentFkBackfill_dry_run_counts_without_writing() { diff --git a/GerbilManagerWebAPI/Import/ImportService.cs b/GerbilManagerWebAPI/Import/ImportService.cs index ae62868..da85725 100644 --- a/GerbilManagerWebAPI/Import/ImportService.cs +++ b/GerbilManagerWebAPI/Import/ImportService.cs @@ -401,9 +401,20 @@ namespace GerbilManagerWebAPI.Import // PARENT-FK BACKFILL (idempotent re-run): already-imported Wurfchronik litters that // have null Father/MotherId because the parent was previously quarantined may now be - // resolvable if that parent is loadable in this run. Counted for dry-run too. + // resolvable. Two lookup sources — must check BOTH: + // (a) createdAnimalByName: animals loaded/re-linked in THIS run (new or existing). + // (b) allDbNormToGid: ALL gerbils already in the DB, for parents loaded in an + // EARLIER run who are no longer in the current extract (e.g. alreadyImported + // animals absent from this run's animals.json, or name normalization mismatch + // between animals.json and the Wurfchronik sire/dam field). + // Counted for dry-run too; writes only when execute=true. int parentFksBackfilled = 0; { + // Build DB-wide normalized-name lookup (supplementary to createdAnimalByName). + var allDbNormToGid = existingRows + .GroupBy(g => Normalize(StripZucht(g.Name))) + .ToDictionary(grp => grp.Key, grp => grp.First().Id); + var existingWithNullParent = await _db.Litters .Where(l => l.FatherId == null || l.MotherId == null) .Select(l => new { l.Id, l.Name, l.FatherId, l.MotherId }) @@ -411,22 +422,23 @@ namespace GerbilManagerWebAPI.Import var sourceByName = litters .GroupBy(sl => $"Wurf {sl.LitterId}".Trim()) .ToDictionary(g => g.Key, g => g.First()); + + Guid? ResolveParentForBackfill(string rawName) + { + var n = Normalize(StripZucht(rawName)); + if (n.Length == 0) return null; + if (createdAnimalByName.TryGetValue(n, out var fromLoadable) && persisted.Contains(fromLoadable)) + return fromLoadable; + if (allDbNormToGid.TryGetValue(n, out var fromDb) && persisted.Contains(fromDb)) + return fromDb; + return null; + } + foreach (var el in existingWithNullParent) { if (!sourceByName.TryGetValue(el.Name, out var sl)) continue; - Guid? newF = null, newM = null; - if (el.FatherId == null) - { - var n = Normalize(StripZucht(sl.SireName)); - if (n.Length > 0 && createdAnimalByName.TryGetValue(n, out var fid) && persisted.Contains(fid)) - newF = fid; - } - if (el.MotherId == null) - { - var n = Normalize(StripZucht(sl.DamName)); - if (n.Length > 0 && createdAnimalByName.TryGetValue(n, out var mid) && persisted.Contains(mid)) - newM = mid; - } + var newF = el.FatherId == null ? ResolveParentForBackfill(sl.SireName) : null; + var newM = el.MotherId == null ? ResolveParentForBackfill(sl.DamName) : null; if (newF is null && newM is null) continue; parentFksBackfilled++; if (execute) diff --git a/tools/import/extract.py b/tools/import/extract.py index 263af89..f8c53b3 100644 --- a/tools/import/extract.py +++ b/tools/import/extract.py @@ -627,9 +627,12 @@ def dedup(animals): if a.get("deaf") is not None: deaf_seen.add(a["deaf"]) tags_set.update(a.get("tags", [])) - # pick the richest genotype (most mapped loci, then longest raw) + # pick the richest genotype: most mapped loci, then fewest unknowns ('?' alleles = specific + # wins, FIX-2), then longest raw string as final tiebreaker. + def _specificity(gd): + return sum(1 for pair in gd["mapped8locus"].values() for a in pair if a != "?") best = max((a["genotype"] for a in grp), - key=lambda gd: (len(gd["mapped8locus"]), len(gd["rawGenotype"]))) + key=lambda gd: (len(gd["mapped8locus"]), _specificity(gd), len(gd["rawGenotype"]))) out = { "id": slug(base["name"], base["dob"]), "name": base["name"], @@ -892,22 +895,30 @@ def apply_dob_remaps(raw_animals, path): """PRE-dedup: a conflict-decision carrying `correctDob` marks a record as a DUPLICATE with a wrong birthdate — remap that raw record's DOB to correctDob so dedup MERGES it into the canonical same-named animal (e.g. Chelsea *15.10.2021 -> *02.04.2021). Match = - canon_pair(name)[0]+norm_dob(dob) (same identity as dedup — strips zucht suffix, folds - v.d.<->von den). Tolerates a missing/garbled file. Returns the remap count. + canon_pair(name)+(dob) with same Zucht-aware logic as apply_conflict_decisions (see there). + Tolerates a missing/garbled file. Returns the remap count. Must run BEFORE dedup (it changes the dedup identity). (god/HUMANQUESTION D — Dubletten.)""" - remaps = {} + remaps_full = {} # (nameCanon, zuchtCanon, dob) -> correctDob — decision carries Zucht + remaps_name = {} # (nameCanon, dob) -> correctDob — no Zucht in decision try: with open(path, encoding="utf-8") as fh: for r in (json.load(fh).get("resolutions") or []): if r.get("correctDob"): - remaps[(canon_pair(r.get("name", ""))[0], norm_dob(r.get("dob", "")))] = r["correctDob"] + nc, zc = canon_pair(r.get("name", "")) + dob = norm_dob(r.get("dob", "")) + if zc: + remaps_full[(nc, zc, dob)] = r["correctDob"] + else: + remaps_name[(nc, dob)] = r["correctDob"] except (OSError, ValueError): return 0 - if not remaps: + if not remaps_full and not remaps_name: return 0 n = 0 for a in raw_animals: - new = remaps.get((canon_pair(a.get("name", ""))[0], norm_dob(a.get("dob", "")))) + nc, zc = canon_pair(a.get("name", "")) + dob = norm_dob(a.get("dob", "")) + new = remaps_full.get((nc, zc, dob)) or remaps_name.get((nc, dob)) if new and a.get("dob") != new: a["dob"] = new n += 1 @@ -917,26 +928,36 @@ def apply_dob_remaps(raw_animals, path): def apply_conflict_decisions(merged, conflicts, path): """Consume human conflict resolutions (tools/import/conflict-decisions.json) so the wife's answers UN-QUARANTINE animals. Schema: {"resolutions":[{name, dob, decision, genotype?, - farbschlag?, source}]}. Match = canon_pair(name)[0]+norm_dob(dob) — the same dedup identity - (call-name only, zucht stripped, v.d.<->von den folded). A matching animal: clear its - conflict, mark resolvedByDecision; an explicit `genotype` (breeder notation) is parsed and - becomes authoritative, `farbschlag` overrides too. Tolerates a missing/empty/garbled file. + farbschlag?, source}]}. Match = canon_pair(name)+(dob): + - When the decision name CARRIES a Zucht (zuchtCanon != ''), match on the FULL + (nameCanon, zuchtCanon, dob) triple — preserves the C3 rule that same name+DOB but + different Zucht = different animal. + - When the decision has NO Zucht, fall back to (nameCanon, dob) name-only match. + Both spellings v.d. / von den fold to the same canon. A matching animal: clear its conflict, + mark resolvedByDecision; an explicit `genotype` (breeder notation) is parsed and becomes + authoritative, `farbschlag` overrides too. Tolerates a missing/empty/garbled file. Returns the number of conflicts resolved. (god/HUMANQUESTION D.)""" - decisions = {} + decisions_full = {} # (nameCanon, zuchtCanon, dob) -> r — when decision carries a Zucht + decisions_name = {} # (nameCanon, dob) -> r — fallback, decision has no Zucht try: with open(path, encoding="utf-8") as fh: for r in (json.load(fh).get("resolutions") or []): - # FIX-1: use dedup identity (call-name only, zucht stripped) so that e.g. - # a decision written as "von den" matches a merged record with "v.d." spelling. - decisions[(canon_pair(r.get("name", ""))[0], norm_dob(r.get("dob", "")))] = r + nc, zc = canon_pair(r.get("name", "")) + dob = norm_dob(r.get("dob", "")) + if zc: + decisions_full[(nc, zc, dob)] = r + else: + decisions_name[(nc, dob)] = r except (OSError, ValueError): return 0 - if not decisions: + if not decisions_full and not decisions_name: return 0 resolved = 0 for a in merged: - d = decisions.get((canon_pair(a["name"])[0], norm_dob(a["dob"]))) + nc, zc = canon_pair(a["name"]) + dob = norm_dob(a["dob"]) + d = decisions_full.get((nc, zc, dob)) or decisions_name.get((nc, dob)) if not d: continue a["resolvedByDecision"] = True diff --git a/tools/import/test_extract.py b/tools/import/test_extract.py index f8643ec..2c8d9f3 100644 --- a/tools/import/test_extract.py +++ b/tools/import/test_extract.py @@ -135,6 +135,30 @@ check("FIX-1: v.d. decision also matches 'von den' record (both spellings match) try: os.remove(dec_vd) except OSError: pass +# FIX-1 C3-rule: same name+DOB, two Zuchten -> decision hits ONLY the correct Zucht (C3 isolation) +dec_c3 = os.path.join(tempfile.gettempdir(), "decisions-c3.json") +_json.dump({"resolutions": [ + # Decision only for Luna from ZdkC, NOT Luna from Black Forest + {"name": "Luna von den Kleinen Chaoten", "dob": "01.01.2020", + "decision": "D-locus = DD", "genotype": "aa CC DD ee gg PP spsp rere", "source": "test"}, +]}, open(dec_c3, "w", encoding="utf-8")) +merged_c3 = [ + {"id": "luna-kc", "name": "Luna von den Kleinen Chaoten", "dob": "01.01.2020", + "conflict": True, "farbschlag": "", "death": "", + "genotype": {"mapped8locus": {"D": ["D","?"]}, "rawGenotype": "D-", "unmappedTokens": []}}, + {"id": "luna-bf", "name": "Luna of Black Forest", "dob": "01.01.2020", + "conflict": True, "farbschlag": "", "death": "", + "genotype": {"mapped8locus": {"D": ["D","?"]}, "rawGenotype": "D-", "unmappedTokens": []}}, +] +conflicts_c3 = [{"id": "luna-kc"}, {"id": "luna-bf"}] +n_c3 = e.apply_conflict_decisions(merged_c3, conflicts_c3, dec_c3) +check("FIX-1 C3: decision hits only the correct Zucht (luna-kc resolved)", n_c3 == 1) +check("FIX-1 C3: luna-kc conflict cleared (correct Zucht)", merged_c3[0]["conflict"] is False) +check("FIX-1 C3: luna-bf conflict NOT cleared (different Zucht)", merged_c3[1]["conflict"] is True) +check("FIX-1 C3: conflicts list has only luna-bf left", len(conflicts_c3) == 1 and conflicts_c3[0]["id"] == "luna-bf") +try: os.remove(dec_c3) +except OSError: pass + # --- correctDob: a wrong-birthdate duplicate is remapped BEFORE dedup so it merges --- dec2 = os.path.join(tempfile.gettempdir(), "decisions-dob.json") _json.dump({"resolutions": [ @@ -192,6 +216,49 @@ check("c[h] vs c[chm] (different modifiers, both specified) -> conflict", check("identical genotypes -> no conflict", not e._genotype_conflict([{"A": ["A", "a"]}, {"A": ["A", "a"]}])) +# FIX-2 MERGE: specific allele must survive the merge regardless of which variant comes first. +# dedup() picks the most specific genotype (fewest '?' alleles); C- vs CC -> CC must win. +def _minimal_animal(name, dob, mapped): + """Build a minimal raw animal dict suitable for dedup().""" + from genotype import parse as gparse + raw = " ".join(f"{l}{''.join(a)}" for l, pa in mapped.items() for a in [pa]) + return { + "name": name, "dob": dob, "death": "", "gender": None, + "farbschlag": "", "breeder": "", "zucht": "", "parentRefs": [], + "photos": [], "sourceFiles": ["test.xlsx"], "tags": [], + "deaf": None, "conflict": False, + "genotype": {"mapped8locus": mapped, "rawGenotype": raw, "unmappedTokens": []}, + "_gen": 0, "_col": 5, "_row": 10, "_file": "test.xlsx", + "_zucht": "", + } + +# Order A: C- first, CC second +animals_merge_a = [ + _minimal_animal("TestTier", "01.01.2020", {"C": ["C", "?"]}), # C- + _minimal_animal("TestTier", "01.01.2020", {"C": ["C", "C"]}), # CC +] +merged_ma, _, _, _ = e.dedup(animals_merge_a) +check("FIX-2 merge A (C- first): result has CC not C-", + merged_ma[0]["genotype"]["mapped8locus"].get("C") == ["C", "C"]) + +# Order B: CC first, C- second (must give same result) +animals_merge_b = [ + _minimal_animal("TestTier2", "02.02.2020", {"C": ["C", "C"]}), # CC + _minimal_animal("TestTier2", "02.02.2020", {"C": ["C", "?"]}), # C- +] +merged_mb, _, _, _ = e.dedup(animals_merge_b) +check("FIX-2 merge B (CC first): result has CC not C-", + merged_mb[0]["genotype"]["mapped8locus"].get("C") == ["C", "C"]) + +# G- vs Gg: Gg must win +animals_merge_g = [ + _minimal_animal("TestGGerbil", "03.03.2020", {"G": ["G", "?"]}), # G- + _minimal_animal("TestGGerbil", "03.03.2020", {"G": ["G", "g"]}), # Gg +] +merged_mg, _, _, _ = e.dedup(animals_merge_g) +check("FIX-2 merge G (G- vs Gg): Gg wins", + merged_mg[0]["genotype"]["mapped8locus"].get("G") == ["G", "g"]) + # --- FIX-4: Skarlett parse artifact — trailing "/ +YEAR" stripped from geno, death captured --- dob4, death4, geno4 = e.parse_detail("Skarlett,*17.04.2016, aa C- DD ee Gg PP spsp rere / +2018") check("FIX-4: '/ +YEAR' artifact stripped from geno tail",