Gulum/GerbilManager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

IMPORT-POLISH v2: 3 Korrekturen nach god-Review

Browse Source 
FIX-1 ZUCHT: apply_conflict_decisions/apply_dob_remaps matchen jetzt
auf das VOLLE canon_pair-Tupel (nameCanon, zuchtCanon, dob) wenn die
Decision eine Zucht traegt; Fallback name-only wenn keine Zucht.
C3-Regel gewahrt: gleicher Name+DOB, andere Zucht -> kein Hit.
Neuer Regression-Test: Luna ZdkC-Decision trifft nur luna-kc,
nicht luna-bf (andere Zucht).

FIX-2 MERGE: dedup() waehlt das spezifischste Genotyp (fewest '?'
alleles) als sekundaeren Tiebreaker nach locus-count. CC schlaegt C-,
Gg schlaegt G- unabhaengig von der Reihenfolge. 3 neue Merge-Tests
(C- first/CC first/G-vsGg).

FIX-3 BACKFILL allDbNormToGid: ResolveParentForBackfill prueft jetzt
BEIDE Quellen: (a) createdAnimalByName (aktiver Lauf) und
(b) allDbNormToGid (alle DB-Tiere). Decktt den kritischen Fall:
Elterntier in fruehrem Lauf geladen, in diesem Lauf absent vom
Extract. Neuer 3-Lauf-SQLite-Test: Lauf 1 null-Vater, Lauf 2 laedt
Vater, Lauf 3 backfillt via allDbNormToGid.

Gate: 125/125 C#-Tests, Python ALL PASS, has-pending-model-changes=No.

Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Gulum
2026-06-06 15:03:17 +02:00
parent 0c94cfcbf1
commit dfcd296119
4 changed files with 210 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
GerbilManager.Tests/ImportServiceTests.cs
View File

@@ -326,7 +326,8 @@ namespace GerbilManager.Tests
{
// Run 1: litter "Wurf A" has sire "Vater" (conflict=true — not loaded) and dam "Mutter"
// (conflict=false — loaded). After run 1: litter.FatherId = null.
// Run 2: sire "Vater" is no longer in conflict. Backfill must set litter.FatherId.
// Run 2: sire "Vater" now conflict=false → loaded as NEW in run 2. Backfill via
// createdAnimalByName sets FatherId. (god steering point 3: run-2 path.)
var dir = Path.Combine(Path.GetTempPath(), "backfill-" + Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(dir);
using var conn = new SqliteConnection("DataSource=:memory:");
@@ -361,7 +362,7 @@ namespace GerbilManager.Tests
Assert.Null(litter1.FatherId); // Vater was quarantined -> null FK
Assert.NotNull(litter1.MotherId); // Mutter was loaded -> set
// Run 2: Vater is now conflict=false
// Run 2: Vater now conflict=false -> loaded as NEW animal in this run
var animals2 = """
[
{"id":"mutter","name":"Mutter [ZdkC]","dob":"01.01.2021","death":"","farbschlag":"","gender":"female","zuchtCanon":"kleinechaote",
@@ -387,6 +388,81 @@ namespace GerbilManager.Tests
}
}
[Fact]
public async Task ParentFkBackfill_uses_allDb_lookup_when_parent_not_in_current_loadable()
{
// god steering point 3: the main case — parent was loaded in a PREVIOUS run (not in
// the current run's animals.json at all). Backfill must find them via allDbNormToGid.
//
// Run 1: litter "Wurf C" + dam loaded, sire quarantined -> FatherId null.
// Run 2: sire loaded (new animal).
// Run 3: animals.json has ONLY the kind (sire absent from extract). Sire is in DB
// from run 2 but NOT in the current run's loadable/createdAnimalByName.
// Backfill must use allDbNormToGid to find him.
var dir = Path.Combine(Path.GetTempPath(), "backfill-db-" + Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(dir);
using var conn = new SqliteConnection("DataSource=:memory:");
conn.Open();
try
{
var littersJson = """
[{"id":"L-C","litterId":"C","date":"10.06.2023","damName":"Dame [ZdkC]","sireName":"Herr [ZdkC]","totalBorn":2,"zuchtnummer":"","note":""}]
""";
// Run 1: sire quarantined
File.WriteAllText(Path.Combine(dir, "litters.json"), littersJson);
File.WriteAllText(Path.Combine(dir, "animals.json"), """
[
{"id":"dame","name":"Dame [ZdkC]","dob":"05.05.2021","death":"","farbschlag":"","gender":"female","zuchtCanon":"kleinechaote",
"genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":false},
{"id":"herr","name":"Herr [ZdkC]","dob":"06.06.2021","death":"","farbschlag":"","gender":"male","zuchtCanon":"kleinechaote",
"genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":true}
]
""");
var opts = new DbContextOptionsBuilder<ApplicationContext>().UseSqlite(conn).Options;
using var db = new ApplicationContext(opts);
await db.Database.EnsureCreatedAsync();
await new ImportService(db, dir, dir).RunAsync(execute: true);
Assert.Null((await db.Litters.SingleAsync(l => l.Name == "Wurf C")).FatherId);
// Run 2: sire now loaded
File.WriteAllText(Path.Combine(dir, "animals.json"), """
[
{"id":"dame","name":"Dame [ZdkC]","dob":"05.05.2021","death":"","farbschlag":"","gender":"female","zuchtCanon":"kleinechaote",
"genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":false},
{"id":"herr","name":"Herr [ZdkC]","dob":"06.06.2021","death":"","farbschlag":"","gender":"male","zuchtCanon":"kleinechaote",
"genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":false}
]
""");
await new ImportService(db, dir, dir).RunAsync(execute: true);
var herrId = (await db.Gerbils.SingleAsync(g => g.ExternalRef == "herr")).Id;
// Run 2 itself may or may not backfill (depends on name normalization alignment).
// For the test we care about run 3.
// Run 3: sire NOT in animals.json at all (absent from new extract).
// litter still has FatherId=null if run 2 didn't backfill; if it did, we simulate
// by manually resetting FatherId to null so run 3 must fix it.
var litter3 = await db.Litters.SingleAsync(l => l.Name == "Wurf C");
litter3.FatherId = null;
await db.SaveChangesAsync();
File.WriteAllText(Path.Combine(dir, "animals.json"), """
[
{"id":"dame","name":"Dame [ZdkC]","dob":"05.05.2021","death":"","farbschlag":"","gender":"female","zuchtCanon":"kleinechaote",
"genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":false}
]
""");
// Run 3: sire absent from loadable (NOT in createdAnimalByName), but IS in DB.
var report3 = await new ImportService(db, dir, dir).RunAsync(execute: true);
Assert.Equal(1, report3.Litters.ParentFksBackfilled); // allDbNormToGid path
var litter3After = await db.Litters.SingleAsync(l => l.Name == "Wurf C");
Assert.Equal(herrId, litter3After.FatherId); // FK set from DB lookup
}
finally
{
try { Directory.Delete(dir, recursive: true); } catch { }
}
}
[Fact]
public async Task ParentFkBackfill_dry_run_counts_without_writing()
{

40
GerbilManagerWebAPI/Import/ImportService.cs
View File

@@ -401,9 +401,20 @@ namespace GerbilManagerWebAPI.Import
// PARENT-FK BACKFILL (idempotent re-run): already-imported Wurfchronik litters that
// have null Father/MotherId because the parent was previously quarantined may now be
// resolvable if that parent is loadable in this run. Counted for dry-run too.
// resolvable. Two lookup sources — must check BOTH:
// (a) createdAnimalByName: animals loaded/re-linked in THIS run (new or existing).
// (b) allDbNormToGid: ALL gerbils already in the DB, for parents loaded in an
// EARLIER run who are no longer in the current extract (e.g. alreadyImported
// animals absent from this run's animals.json, or name normalization mismatch
// between animals.json and the Wurfchronik sire/dam field).
// Counted for dry-run too; writes only when execute=true.
int parentFksBackfilled = 0;
{
// Build DB-wide normalized-name lookup (supplementary to createdAnimalByName).
var allDbNormToGid = existingRows
.GroupBy(g => Normalize(StripZucht(g.Name)))
.ToDictionary(grp => grp.Key, grp => grp.First().Id);
var existingWithNullParent = await _db.Litters
.Where(l => l.FatherId == null || l.MotherId == null)
.Select(l => new { l.Id, l.Name, l.FatherId, l.MotherId })
@@ -411,22 +422,23 @@ namespace GerbilManagerWebAPI.Import
var sourceByName = litters
.GroupBy(sl => $"Wurf {sl.LitterId}".Trim())
.ToDictionary(g => g.Key, g => g.First());
Guid? ResolveParentForBackfill(string rawName)
{
var n = Normalize(StripZucht(rawName));
if (n.Length == 0) return null;
if (createdAnimalByName.TryGetValue(n, out var fromLoadable) && persisted.Contains(fromLoadable))
return fromLoadable;
if (allDbNormToGid.TryGetValue(n, out var fromDb) && persisted.Contains(fromDb))
return fromDb;
return null;
}
foreach (var el in existingWithNullParent)
{
if (!sourceByName.TryGetValue(el.Name, out var sl)) continue;
Guid? newF = null, newM = null;
if (el.FatherId == null)
{
var n = Normalize(StripZucht(sl.SireName));
if (n.Length > 0 && createdAnimalByName.TryGetValue(n, out var fid) && persisted.Contains(fid))
newF = fid;
}
if (el.MotherId == null)
{
var n = Normalize(StripZucht(sl.DamName));
if (n.Length > 0 && createdAnimalByName.TryGetValue(n, out var mid) && persisted.Contains(mid))
newM = mid;
}
var newF = el.FatherId == null ? ResolveParentForBackfill(sl.SireName) : null;
var newM = el.MotherId == null ? ResolveParentForBackfill(sl.DamName) : null;
if (newF is null && newM is null) continue;
parentFksBackfilled++;
if (execute)

57
tools/import/extract.py
View File

@@ -627,9 +627,12 @@ def dedup(animals):
if a.get("deaf") is not None:
deaf_seen.add(a["deaf"])
tags_set.update(a.get("tags", []))
# pick the richest genotype (most mapped loci, then longest raw)
# pick the richest genotype: most mapped loci, then fewest unknowns ('?' alleles = specific
# wins, FIX-2), then longest raw string as final tiebreaker.
def _specificity(gd):
return sum(1 for pair in gd["mapped8locus"].values() for a in pair if a != "?")
best = max((a["genotype"] for a in grp),
key=lambda gd: (len(gd["mapped8locus"]), len(gd["rawGenotype"])))
key=lambda gd: (len(gd["mapped8locus"]), _specificity(gd), len(gd["rawGenotype"])))
out = {
"id": slug(base["name"], base["dob"]),
"name": base["name"],
@@ -892,22 +895,30 @@ def apply_dob_remaps(raw_animals, path):
"""PRE-dedup: a conflict-decision carrying `correctDob` marks a record as a DUPLICATE with a
wrong birthdate — remap that raw record's DOB to correctDob so dedup MERGES it into the
canonical same-named animal (e.g. Chelsea *15.10.2021 -> *02.04.2021). Match =
canon_pair(name)[0]+norm_dob(dob) (same identity as dedup — strips zucht suffix, folds
v.d.<->von den). Tolerates a missing/garbled file. Returns the remap count.
canon_pair(name)+(dob) with same Zucht-aware logic as apply_conflict_decisions (see there).
Tolerates a missing/garbled file. Returns the remap count.
Must run BEFORE dedup (it changes the dedup identity). (god/HUMANQUESTION D — Dubletten.)"""
remaps = {}
remaps_full = {} # (nameCanon, zuchtCanon, dob) -> correctDob — decision carries Zucht
remaps_name = {} # (nameCanon, dob) -> correctDob — no Zucht in decision
try:
with open(path, encoding="utf-8") as fh:
for r in (json.load(fh).get("resolutions") or []):
if r.get("correctDob"):
remaps[(canon_pair(r.get("name", ""))[0], norm_dob(r.get("dob", "")))] = r["correctDob"]
nc, zc = canon_pair(r.get("name", ""))
dob = norm_dob(r.get("dob", ""))
if zc:
remaps_full[(nc, zc, dob)] = r["correctDob"]
else:
remaps_name[(nc, dob)] = r["correctDob"]
except (OSError, ValueError):
return 0
if not remaps:
if not remaps_full and not remaps_name:
return 0
n = 0
for a in raw_animals:
new = remaps.get((canon_pair(a.get("name", ""))[0], norm_dob(a.get("dob", ""))))
nc, zc = canon_pair(a.get("name", ""))
dob = norm_dob(a.get("dob", ""))
new = remaps_full.get((nc, zc, dob)) or remaps_name.get((nc, dob))
if new and a.get("dob") != new:
a["dob"] = new
n += 1
@@ -917,26 +928,36 @@ def apply_dob_remaps(raw_animals, path):
def apply_conflict_decisions(merged, conflicts, path):
"""Consume human conflict resolutions (tools/import/conflict-decisions.json) so the wife's
answers UN-QUARANTINE animals. Schema: {"resolutions":[{name, dob, decision, genotype?,
farbschlag?, source}]}. Match = canon_pair(name)[0]+norm_dob(dob) — the same dedup identity
(call-name only, zucht stripped, v.d.<->von den folded). A matching animal: clear its
conflict, mark resolvedByDecision; an explicit `genotype` (breeder notation) is parsed and
becomes authoritative, `farbschlag` overrides too. Tolerates a missing/empty/garbled file.
farbschlag?, source}]}. Match = canon_pair(name)+(dob):
- When the decision name CARRIES a Zucht (zuchtCanon != ''), match on the FULL
(nameCanon, zuchtCanon, dob) triple — preserves the C3 rule that same name+DOB but
different Zucht = different animal.
- When the decision has NO Zucht, fall back to (nameCanon, dob) name-only match.
Both spellings v.d. / von den fold to the same canon. A matching animal: clear its conflict,
mark resolvedByDecision; an explicit `genotype` (breeder notation) is parsed and becomes
authoritative, `farbschlag` overrides too. Tolerates a missing/empty/garbled file.
Returns the number of conflicts resolved. (god/HUMANQUESTION D.)"""
decisions = {}
decisions_full = {} # (nameCanon, zuchtCanon, dob) -> r — when decision carries a Zucht
decisions_name = {} # (nameCanon, dob) -> r — fallback, decision has no Zucht
try:
with open(path, encoding="utf-8") as fh:
for r in (json.load(fh).get("resolutions") or []):
# FIX-1: use dedup identity (call-name only, zucht stripped) so that e.g.
# a decision written as "von den" matches a merged record with "v.d." spelling.
decisions[(canon_pair(r.get("name", ""))[0], norm_dob(r.get("dob", "")))] = r
nc, zc = canon_pair(r.get("name", ""))
dob = norm_dob(r.get("dob", ""))
if zc:
decisions_full[(nc, zc, dob)] = r
else:
decisions_name[(nc, dob)] = r
except (OSError, ValueError):
return 0
if not decisions:
if not decisions_full and not decisions_name:
return 0
resolved = 0
for a in merged:
d = decisions.get((canon_pair(a["name"])[0], norm_dob(a["dob"])))
nc, zc = canon_pair(a["name"])
dob = norm_dob(a["dob"])
d = decisions_full.get((nc, zc, dob)) or decisions_name.get((nc, dob))
if not d:
continue
a["resolvedByDecision"] = True

67
tools/import/test_extract.py
View File

@@ -135,6 +135,30 @@ check("FIX-1: v.d. decision also matches 'von den' record (both spellings match)
try: os.remove(dec_vd)
except OSError: pass
# FIX-1 C3-rule: same name+DOB, two Zuchten -> decision hits ONLY the correct Zucht (C3 isolation)
dec_c3 = os.path.join(tempfile.gettempdir(), "decisions-c3.json")
_json.dump({"resolutions": [
# Decision only for Luna from ZdkC, NOT Luna from Black Forest
{"name": "Luna von den Kleinen Chaoten", "dob": "01.01.2020",
"decision": "D-locus = DD", "genotype": "aa CC DD ee gg PP spsp rere", "source": "test"},
]}, open(dec_c3, "w", encoding="utf-8"))
merged_c3 = [
{"id": "luna-kc", "name": "Luna von den Kleinen Chaoten", "dob": "01.01.2020",
"conflict": True, "farbschlag": "", "death": "",
"genotype": {"mapped8locus": {"D": ["D","?"]}, "rawGenotype": "D-", "unmappedTokens": []}},
{"id": "luna-bf", "name": "Luna of Black Forest", "dob": "01.01.2020",
"conflict": True, "farbschlag": "", "death": "",
"genotype": {"mapped8locus": {"D": ["D","?"]}, "rawGenotype": "D-", "unmappedTokens": []}},
]
conflicts_c3 = [{"id": "luna-kc"}, {"id": "luna-bf"}]
n_c3 = e.apply_conflict_decisions(merged_c3, conflicts_c3, dec_c3)
check("FIX-1 C3: decision hits only the correct Zucht (luna-kc resolved)", n_c3 == 1)
check("FIX-1 C3: luna-kc conflict cleared (correct Zucht)", merged_c3[0]["conflict"] is False)
check("FIX-1 C3: luna-bf conflict NOT cleared (different Zucht)", merged_c3[1]["conflict"] is True)
check("FIX-1 C3: conflicts list has only luna-bf left", len(conflicts_c3) == 1 and conflicts_c3[0]["id"] == "luna-bf")
try: os.remove(dec_c3)
except OSError: pass
# --- correctDob: a wrong-birthdate duplicate is remapped BEFORE dedup so it merges ---
dec2 = os.path.join(tempfile.gettempdir(), "decisions-dob.json")
_json.dump({"resolutions": [
@@ -192,6 +216,49 @@ check("c[h] vs c[chm] (different modifiers, both specified) -> conflict",
check("identical genotypes -> no conflict",
not e._genotype_conflict([{"A": ["A", "a"]}, {"A": ["A", "a"]}]))
# FIX-2 MERGE: specific allele must survive the merge regardless of which variant comes first.
# dedup() picks the most specific genotype (fewest '?' alleles); C- vs CC -> CC must win.
def _minimal_animal(name, dob, mapped):
"""Build a minimal raw animal dict suitable for dedup()."""
from genotype import parse as gparse
raw = " ".join(f"{l}{''.join(a)}" for l, pa in mapped.items() for a in [pa])
return {
"name": name, "dob": dob, "death": "", "gender": None,
"farbschlag": "", "breeder": "", "zucht": "", "parentRefs": [],
"photos": [], "sourceFiles": ["test.xlsx"], "tags": [],
"deaf": None, "conflict": False,
"genotype": {"mapped8locus": mapped, "rawGenotype": raw, "unmappedTokens": []},
"_gen": 0, "_col": 5, "_row": 10, "_file": "test.xlsx",
"_zucht": "",
}
# Order A: C- first, CC second
animals_merge_a = [
_minimal_animal("TestTier", "01.01.2020", {"C": ["C", "?"]}), # C-
_minimal_animal("TestTier", "01.01.2020", {"C": ["C", "C"]}), # CC
]
merged_ma, _, _, _ = e.dedup(animals_merge_a)
check("FIX-2 merge A (C- first): result has CC not C-",
merged_ma[0]["genotype"]["mapped8locus"].get("C") == ["C", "C"])
# Order B: CC first, C- second (must give same result)
animals_merge_b = [
_minimal_animal("TestTier2", "02.02.2020", {"C": ["C", "C"]}), # CC
_minimal_animal("TestTier2", "02.02.2020", {"C": ["C", "?"]}), # C-
]
merged_mb, _, _, _ = e.dedup(animals_merge_b)
check("FIX-2 merge B (CC first): result has CC not C-",
merged_mb[0]["genotype"]["mapped8locus"].get("C") == ["C", "C"])
# G- vs Gg: Gg must win
animals_merge_g = [
_minimal_animal("TestGGerbil", "03.03.2020", {"G": ["G", "?"]}), # G-
_minimal_animal("TestGGerbil", "03.03.2020", {"G": ["G", "g"]}), # Gg
]
merged_mg, _, _, _ = e.dedup(animals_merge_g)
check("FIX-2 merge G (G- vs Gg): Gg wins",
merged_mg[0]["genotype"]["mapped8locus"].get("G") == ["G", "g"])
# --- FIX-4: Skarlett parse artifact — trailing "/ +YEAR" stripped from geno, death captured ---
dob4, death4, geno4 = e.parse_detail("Skarlett,*17.04.2016, aa C- DD ee Gg PP spsp rere / +2018")
check("FIX-4: '/ +YEAR' artifact stripped from geno tail",