From ea79703dbf8819e61d22b9cdb538d6312c2d9e18 Mon Sep 17 00:00:00 2001 From: Gulum Date: Sat, 6 Jun 2026 10:48:30 +0200 Subject: [PATCH] =?UTF-8?q?PEDIGREE-LINK:=20chart=20parentRefs=E2=86=92lit?= =?UTF-8?q?ters,=20box-colour=3Dsex,=20name-bleed=20fix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Structural fix (god, Julian-reported via 'C'): the loader ignored SourceAnimal.ParentRefs, so animals whose ancestry exists only as Stammbaum chart-position refs loaded with LitterId=null ("unbekannt"). ImportService now synthesizes/reuses a derived litter from parentRefs: resolves father+mother via name+DOB, groups siblings (same parents+dob) into one litter, sets Father/Mother + offspring LitterId, dates it to the offspring DOB, and tags Notes "aus Stammbaum-Diagramm abgeleitet (Konfidenz: …)" so it's transparent/reversible. Existing animals that become linkable are re-linked on re-run (sweep-idempotent). Dry-run counts included. Projected: ~124 loadable animals gain a parent link. Box-colour = sex (Julian): blue box = male, white box = female. All 11 pedigrees encode this as a solid theme-8 (accent5/blue) fill vs no fill. xlsx_util.cell_fill_sex reads it; extract.py sets animal.gender from the box; ImportService.InferGender prefers it over sire/dam name inference. Result: 306/306 loadable animals now sexed (154♂/152♀). Extractor noise fix (god): reject Farbschlag values that are actually a parent NAME bled across cells (contain v.d./von/of/gen.) — cleared phantom conflicts (e.g. Chayton). Combined with GEN-3 Uw→G: Konflikte 32→21. Also skip Excel "~$" lock files in the glob. GEN-3a contract (Kevin): ComposeGenotype appends "Slsl" for WP/Sls carriers (wild-type sl/sl omitted) so 8-locus strings stay unchanged. Importer-only. The live re-import into Julian's DB stays a separate supervised gated step. 95 C# tests + python genotype tests green; has-pending-model-changes clean (no schema change on this branch). Co-Authored-By: Claude Opus 4.8 (1M context) --- GerbilManager.Tests/ImportServiceTests.cs | 63 +++++- GerbilManagerWebAPI/Import/ImportModels.cs | 9 +- GerbilManagerWebAPI/Import/ImportService.cs | 205 ++++++++++++++++---- tools/import/extract.py | 29 ++- tools/import/output/review-report.md | 16 +- tools/import/xlsx_util.py | 40 ++++ 6 files changed, 304 insertions(+), 58 deletions(-) diff --git a/GerbilManager.Tests/ImportServiceTests.cs b/GerbilManager.Tests/ImportServiceTests.cs index f3da78e..971fe3a 100644 --- a/GerbilManager.Tests/ImportServiceTests.cs +++ b/GerbilManager.Tests/ImportServiceTests.cs @@ -115,8 +115,67 @@ namespace GerbilManager.Tests // (kept out of the 8-locus compact Genotype contract until GEN-3a adopts them). Assert.Contains("Sls", a1.RawImportData!); Assert.Contains("WFNZ", a1.RawImportData!); - // and Sls must NOT leak into the compact 8-locus genotype string - Assert.DoesNotContain("Sl", a1.Genotype!); + // GEN-3a contract: a WP/Sls carrier appends the trailing "Slsl" token (Kevin). + Assert.EndsWith("Slsl", a1.Genotype!); + } + + [Fact] + public void ComposeGenotype_appends_Slsl_only_for_carriers() + { + // wild-type sl/sl is omitted -> plain 8-locus string + var wild = new SourceGenotype { Mapped8locus = new() { ["A"] = new() { "a", "a" }, ["Sls"] = new() { "sl", "sl" } } }; + Assert.DoesNotContain("Sl", ImportService.ComposeGenotype(wild)); + // WP heterozygote -> trailing Slsl + var wp = new SourceGenotype { Mapped8locus = new() { ["A"] = new() { "a", "a" }, ["Sls"] = new() { "Sl", "sl" } } }; + Assert.EndsWith("Slsl", ImportService.ComposeGenotype(wp)); + } + + [Fact] + public async Task Synthesizes_litter_from_chart_parentRefs_links_offspring_and_parents() + { + // Offspring 'C' has chart-position parentRefs to father 'Papa' (loaded) and mother + // 'Mama' (loaded), but NO Wurfchronik litterRef -> the loader must synthesize a derived + // litter, link C to it, and set the litter's Father/Mother (PEDIGREE-LINK structural fix). + var dir = Path.Combine(Path.GetTempPath(), "pedlink-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(dir); + try + { + File.WriteAllText(Path.Combine(dir, "litters.json"), "[]"); + File.WriteAllText(Path.Combine(dir, "animals.json"), """ + [ + {"id":"papa","name":"Papa v.d. Test","dob":"01.01.2022","death":"","farbschlag":"","gender":"male", + "genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":false}, + {"id":"mama","name":"Mama v.d. Test","dob":"02.02.2022","death":"","farbschlag":"","gender":"female", + "genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":false}, + {"id":"c","name":"C","dob":"29.04.2024","death":"","farbschlag":"", + "genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":false, + "parentRefs":[ + {"name":"Papa v.d. Test","dob":"01.01.2022","roleGuess":"father","method":"chart-position","confidence":"medium"}, + {"name":"Mama v.d. Test","dob":"02.02.2022","roleGuess":"mother","method":"chart-position","confidence":"medium"} + ]} + ] + """); + using var db = NewDb(); + var report = await new ImportService(db, dir, dir).RunAsync(execute: true); + + Assert.Equal(1, report.Animals.ParentLinksFromChart); + Assert.Equal(1, report.Litters.DerivedFromChart); + + var papa = await db.Gerbils.SingleAsync(g => g.ExternalRef == "papa"); + var mama = await db.Gerbils.SingleAsync(g => g.ExternalRef == "mama"); + var c = await db.Gerbils.SingleAsync(g => g.ExternalRef == "c"); + Assert.NotNull(c.LitterId); // C no longer "unbekannt" + + // box-colour sex flows through (blue=male, white=female) + Assert.Equal(Gender.male, papa.Gender); + Assert.Equal(Gender.female, mama.Gender); + + var litter = await db.Litters.SingleAsync(l => l.Id == c.LitterId); + Assert.Equal(papa.Id, litter.FatherId); + Assert.Equal(mama.Id, litter.MotherId); + Assert.Contains("Diagramm", litter.Notes!); // transparent + reversible + } + finally { try { Directory.Delete(dir, recursive: true); } catch { } } } [Fact] diff --git a/GerbilManagerWebAPI/Import/ImportModels.cs b/GerbilManagerWebAPI/Import/ImportModels.cs index 1aad635..8e4ed3d 100644 --- a/GerbilManagerWebAPI/Import/ImportModels.cs +++ b/GerbilManagerWebAPI/Import/ImportModels.cs @@ -39,7 +39,9 @@ namespace GerbilManagerWebAPI.Import { public string Name { get; set; } = ""; public string Dob { get; set; } = ""; - public string RoleGuess { get; set; } = ""; + public string RoleGuess { get; set; } = ""; // "father" | "mother" + public string Method { get; set; } = ""; // e.g. "chart-position" + public string Confidence { get; set; } = ""; // "hoch" | "medium" | "niedrig" } public sealed class SourceLitterRef @@ -73,7 +75,7 @@ namespace GerbilManagerWebAPI.Import IReadOnlyList Samples, IReadOnlyList Notes); - public sealed record LitterSummary(int InSource, int Created, int AlreadyImported); + public sealed record LitterSummary(int InSource, int Created, int AlreadyImported, int DerivedFromChart = 0); public sealed record AnimalSummary( int InSource, @@ -82,7 +84,8 @@ namespace GerbilManagerWebAPI.Import int FarbschlagMatched, int FarbschlagUnmatched, int AlreadyImported, - QuarantineSummary Quarantined); + QuarantineSummary Quarantined, + int ParentLinksFromChart = 0); public sealed record QuarantineSummary( int Conflicts, diff --git a/GerbilManagerWebAPI/Import/ImportService.cs b/GerbilManagerWebAPI/Import/ImportService.cs index 4142c90..0dc4fe4 100644 --- a/GerbilManagerWebAPI/Import/ImportService.cs +++ b/GerbilManagerWebAPI/Import/ImportService.cs @@ -128,74 +128,165 @@ namespace GerbilManagerWebAPI.Import int photosAttached = 0, photosMissing = 0; var createdAnimalByName = new Dictionary(); // normalized name -> gerbil id (for litter back-link) + // name+DOB -> gid index, across EXISTING rows AND this run's planned animals, so that + // chart-position parentRefs (PEDIGREE-LINK) can resolve a parent to a real gerbil id. + var existingRows = await _db.Gerbils + .Select(g => new { g.Id, g.Name, g.DateOfBirth, g.ExternalRef, g.LitterId }).ToListAsync(); + var gidByNameDob = new Dictionary(); + foreach (var g in existingRows) + gidByNameDob[NameDobKey(g.Name, g.DateOfBirth)] = g.Id; + var existingLitterByExtRef = existingRows.Where(g => g.ExternalRef != null) + .ToDictionary(g => g.ExternalRef!, g => g.LitterId); + + // PASS 1: assign ids + resolve fb/gender/Wurfchronik link (no writes yet). + var plan = new List(); foreach (var a in loadable) { - if (existingGerbilSet.Contains(a.Id)) { animalsExisting++; continue; } - animalsCreated++; + bool exists = existingGerbilSet.Contains(a.Id); + var gid = exists ? gidByNameDob[NameDobKey(a.Name, ParseDate(a.Dob))] : Guid.NewGuid(); - Guid? litterId = null; + Guid? wurfLitterId = null; if (a.LitterRef?.Confidence == "hoch" && a.LitterRef.Candidates is not { Count: > 0 } && litterIdMap.TryGetValue(a.LitterRef.LitterId, out var lid)) - { - litterId = lid; - linked++; - } + wurfLitterId = lid; Guid? colorVarietyId = null; var fbCandidates = new[] { a.Farbschlag }.Concat(a.FarbschlagVariants) .Where(s => !string.IsNullOrWhiteSpace(s)); foreach (var fb in fbCandidates) - { if (varietyByName.TryGetValue(fb.Trim().ToLowerInvariant(), out var vid)) { colorVarietyId = vid; break; } - } - if (colorVarietyId is null) fbUnmatched++; else fbMatched++; var gender = InferGender(a, sireNames, damNames); - var gid = Guid.NewGuid(); var norm = Normalize(StripZucht(a.Name)); if (norm.Length > 0) createdAnimalByName.TryAdd(norm, gid); + if (!exists) gidByNameDob.TryAdd(NameDobKey(a.Name, ParseDate(a.Dob)), gid); + + var currentLitter = exists && existingLitterByExtRef.TryGetValue(a.Id, out var el) ? el : null; + plan.Add(new AnimalPlan(a, gid, exists, wurfLitterId, currentLitter, colorVarietyId, gender)); + } + + // PASS 1.5: PEDIGREE-LINK — synthesize/reuse a litter from chart-position parentRefs for + // any animal that has no Wurfchronik litter and isn't already litter-linked. Siblings + // (same father+mother+dob) share one derived litter. Computed for dry-run counts too. + int parentLinksAdded = 0, derivedLitters = 0; + var synthLitterForGid = new Dictionary(); // offspring gid -> synth litter id + var synthLitters = new Dictionary(); // parents+date key -> synth litter + foreach (var p in plan) + { + if (p.WurfLitterId is not null || p.CurrentLitterId is not null) continue; + if (p.A.ParentRefs is not { Count: > 0 }) continue; + var father = ResolveParentGid(p.A, "father", gidByNameDob); + var mother = ResolveParentGid(p.A, "mother", gidByNameDob); + if (father is null && mother is null) continue; // nothing resolvable to link + var dob = ParseDate(p.A.Dob); + var conf = p.A.ParentRefs.FirstOrDefault()?.Confidence ?? "medium"; + var key = $"{father}|{mother}|{dob:yyyy-MM-dd}"; + if (!synthLitters.TryGetValue(key, out var sl)) + { + sl = new SynthLitter(Guid.NewGuid(), father, mother, dob, conf); + synthLitters[key] = sl; + derivedLitters++; + } + synthLitterForGid[p.Gid] = sl.Id; + parentLinksAdded++; + } + + // PASS 2: write (litters synthesized first so offspring FK resolves), then animals + photos. + if (execute) + { + // reuse an existing litter with the same parents+date instead of duplicating. + var existingLitterRows = await _db.Litters + .Select(l => new { l.Id, l.FatherId, l.MotherId, l.Date }).ToListAsync(); + var litterByParentsDate = new Dictionary(); + foreach (var l in existingLitterRows) + litterByParentsDate[$"{l.FatherId}|{l.MotherId}|{l.Date:yyyy-MM-dd}"] = l.Id; + + foreach (var sl in synthLitters.Values.ToList()) + { + var reuseKey = $"{sl.Father}|{sl.Mother}|{sl.Date:yyyy-MM-dd}"; + if (litterByParentsDate.TryGetValue(reuseKey, out var existingId)) + { + // remap offspring to the existing litter; don't create a duplicate. + foreach (var g in synthLitterForGid.Where(kv => kv.Value == sl.Id).Select(kv => kv.Key).ToList()) + synthLitterForGid[g] = existingId; + derivedLitters--; + continue; + } + _db.Litters.Add(new Litter + { + Id = sl.Id, + Name = $"Wurf (aus Diagramm) {sl.Date:yyyy-MM-dd}".Trim(), + Date = sl.Date ?? default, + FatherId = sl.Father, + MotherId = sl.Mother, + Notes = $"aus Stammbaum-Diagramm abgeleitet (Konfidenz: {sl.Confidence})", + }); + } + await _db.SaveChangesAsync(); + } + + foreach (var p in plan) + { + Guid? litterId = p.WurfLitterId + ?? (synthLitterForGid.TryGetValue(p.Gid, out var slid) ? slid : (Guid?)null); + if (litterId is not null) linked++; + + if (p.ColorVarietyId is null) fbUnmatched++; else fbMatched++; if (samples.Count < 16) - samples.Add($"Tier: {a.Name} (*{a.Dob}), Genotyp {ComposeGenotype(a.Genotype)}" - + (litterId is not null ? ", Wurf-verknüpft" : "") - + (colorVarietyId is not null ? $", Farbschlag „{a.Farbschlag}\"" : "")); + samples.Add($"Tier: {p.A.Name} (*{p.A.Dob}), Genotyp {ComposeGenotype(p.A.Genotype)}" + + (litterId is not null ? (p.WurfLitterId is not null ? ", Wurf-verknüpft" : ", Eltern aus Diagramm") : "") + + (p.ColorVarietyId is not null ? $", Farbschlag „{p.A.Farbschlag}\"" : "")); + + if (p.Exists) + { + animalsExisting++; + // re-link an existing animal that just became linkable (sweep idempotency). + if (execute && p.CurrentLitterId is null && litterId is not null) + { + var row = await _db.Gerbils.FirstOrDefaultAsync(g => g.Id == p.Gid); + if (row is not null) row.LitterId = litterId; + } + continue; + } + animalsCreated++; if (execute) { _db.Gerbils.Add(new Gerbil { - Id = gid, - Name = a.Name, - Gender = gender, + Id = p.Gid, + Name = p.A.Name, + Gender = p.Gender, Status = GerbilStatus.Active, - DateOfBirth = ParseDate(a.Dob), - DateOfDeath = ParseDate(a.Death), + DateOfBirth = ParseDate(p.A.Dob), + DateOfDeath = ParseDate(p.A.Death), LitterId = litterId, - ColorVarietyId = colorVarietyId, - Genotype = ComposeGenotype(a.Genotype), - IsDeaf = a.Deaf, + ColorVarietyId = p.ColorVarietyId, + Genotype = ComposeGenotype(p.A.Genotype), + IsDeaf = p.A.Deaf, ImportSource = ImportSourceTag, - ExternalRef = a.Id, - OriginBreeder = string.IsNullOrWhiteSpace(a.Zucht) ? null : a.Zucht.Trim(), + ExternalRef = p.A.Id, + OriginBreeder = string.IsNullOrWhiteSpace(p.A.Zucht) ? null : p.A.Zucht.Trim(), RawImportData = JsonSerializer.Serialize(new { - a.Genotype.RawGenotype, - a.Genotype.UnmappedTokens, + p.A.Genotype.RawGenotype, + p.A.Genotype.UnmappedTokens, // GEN-3b: Sls (2nd spotting locus) preserved here until Kevin's GEN-3a // parser adopts it into the compact Genotype contract; tags + deaf too. - Sls = a.Genotype.Mapped8locus.TryGetValue("Sls", out var sls) ? sls : null, - a.Tags, - a.Deaf, - a.Zucht, - a.SourceFiles, - FarbschlagRaw = a.Farbschlag, + Sls = p.A.Genotype.Mapped8locus.TryGetValue("Sls", out var sls) ? sls : null, + p.A.Tags, + p.A.Deaf, + p.A.Zucht, + p.A.SourceFiles, + FarbschlagRaw = p.A.Farbschlag, }), }); } // photos - foreach (var rel in a.Photos) + foreach (var rel in p.A.Photos) { var src = Path.Combine(_sourceDir, rel.Replace('/', Path.DirectorySeparatorChar)); if (!File.Exists(src)) { photosMissing++; continue; } @@ -208,7 +299,7 @@ namespace GerbilManagerWebAPI.Import _db.GerbilPhotos.Add(new GerbilPhoto { Id = Guid.NewGuid(), - GerbilId = gid, + GerbilId = p.Gid, FileName = fileName, SortOrder = 0, CreatedAt = DateTimeOffset.UtcNow, @@ -218,7 +309,7 @@ namespace GerbilManagerWebAPI.Import } if (execute) await _db.SaveChangesAsync(); - // ---- back-link litter parents by name (best effort) ---- + // ---- back-link Wurfchronik litter parents by name (best effort) ---- if (execute) { foreach (var sl in litters) @@ -235,14 +326,17 @@ namespace GerbilManagerWebAPI.Import } notes.Add("Quarantäne (kein Import): Konflikte + Stubs ohne Geburtsdatum + unsichere Wurf-Zuordnungen — warten auf die Prüfung durch die Züchterin."); + if (parentLinksAdded > 0) + notes.Add($"Stammbaum-Diagramm: {parentLinksAdded} Tiere über Eltern-Verknüpfung einem (abgeleiteten) Wurf zugeordnet ({derivedLitters} abgeleitete Würfe)."); if (!execute) notes.Add("DRY-RUN: nichts gespeichert. /import/execute lädt die konfliktfreien Daten."); return new ImportReport( Executed: execute, - Litters: new LitterSummary(litters.Count, littersCreated, littersExisting), + Litters: new LitterSummary(litters.Count, littersCreated, littersExisting, derivedLitters), Animals: new AnimalSummary( animals.Count, animalsCreated, linked, fbMatched, fbUnmatched, animalsExisting, - new QuarantineSummary(conflicts, stubs, dateOnly, ambiguous, conflicts + stubs)), + new QuarantineSummary(conflicts, stubs, dateOnly, ambiguous, conflicts + stubs), + parentLinksAdded), Photos: new PhotoSummary(photosAttached, photosMissing), Samples: samples, Notes: notes); @@ -263,7 +357,16 @@ namespace GerbilManagerWebAPI.Import if (g.Mapped8locus.TryGetValue(locus, out var pair) && pair.Count == 2) return StripCaret(pair[0]) + StripCaret(pair[1]); return "??"; - }); + }).ToList(); + + // GEN-3a contract (Kevin): Sls is appended LAST and ONLY for carriers — the + // wild-type sl/sl is omitted so existing 8-locus strings stay unchanged. WP het + // renders as the trailing token "Slsl"; S(l)S(l) is lethal so never appears. + if (g.Mapped8locus.TryGetValue("Sls", out var sls) && sls.Count == 2 + && !(sls[0] == "sl" && sls[1] == "sl")) + { + tokens.Add(StripCaret(sls[0]) + StripCaret(sls[1])); + } return string.Join(' ', tokens); } @@ -271,6 +374,11 @@ namespace GerbilManagerWebAPI.Import private static Gender InferGender(SourceAnimal a, HashSet sires, HashSet dams) { + // Box colour (blue=male, white=female) is the authoritative breeder signal — prefer it + // over sire/dam name inference (PEDIGREE-LINK, Julian 2026-06-06). + if (string.Equals(a.Gender, "male", StringComparison.OrdinalIgnoreCase)) return Gender.male; + if (string.Equals(a.Gender, "female", StringComparison.OrdinalIgnoreCase)) return Gender.female; + var n = Normalize(StripZucht(a.Name)); bool isSire = sires.Contains(n), isDam = dams.Contains(n); if (isSire && !isDam) return Gender.male; @@ -304,5 +412,26 @@ namespace GerbilManagerWebAPI.Import n = Regex.Replace(n, @"[^a-z0-9äöüß]", ""); return n; } + + /// Dedup identity for parent resolution: normalized call-name + DOB. + private static string NameDobKey(string name, DateOnly? dob) => + $"{Normalize(StripZucht(name))}|{dob:yyyy-MM-dd}"; + + /// Resolve a chart-position parentRef (by role) to a known gerbil id, or null. + private static Guid? ResolveParentGid(SourceAnimal a, string role, Dictionary gidByNameDob) + { + var pr = a.ParentRefs.FirstOrDefault(p => + string.Equals(p.RoleGuess, role, StringComparison.OrdinalIgnoreCase)); + if (pr is null || string.IsNullOrWhiteSpace(pr.Name)) return null; + return gidByNameDob.TryGetValue(NameDobKey(pr.Name, ParseDate(pr.Dob)), out var id) ? id : null; + } + + /// Per-animal plan computed before any write so synthesis can run in dry-run too. + private sealed record AnimalPlan( + SourceAnimal A, Guid Gid, bool Exists, Guid? WurfLitterId, + Guid? CurrentLitterId, Guid? ColorVarietyId, Gender Gender); + + /// A litter synthesized from chart-position parentRefs (PEDIGREE-LINK). + private sealed record SynthLitter(Guid Id, Guid? Father, Guid? Mother, DateOnly? Date, string Confidence); } } diff --git a/tools/import/extract.py b/tools/import/extract.py index 460519b..24e9cb9 100644 --- a/tools/import/extract.py +++ b/tools/import/extract.py @@ -20,6 +20,7 @@ import glob import shutil import argparse import unicodedata +from collections import Counter import xlsx_util as xu import genotype as gt @@ -96,6 +97,18 @@ ZUCHT_ALIASES = { "zdkc": "kleinechaote", # "Zucht der kleinen Chaoten" (home cattery shorthand) } +# A Farbschlag value must NOT contain cattery/line connectors (v.d./von/of/gen.) — when it +# does, a parent's NAME has bled into the Farbschlag cell (cross-cell chart read, PEDIGREE-LINK +# bug: e.g. "Victoria Welby gen. Welby v.d. Kleinen Chaoten" became a Farbschlag variant and +# spawned a phantom conflict). Reject such values so they don't pollute farbschlag/conflicts. +_NAME_MARKER = re.compile(r"\bv\.\s?d\.|\bvon\b|\bof\b|\bgen\.", re.IGNORECASE) + + +def looks_like_animal_name(text): + """True if a candidate Farbschlag cell actually looks like an animal name (has a + cattery/line connector). Real Farbschläge are short colour words without these.""" + return bool(_NAME_MARKER.search(text or "")) + def split_name_zucht(raw): """'Luna [ZdkC]' -> ('Luna','ZdkC'); 'Pikachu of Black Forest' -> @@ -159,6 +172,7 @@ def extract_stammbaum(path): ss = xu.shared_strings(z) sheets = xu.sheet_paths(z) cells = xu.read_cells(z, sheets[0], ss) + fillsex = xu.cell_fill_sex(z, sheets[0]) # box colour -> sex (blue=male, white=female) # group cells by column for block reconstruction by_col = {} @@ -207,7 +221,8 @@ def extract_stammbaum(path): elif re.search(r"\b(Zucht|Privatzucht)\b", cell) or cell.startswith("("): breeder = cell used.add((c, rr)) - elif not farbschlag and not re.match(r"^\*?\s?\d", cell): + elif not farbschlag and not re.match(r"^\*?\s?\d", cell) \ + and not looks_like_animal_name(cell): farbschlag = cell used.add((c, rr)) used.add((c, r)) @@ -224,7 +239,7 @@ def extract_stammbaum(path): "nameVariants": [], "dob": dob, "death": death, - "gender": None, + "gender": fillsex.get((c, r)), # box colour: blue=male, white=female "farbschlag": farbschlag, "genotype": genodict, "deaf": genodict.get("deaf"), @@ -536,11 +551,14 @@ def dedup(animals): deaths = set() deaf_seen = set() tags_set = set() + genders = [] for a in grp: variants.add(a["name"]) files.update(a["sourceFiles"]) photos.extend(a["photos"]) parent_refs.extend(a["parentRefs"]) + if a.get("gender"): + genders.append(a["gender"]) if a["genotype"]["mapped8locus"]: genos.add(a["genotype"]["rawGenotype"]) geno_keys.add(_geno_key(a["genotype"])) @@ -560,7 +578,8 @@ def dedup(animals): "nameVariants": sorted(v for v in variants if v), "dob": norm_dob(base["dob"]), "death": sorted(deaths)[0] if deaths else "", - "gender": None, + # box-colour sex (blue=male, white=female): majority across mentions, else None. + "gender": Counter(genders).most_common(1)[0][0] if genders else None, "farbschlag": sorted(farb)[0] if farb else "", "farbschlagVariants": sorted(farb), "genotype": best, @@ -829,7 +848,9 @@ def main(): os.makedirs(OUT, exist_ok=True) raw_animals = [] - files = sorted(glob.glob(os.path.join(args.stammbaeume, "*.xlsx"))) + # Skip Excel lock/owner files ("~$...") that appear while a workbook is open. + files = sorted(f for f in glob.glob(os.path.join(args.stammbaeume, "*.xlsx")) + if not os.path.basename(f).startswith("~$")) print(f"Stammbaum-Dateien: {len(files)}") for path in files: got = extract_stammbaum(path) diff --git a/tools/import/output/review-report.md b/tools/import/output/review-report.md index 18ab88f..0f109ca 100644 --- a/tools/import/output/review-report.md +++ b/tools/import/output/review-report.md @@ -8,7 +8,7 @@ _Automatisch erzeugt von `tools/import/extract.py` — **noch nichts in die Date - Nach Zusammenführung (eindeutige Tiere): **622** - davon mit Geburtsdatum: 327 - in mehreren Dateien gefunden (Dubletten zusammengeführt): 158 -- Konflikte zur Klärung: **27** +- Konflikte zur Klärung: **21** - Mehrdeutige / unvollständige Einträge (ohne Name+Datum): **310** - Fotos zugeordnet: **137** - Würfe aus der Wurfchronik: **752** @@ -26,30 +26,24 @@ Gleiches Tier (Name+Datum), aber widersprüchliche Angaben in verschiedenen Date | Tier | Geburtsdatum | abweichende Genotypen | abweichende Farbschläge | Sterbedaten | Dateien | |---|---|---|---|---|---| | Ella | 10.06.2019 | Aa C D- ee[f] GG P- spsp // Aa Cc[chm] D- ee[f] UwUw P- spsp | Algierfuchsschimmel, hell | 03.02.2023 | Stammbaum von Akio Kids, Stammbaum von CP-Fuchs, CP-Sa Sp von Unity, Stammbaum von Valentino Firehearts Kids | -| ZoneFire | 07.12.2020 | Aa c[chm]c[chm] D- Ee Gg P- Spsp | CP-Agouti Kragenschecke // Kalea von den Kleinen Chaoten | — | Stammbaum von Akio Kids | -| Louis von den Kleinen Chaoten | 15.07.2017 | Aa Cc[] D- Ee Gg P- spsp // Aa Cc[chm] D- Ee Uwuw[d] P- spsp | Roswitha von den Kleinen Chaoten | 01.07.2020 | Stammbaum von Akio Kids, Stammbaum von CP-Fuchs, CP-Sa Sp von Unity | +| Louis von den Kleinen Chaoten | 15.07.2017 | Aa Cc[] D- Ee Gg P- spsp // Aa Cc[chm] D- Ee Uwuw[d] P- spsp | — | 01.07.2020 | Stammbaum von Akio Kids, Stammbaum von CP-Fuchs, CP-Sa Sp von Unity | | Firefly von den Kleinen Chaoten | 18.12.2019 | /+, Aa c[chm]c[chm] D- Ee Gg PP Spsp // Aa c[chm]c[chm] DD Ee Gg PP Spsp | — | 2024 | Stammbaum von Akio Kids, Stammbaum von CP-Fuchs, CP-Sa Sp von Unity, Stammbaum von Fire Kids, Stammbaum von Valentino Firehearts Kids | | Zuleika von den Kleinen Chaoten | 24.10.2015 | aa c[chm]c[h] D- E G P- spsp // aa c[chm]c[h] D- Ee Gg P- spsp // aa c[chm]c[h] DD Ee Gg P- spsp | — | 24.02.2019 | Stammbaum von Akio Kids, Stammbaum von CP-Fuchs, CP-Sa Sp von Unity, Stammbaum von Valentino Firehearts Kids | | WildFire von den Kleinen Chaoten | 05.10.2017 | aa c[chm]c[chm] D- Ee gg P- spsp // aa c[chm]c[chm] D- Ee gg PP spsp | — | — | Stammbaum von Akio Kids, Stammbaum von CP-Fuchs, CP-Sa Sp von Unity, Stammbaum von Fire Kids, Stammbaum von Valentino Firehearts Kids | | Vestra von den Schlossmäusen | 08.02.2019 | Aa Cc[chm] D- EE GG PP Spsp [WP] // Aa Cc[chm] DD EE GG PP Spsp [WP] | — | 26.05.2023 | Stammbaum von Akio Kids, Stammbaum von CP-Fuchs, CP-Sa Sp von Unity, Stammbaum von Fire Kids, Stammbaum von Valentino Firehearts Kids | | Flint von den Kleinen Chaoten | 23.12.2017 | aa Cc[chm] D- ee Gg P- spsp | — | 10.05.2021 // 10.05.2022 | Stammbaum von Akio Kids, Stammbaum von CP-Fuchs, CP-Sa Sp von Unity | | Kazu von den Kleinen Chaoten | 23.04.2013 | Aa Cc[chm] DD e[f]e[f] Gg P Spsp // Aa Cc[chm] DD ee[f] UwUw PP Spsp | — | 03.09.2017 | Stammbaum von Akio Kids, Stammbaum von Vance | -| Bruno of Black Forest | 01.06.2022 | aa C- dd Ee Gg P- spsp | Blau // Mystique of Black Forest | — | Stammbaum von Alberto Kids, Stammbaum von Fire Kids, Stammbaum von Stella Kids | | Milka of LennyLengo | 09.12.2018 | aa C- dd E- Gg P- Spsp // aa Cc[h] dd EE Gg P- Spsp | — | 22.12.2021 | Stammbaum von Alberto Kids, Stammbaum von Stella Kids | | Silvain von den Kleinen Chaoten | 27.03.2022 | aa c[chm]c[chm] Dd Ee[-] Gg P- Spsp // aa c[chm]c[chm] Dd ee[-] Gg Pp Spsp | — | 31.12.2024 | Stammbaum von Alberto Kids, Stammbaum von CP-Fuchs, CP-Sa Sp von Unity | | Enya von den Kleinen Chaoten | 01.11.2017 | Aa c[chm]c[chm] D- ee[-] G- P- spsp // Aa c[chm]c[chm] D- ee[-] Uwuw[d] P- spsp | — | — | Stammbaum von Alberto Kids, Stammbaum von Fire Kids, Stammbaum von Kohlief, Goldfuchsef Sp von Chrissi, Stammbaum von Stella Kids | | Little Hero of Black Forest | 22.02.2018 | AA CC DD EE GG PP [WFNZ] // AA CC DD EE GG PP spsp [WFNZ] | — | 18.06.2021 | Stammbaum von Alberto Kids, Stammbaum von CP-Fuchs, CP-Sa Sp von Unity, Stammbaum von Fire Kids, Stammbaum von Kohlief, Goldfuchsef Sp von Chrissi, Stammbaum von Stella Kids, Stammbaum von Valentino Firehearts Kids | | Molly of Black Forest | 13.09.2021 | /+, Aa Cc[chm] D- Ee gg P- spsp // Aa Cc[chm] Dd Ee gg Pp spsp | — | 03.05.2021 | Stammbaum von Alberto Kids, Stammbaum von CP-Fuchs, CP-Sa Sp von Unity | -| Little Runner's Big Ben | 03.02.2020 | Aa Cc[chm] DD Ee Gg PP Spsp // Aa Cc[chm] DD Ee Gg Pp Spsp | Daja of Little Rose | 14.10.2023 | Stammbaum von CP-Fuchs, CP-Sa Sp von Unity, Stammbaum von Fire Kids, Stammbaum von Goldfuchs Sp (Pikachu) Kids, Stammbaum von Kohlief, Goldfuchsef Sp von Chrissi, Stammbaum von Valentino Firehearts Kids, Stammbaum von Watarus Kids | +| Little Runner's Big Ben | 03.02.2020 | Aa Cc[chm] DD Ee Gg PP Spsp // Aa Cc[chm] DD Ee Gg Pp Spsp | — | 14.10.2023 | Stammbaum von CP-Fuchs, CP-Sa Sp von Unity, Stammbaum von Fire Kids, Stammbaum von Goldfuchs Sp (Pikachu) Kids, Stammbaum von Kohlief, Goldfuchsef Sp von Chrissi, Stammbaum von Valentino Firehearts Kids, Stammbaum von Watarus Kids | | Daja of Little Rose | 16.05.2021 | aa chmchm D- EE Gg P- // aa chmchm D- EE Gg P- spsp | — | — | Stammbaum von CP-Fuchs, CP-Sa Sp von Unity, Stammbaum von Fire Kids, Stammbaum von Valentino Firehearts Kids | -| Vance Jr. von den Kleinen Chaoten | 10.04.2022 | aa Cc[hm] Dd Ee gg P- Spsp // aa Cc[hm] Dd Ee gg P- spsp | Kohlfuchs, hell // Velvet von den Kleinen Chaoten | — | Stammbaum von Fire Kids, Stammbaum von Stella Kids | +| Vance Jr. von den Kleinen Chaoten | 10.04.2022 | aa Cc[hm] Dd Ee gg P- Spsp // aa Cc[hm] Dd Ee gg P- spsp | Kohlfuchs, hell | — | Stammbaum von Fire Kids, Stammbaum von Stella Kids | | Ichika von den Kleinen Chaoten | 19.04.2020 | aa CC D- ee Gg pp spsp // aa CC D- ee[f] Gg pp spsp | — | 27.11.2023 | Stammbaum von Goldfuchs Sp (Pikachu) Kids, Stammbaum von Watarus Kids | -| Trogir von den Kleinen Chaoten | 21.03.2022 | Aa CC DD EE GG pp Spsp | Gold Ansatzschecke // Mahima von den Kleinen Chaoten | 29.05.2024 | Stammbaum von Goldfuchs Sp (Pikachu) Kids, Stammbaum von Kohlief, Goldfuchsef Sp von Chrissi, Stammbaum von Watarus Kids | -| Chayton v.d. Kleinen Chaoten (extern SC) | 04.02.2022 | aa Cc[-] D- e[f]e[f] Gg Pp spsp | Orangeschimmel, hell // Victoria Welby gen. Welby v.d. Kleinen Chaoten | 30.04.2024 | Stammbaum von Goldfuchs Sp (Pikachu) Kids, Stammbaum von Kohlief, Goldfuchsef Sp von Chrissi, Stammbaum von Watarus Kids | | Victoria Welby gen. Welby v.d. Kleinen Chaoten | 16.01.2023 | Aa CC D- Ee[f] Gg pp Spsp [DP] // Aa CC D- ee[f] Gg pp Spsp [DP] | Goldfuchsschimmel Punktschecke DP | 17.02.2026 | Stammbaum von Goldfuchs Sp (Pikachu) Kids, Stammbaum von Kohlief, Goldfuchsef Sp von Chrissi, Stammbaum von Watarus Kids | -| Zac gen. Action von den Kleinen Chaoten | 25.12.2020 | aa C- D- Ee G- Pp Spsp [DP] // aa CC D- Ee G- Pp Spsp [DP] | Belica gen. Emi von den Kleinen Chaoten | 31.01.2025 | Stammbaum von Goldfuchs Sp (Pikachu) Kids, Stammbaum von Kohlief, Goldfuchsef Sp von Chrissi, Stammbaum von Watarus Kids | -| Chesnut | 13.11.2019 | aa C- D- ee[f] GG PP spsp | Kohlfuchsschimmel // Tennessee von den Kleinen Chaoten | 22.11.2023 | Stammbaum von Kentucky | -| Ethan von den Kleinen Chaoten | 09.07.2020 | Aa Cc[chm] D- ee[f] Gg Pp Spsp | Ichika von den Kleinen Chaoten // Orangeschimmel, hell Kragenschecke | 30.07.2024 | Stammbaum von Kentucky, Stammbaum von Watarus Kids | +| Zac gen. Action von den Kleinen Chaoten | 25.12.2020 | aa C- D- Ee G- Pp Spsp [DP] // aa CC D- Ee G- Pp Spsp [DP] | — | 31.01.2025 | Stammbaum von Goldfuchs Sp (Pikachu) Kids, Stammbaum von Kohlief, Goldfuchsef Sp von Chrissi, Stammbaum von Watarus Kids | | Hanami von den Kleinen Chaoten | 10.09.2015 | aa Cc[chm] D- Ee gg P- spsp | — | 12.12.2019 // 14.01.2020 | Stammbaum von Kentucky, Stammbaum von Stella Kids | | Skarlett v.d. Kleinen Chaoten | 14.07.2013 | / +2018, Aa Cc[chm] DD ee uw[d]uw[d] PP spsp // Aa Cc[chm] DD ee uw[d]uw[d] PP spsp | — | 17.04.2016 // 2018 | Stammbaum von Vance | diff --git a/tools/import/xlsx_util.py b/tools/import/xlsx_util.py index 2a8412a..02271f6 100644 --- a/tools/import/xlsx_util.py +++ b/tools/import/xlsx_util.py @@ -78,6 +78,46 @@ def read_cells(z, sheet_path, ss=None): return cells +def cell_fill_sex(z, sheet_path): + """{(colnum, row): 'male' | 'female'} from the cell's box fill colour. + + Breeder convention (Julian, 2026-06-06): a BLUE box = männlich (male), a WHITE box = + weiblich (female). In these Stammbaum templates every coloured box uses one solid theme + fill (Office accent5 = blue); unfilled/none cells render white. So: a cell whose style + uses a real solid fill -> 'male'; an unfilled (none/gray125) cell -> 'female'. + """ + try: + st = z.read("xl/styles.xml").decode("utf-8") + except KeyError: + return {} + fills = re.search(r"", st, re.S) + colored = set() + if fills: + for i, fb in enumerate(re.findall(r"(.*?)", fills.group(0), re.S)): + if 'patternType="solid"' in fb and re.search(r"", st, re.S) + idx2fill = {} + if xfs: + for i, xf in enumerate(re.findall(r"]*?)/?>", xfs.group(0))): + m = re.search(r'fillId="(\d+)"', xf) + idx2fill[i] = int(m.group(1)) if m else 0 + raw = z.read(sheet_path).decode("utf-8") + out = {} + for m in re.finditer(r"]*?)>", raw): + a = dict(_ATTR.findall(m.group(1))) + ref = a.get("r") + if not ref: + continue + mm = re.match(r"([A-Z]+)(\d+)", ref) + if not mm: + continue + s = int(a.get("s", "0")) + out[(col_to_num(mm.group(1)), int(mm.group(2)))] = ( + "male" if idx2fill.get(s, 0) in colored else "female") + return out + + def header_row(cells): """Return {colnum: header_text} for the topmost row that has text.""" if not cells: