Files
GerbilManager/GerbilManagerWebAPI/Import/ImportService.cs
Gulum ea79703dbf PEDIGREE-LINK: chart parentRefs→litters, box-colour=sex, name-bleed fix
Structural fix (god, Julian-reported via 'C'): the loader ignored
SourceAnimal.ParentRefs, so animals whose ancestry exists only as
Stammbaum chart-position refs loaded with LitterId=null ("unbekannt").
ImportService now synthesizes/reuses a derived litter from parentRefs:
resolves father+mother via name+DOB, groups siblings (same parents+dob)
into one litter, sets Father/Mother + offspring LitterId, dates it to the
offspring DOB, and tags Notes "aus Stammbaum-Diagramm abgeleitet
(Konfidenz: …)" so it's transparent/reversible. Existing animals that
become linkable are re-linked on re-run (sweep-idempotent). Dry-run counts
included. Projected: ~124 loadable animals gain a parent link.

Box-colour = sex (Julian): blue box = male, white box = female. All 11
pedigrees encode this as a solid theme-8 (accent5/blue) fill vs no fill.
xlsx_util.cell_fill_sex reads it; extract.py sets animal.gender from the
box; ImportService.InferGender prefers it over sire/dam name inference.
Result: 306/306 loadable animals now sexed (154♂/152♀).

Extractor noise fix (god): reject Farbschlag values that are actually a
parent NAME bled across cells (contain v.d./von/of/gen.) — cleared phantom
conflicts (e.g. Chayton). Combined with GEN-3 Uw→G: Konflikte 32→21.
Also skip Excel "~$" lock files in the glob.

GEN-3a contract (Kevin): ComposeGenotype appends "Slsl" for WP/Sls
carriers (wild-type sl/sl omitted) so 8-locus strings stay unchanged.

Importer-only. The live re-import into Julian's DB stays a separate
supervised gated step. 95 C# tests + python genotype tests green;
has-pending-model-changes clean (no schema change on this branch).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-06 10:48:30 +02:00

438 lines
22 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
using System.Text.Json;
using System.Text.RegularExpressions;
using GerbilManagerWebAPI.Models;
using Microsoft.EntityFrameworkCore;
namespace GerbilManagerWebAPI.Import
{
/// <summary>
/// FEAT-8c import loader. Consumes tools/import/output (animals.json + litters.json
/// produced by extract.py) and loads conflict-free data into the database:
/// litters first (Wurfchronik = authoritative), then animals matched onto them.
///
/// Load policy:
/// - Litters: all created (idempotent by ExternalRef = source litter id).
/// - Animals: created when they have a DOB and are NOT in conflict. The birth-litter
/// link is set only for HIGH-confidence matches (litterRef.confidence == "hoch");
/// date-only/ambiguous links are quarantined (animal loads with LitterId = null).
/// - QUARANTINED (never loaded): conflicts + stubs (no DOB) — await the wife's review.
/// Idempotent: re-running matches on ExternalRef and skips existing rows.
/// Execute is gated by the endpoint; this service only acts when asked.
/// </summary>
public sealed class ImportService
{
private static readonly string[] LocusOrder = { "A", "C", "D", "E", "G", "P", "Sp", "Re" };
private static readonly JsonSerializerOptions Json = new()
{
PropertyNameCaseInsensitive = true,
};
private const string ImportSourceTag = "FEAT-8 Stammbaum/Wurfchronik";
private readonly ApplicationContext _db;
private readonly string _sourceDir;
private readonly string _photoRoot;
/// <summary>Test-friendly constructor with explicit paths.</summary>
public ImportService(ApplicationContext db, string sourceDir, string photoRoot)
{
_db = db;
_sourceDir = sourceDir;
_photoRoot = photoRoot;
}
public ImportService(ApplicationContext db, IConfiguration config, IWebHostEnvironment env)
: this(db,
config["Import:SourcePath"]
?? Path.GetFullPath(Path.Combine(env.ContentRootPath, "..", "tools", "import", "output")),
config["Photos:RootPath"] ?? Path.Combine(env.ContentRootPath, "photo-storage"))
{
}
public async Task<ImportReport> RunAsync(bool execute)
{
var notes = new List<string>();
var samples = new List<string>();
var animals = Load<List<SourceAnimal>>("animals.json") ?? new();
var litters = Load<List<SourceLitter>>("litters.json") ?? new();
if (animals.Count == 0 && litters.Count == 0)
notes.Add($"Keine Quelldaten gefunden in {_sourceDir} (animals.json/litters.json). extract.py zuerst ausführen.");
// ---- categorise animals ----
var loadable = new List<SourceAnimal>();
int conflicts = 0, stubs = 0, dateOnly = 0, ambiguous = 0;
foreach (var a in animals)
{
if (a.Conflict) { conflicts++; continue; }
if (string.IsNullOrEmpty(a.Dob)) { stubs++; continue; }
loadable.Add(a);
var conf = a.LitterRef?.Confidence;
if (a.LitterRef?.Candidates is { Count: > 0 }) ambiguous++;
else if (conf == "niedrig") dateOnly++;
}
// existing rows (idempotency). Gerbils carry ExternalRef; Litters have no such
// column, so we key litter idempotency on the stable (Name + Date) pair instead.
var existingGerbilExtRefs = await _db.Gerbils
.Where(g => g.ExternalRef != null)
.Select(g => g.ExternalRef!).ToListAsync();
var existingGerbilSet = existingGerbilExtRefs.ToHashSet();
var existingLitterKeys = await _db.Litters
.Select(l => new { l.Name, l.Date }).ToListAsync();
var existingLitterKeySet = existingLitterKeys
.Select(x => $"{x.Name}|{x.Date:yyyy-MM-dd}").ToHashSet();
// colour-variety name -> id (case-insensitive)
var varieties = await _db.ColorVarieties.Select(v => new { v.Id, v.Name }).ToListAsync();
var varietyByName = varieties
.GroupBy(v => v.Name.Trim().ToLowerInvariant())
.ToDictionary(g => g.Key, g => g.First().Id);
// gender inference from litter roles (sire -> male, dam -> female; both -> unknown)
var sireNames = litters.Select(l => Normalize(StripZucht(l.SireName))).Where(s => s.Length > 0).ToHashSet();
var damNames = litters.Select(l => Normalize(StripZucht(l.DamName))).Where(s => s.Length > 0).ToHashSet();
// ---- litters: create map source.id -> Litter (for high-confidence animal links) ----
int littersCreated = 0, littersExisting = 0;
var litterIdMap = new Dictionary<string, Guid>(); // source litter id -> Litter.Id
foreach (var sl in litters)
{
var date = ParseDate(sl.Date);
var name = $"Wurf {sl.LitterId}".Trim();
var key = $"{name}|{date:yyyy-MM-dd}";
if (existingLitterKeySet.Contains(key)) { littersExisting++; continue; }
var id = Guid.NewGuid();
litterIdMap[sl.Id] = id;
littersCreated++;
if (execute && date is DateOnly d)
{
_db.Litters.Add(new Litter
{
Id = id,
Name = name,
Date = d,
TotalBorn = sl.TotalBorn,
Notes = string.IsNullOrWhiteSpace(sl.Note) ? null : sl.Note,
PairingCode = string.IsNullOrWhiteSpace(sl.Zuchtnummer) ? null : sl.Zuchtnummer,
});
}
if (samples.Count < 8 && date is not null)
samples.Add($"Wurf: {name} ({sl.Date}) — {sl.DamName} × {sl.SireName}");
}
if (execute) await _db.SaveChangesAsync();
// ---- animals ----
int animalsCreated = 0, linked = 0, fbMatched = 0, fbUnmatched = 0, animalsExisting = 0;
int photosAttached = 0, photosMissing = 0;
var createdAnimalByName = new Dictionary<string, Guid>(); // normalized name -> gerbil id (for litter back-link)
// name+DOB -> gid index, across EXISTING rows AND this run's planned animals, so that
// chart-position parentRefs (PEDIGREE-LINK) can resolve a parent to a real gerbil id.
var existingRows = await _db.Gerbils
.Select(g => new { g.Id, g.Name, g.DateOfBirth, g.ExternalRef, g.LitterId }).ToListAsync();
var gidByNameDob = new Dictionary<string, Guid>();
foreach (var g in existingRows)
gidByNameDob[NameDobKey(g.Name, g.DateOfBirth)] = g.Id;
var existingLitterByExtRef = existingRows.Where(g => g.ExternalRef != null)
.ToDictionary(g => g.ExternalRef!, g => g.LitterId);
// PASS 1: assign ids + resolve fb/gender/Wurfchronik link (no writes yet).
var plan = new List<AnimalPlan>();
foreach (var a in loadable)
{
bool exists = existingGerbilSet.Contains(a.Id);
var gid = exists ? gidByNameDob[NameDobKey(a.Name, ParseDate(a.Dob))] : Guid.NewGuid();
Guid? wurfLitterId = null;
if (a.LitterRef?.Confidence == "hoch" && a.LitterRef.Candidates is not { Count: > 0 }
&& litterIdMap.TryGetValue(a.LitterRef.LitterId, out var lid))
wurfLitterId = lid;
Guid? colorVarietyId = null;
var fbCandidates = new[] { a.Farbschlag }.Concat(a.FarbschlagVariants)
.Where(s => !string.IsNullOrWhiteSpace(s));
foreach (var fb in fbCandidates)
if (varietyByName.TryGetValue(fb.Trim().ToLowerInvariant(), out var vid))
{ colorVarietyId = vid; break; }
var gender = InferGender(a, sireNames, damNames);
var norm = Normalize(StripZucht(a.Name));
if (norm.Length > 0) createdAnimalByName.TryAdd(norm, gid);
if (!exists) gidByNameDob.TryAdd(NameDobKey(a.Name, ParseDate(a.Dob)), gid);
var currentLitter = exists && existingLitterByExtRef.TryGetValue(a.Id, out var el) ? el : null;
plan.Add(new AnimalPlan(a, gid, exists, wurfLitterId, currentLitter, colorVarietyId, gender));
}
// PASS 1.5: PEDIGREE-LINK — synthesize/reuse a litter from chart-position parentRefs for
// any animal that has no Wurfchronik litter and isn't already litter-linked. Siblings
// (same father+mother+dob) share one derived litter. Computed for dry-run counts too.
int parentLinksAdded = 0, derivedLitters = 0;
var synthLitterForGid = new Dictionary<Guid, Guid>(); // offspring gid -> synth litter id
var synthLitters = new Dictionary<string, SynthLitter>(); // parents+date key -> synth litter
foreach (var p in plan)
{
if (p.WurfLitterId is not null || p.CurrentLitterId is not null) continue;
if (p.A.ParentRefs is not { Count: > 0 }) continue;
var father = ResolveParentGid(p.A, "father", gidByNameDob);
var mother = ResolveParentGid(p.A, "mother", gidByNameDob);
if (father is null && mother is null) continue; // nothing resolvable to link
var dob = ParseDate(p.A.Dob);
var conf = p.A.ParentRefs.FirstOrDefault()?.Confidence ?? "medium";
var key = $"{father}|{mother}|{dob:yyyy-MM-dd}";
if (!synthLitters.TryGetValue(key, out var sl))
{
sl = new SynthLitter(Guid.NewGuid(), father, mother, dob, conf);
synthLitters[key] = sl;
derivedLitters++;
}
synthLitterForGid[p.Gid] = sl.Id;
parentLinksAdded++;
}
// PASS 2: write (litters synthesized first so offspring FK resolves), then animals + photos.
if (execute)
{
// reuse an existing litter with the same parents+date instead of duplicating.
var existingLitterRows = await _db.Litters
.Select(l => new { l.Id, l.FatherId, l.MotherId, l.Date }).ToListAsync();
var litterByParentsDate = new Dictionary<string, Guid>();
foreach (var l in existingLitterRows)
litterByParentsDate[$"{l.FatherId}|{l.MotherId}|{l.Date:yyyy-MM-dd}"] = l.Id;
foreach (var sl in synthLitters.Values.ToList())
{
var reuseKey = $"{sl.Father}|{sl.Mother}|{sl.Date:yyyy-MM-dd}";
if (litterByParentsDate.TryGetValue(reuseKey, out var existingId))
{
// remap offspring to the existing litter; don't create a duplicate.
foreach (var g in synthLitterForGid.Where(kv => kv.Value == sl.Id).Select(kv => kv.Key).ToList())
synthLitterForGid[g] = existingId;
derivedLitters--;
continue;
}
_db.Litters.Add(new Litter
{
Id = sl.Id,
Name = $"Wurf (aus Diagramm) {sl.Date:yyyy-MM-dd}".Trim(),
Date = sl.Date ?? default,
FatherId = sl.Father,
MotherId = sl.Mother,
Notes = $"aus Stammbaum-Diagramm abgeleitet (Konfidenz: {sl.Confidence})",
});
}
await _db.SaveChangesAsync();
}
foreach (var p in plan)
{
Guid? litterId = p.WurfLitterId
?? (synthLitterForGid.TryGetValue(p.Gid, out var slid) ? slid : (Guid?)null);
if (litterId is not null) linked++;
if (p.ColorVarietyId is null) fbUnmatched++; else fbMatched++;
if (samples.Count < 16)
samples.Add($"Tier: {p.A.Name} (*{p.A.Dob}), Genotyp {ComposeGenotype(p.A.Genotype)}"
+ (litterId is not null ? (p.WurfLitterId is not null ? ", Wurf-verknüpft" : ", Eltern aus Diagramm") : "")
+ (p.ColorVarietyId is not null ? $", Farbschlag „{p.A.Farbschlag}\"" : ""));
if (p.Exists)
{
animalsExisting++;
// re-link an existing animal that just became linkable (sweep idempotency).
if (execute && p.CurrentLitterId is null && litterId is not null)
{
var row = await _db.Gerbils.FirstOrDefaultAsync(g => g.Id == p.Gid);
if (row is not null) row.LitterId = litterId;
}
continue;
}
animalsCreated++;
if (execute)
{
_db.Gerbils.Add(new Gerbil
{
Id = p.Gid,
Name = p.A.Name,
Gender = p.Gender,
Status = GerbilStatus.Active,
DateOfBirth = ParseDate(p.A.Dob),
DateOfDeath = ParseDate(p.A.Death),
LitterId = litterId,
ColorVarietyId = p.ColorVarietyId,
Genotype = ComposeGenotype(p.A.Genotype),
IsDeaf = p.A.Deaf,
ImportSource = ImportSourceTag,
ExternalRef = p.A.Id,
OriginBreeder = string.IsNullOrWhiteSpace(p.A.Zucht) ? null : p.A.Zucht.Trim(),
RawImportData = JsonSerializer.Serialize(new
{
p.A.Genotype.RawGenotype,
p.A.Genotype.UnmappedTokens,
// GEN-3b: Sls (2nd spotting locus) preserved here until Kevin's GEN-3a
// parser adopts it into the compact Genotype contract; tags + deaf too.
Sls = p.A.Genotype.Mapped8locus.TryGetValue("Sls", out var sls) ? sls : null,
p.A.Tags,
p.A.Deaf,
p.A.Zucht,
p.A.SourceFiles,
FarbschlagRaw = p.A.Farbschlag,
}),
});
}
// photos
foreach (var rel in p.A.Photos)
{
var src = Path.Combine(_sourceDir, rel.Replace('/', Path.DirectorySeparatorChar));
if (!File.Exists(src)) { photosMissing++; continue; }
photosAttached++;
if (execute)
{
Directory.CreateDirectory(_photoRoot);
var fileName = $"{Guid.NewGuid():N}{Path.GetExtension(src)}";
File.Copy(src, Path.Combine(_photoRoot, fileName), overwrite: true);
_db.GerbilPhotos.Add(new GerbilPhoto
{
Id = Guid.NewGuid(),
GerbilId = p.Gid,
FileName = fileName,
SortOrder = 0,
CreatedAt = DateTimeOffset.UtcNow,
});
}
}
}
if (execute) await _db.SaveChangesAsync();
// ---- back-link Wurfchronik litter parents by name (best effort) ----
if (execute)
{
foreach (var sl in litters)
{
if (!litterIdMap.TryGetValue(sl.Id, out var lid)) continue;
var litter = await _db.Litters.FirstOrDefaultAsync(l => l.Id == lid);
if (litter is null) continue;
if (createdAnimalByName.TryGetValue(Normalize(StripZucht(sl.SireName)), out var fId))
litter.FatherId = fId;
if (createdAnimalByName.TryGetValue(Normalize(StripZucht(sl.DamName)), out var mId))
litter.MotherId = mId;
}
await _db.SaveChangesAsync();
}
notes.Add("Quarantäne (kein Import): Konflikte + Stubs ohne Geburtsdatum + unsichere Wurf-Zuordnungen — warten auf die Prüfung durch die Züchterin.");
if (parentLinksAdded > 0)
notes.Add($"Stammbaum-Diagramm: {parentLinksAdded} Tiere über Eltern-Verknüpfung einem (abgeleiteten) Wurf zugeordnet ({derivedLitters} abgeleitete Würfe).");
if (!execute) notes.Add("DRY-RUN: nichts gespeichert. /import/execute lädt die konfliktfreien Daten.");
return new ImportReport(
Executed: execute,
Litters: new LitterSummary(litters.Count, littersCreated, littersExisting, derivedLitters),
Animals: new AnimalSummary(
animals.Count, animalsCreated, linked, fbMatched, fbUnmatched, animalsExisting,
new QuarantineSummary(conflicts, stubs, dateOnly, ambiguous, conflicts + stubs),
parentLinksAdded),
Photos: new PhotoSummary(photosAttached, photosMissing),
Samples: samples,
Notes: notes);
}
private T? Load<T>(string file)
{
var path = Path.Combine(_sourceDir, file);
if (!File.Exists(path)) return default;
using var fs = File.OpenRead(path);
return JsonSerializer.Deserialize<T>(fs, Json);
}
public static string ComposeGenotype(SourceGenotype g)
{
var tokens = LocusOrder.Select(locus =>
{
if (g.Mapped8locus.TryGetValue(locus, out var pair) && pair.Count == 2)
return StripCaret(pair[0]) + StripCaret(pair[1]);
return "??";
}).ToList();
// GEN-3a contract (Kevin): Sls is appended LAST and ONLY for carriers — the
// wild-type sl/sl is omitted so existing 8-locus strings stay unchanged. WP het
// renders as the trailing token "Slsl"; S(l)S(l) is lethal so never appears.
if (g.Mapped8locus.TryGetValue("Sls", out var sls) && sls.Count == 2
&& !(sls[0] == "sl" && sls[1] == "sl"))
{
tokens.Add(StripCaret(sls[0]) + StripCaret(sls[1]));
}
return string.Join(' ', tokens);
}
private static string StripCaret(string allele) => allele.Replace("^", "");
private static Gender InferGender(SourceAnimal a, HashSet<string> sires, HashSet<string> dams)
{
// Box colour (blue=male, white=female) is the authoritative breeder signal — prefer it
// over sire/dam name inference (PEDIGREE-LINK, Julian 2026-06-06).
if (string.Equals(a.Gender, "male", StringComparison.OrdinalIgnoreCase)) return Gender.male;
if (string.Equals(a.Gender, "female", StringComparison.OrdinalIgnoreCase)) return Gender.female;
var n = Normalize(StripZucht(a.Name));
bool isSire = sires.Contains(n), isDam = dams.Contains(n);
if (isSire && !isDam) return Gender.male;
if (isDam && !isSire) return Gender.female;
return Gender.unknown;
}
public static DateOnly? ParseDate(string s)
{
if (string.IsNullOrWhiteSpace(s)) return null;
var m = Regex.Match(s, @"(\d{1,2})\.(\d{1,2})\.(\d{2,4})");
if (!m.Success) return null;
int d = int.Parse(m.Groups[1].Value), mo = int.Parse(m.Groups[2].Value);
int y = int.Parse(m.Groups[3].Value);
if (y < 100) y += 2000;
try { return new DateOnly(y, mo, d); }
catch { return null; }
}
private static string StripZucht(string name)
{
var n = Regex.Replace(name ?? "", @"\[.*?\]", " ");
n = Regex.Replace(n, @"\s+(?:of|von\s+den|von\s+der|v\.\s?d\.|von)\s+.*$", "", RegexOptions.IgnoreCase);
return n.Trim();
}
private static string Normalize(string name)
{
var n = (name ?? "").ToLowerInvariant();
n = Regex.Replace(n, @"\bgen\.\b", " ");
n = Regex.Replace(n, @"[^a-z0-9äöüß]", "");
return n;
}
/// <summary>Dedup identity for parent resolution: normalized call-name + DOB.</summary>
private static string NameDobKey(string name, DateOnly? dob) =>
$"{Normalize(StripZucht(name))}|{dob:yyyy-MM-dd}";
/// <summary>Resolve a chart-position parentRef (by role) to a known gerbil id, or null.</summary>
private static Guid? ResolveParentGid(SourceAnimal a, string role, Dictionary<string, Guid> gidByNameDob)
{
var pr = a.ParentRefs.FirstOrDefault(p =>
string.Equals(p.RoleGuess, role, StringComparison.OrdinalIgnoreCase));
if (pr is null || string.IsNullOrWhiteSpace(pr.Name)) return null;
return gidByNameDob.TryGetValue(NameDobKey(pr.Name, ParseDate(pr.Dob)), out var id) ? id : null;
}
/// <summary>Per-animal plan computed before any write so synthesis can run in dry-run too.</summary>
private sealed record AnimalPlan(
SourceAnimal A, Guid Gid, bool Exists, Guid? WurfLitterId,
Guid? CurrentLitterId, Guid? ColorVarietyId, Gender Gender);
/// <summary>A litter synthesized from chart-position parentRefs (PEDIGREE-LINK).</summary>
private sealed record SynthLitter(Guid Id, Guid? Father, Guid? Mother, DateOnly? Date, string Confidence);
}
}