Files
GerbilManager/GerbilManagerWebAPI/Import/ImportService.cs
Gulum 6775707387 OWNERSHIP: residency classification (Bestand vs external ancestors)
Per hive/agents/god/OWNERSHIP-residency.md (Julian): "Clan kleine Chaoten"
is the wife's own Zucht. Classify each animal:
 (a) RESIDENT if zuchtCanon contains 'kleinechaote' (separator/declension-
     insensitive: "v.d. Kleinen Chaoten", "Clan-Kleine-Chaoten", …); OR
 (b) PARENT EXCEPTION — a parent of a Clan offspring is resident even if its
     own Zuchtname is foreign (you can only breed a Clan litter if the parents
     were in your care). Runs AFTER the parentRefs→litter linking.
Otherwise EXTERNAL (pedigree ancestor, not living stock — kept, not deleted).

Stored as a dedicated Gerbil.IsResident bool (distinct from Status/Abgabe —
origin/identity, not current location), defaulting true (own stock unless
marked external; DB HasDefaultValue(true)). Additive migration
AddGerbilResidency; has-pending-model-changes clean. Round-trips in
GerbilDto/GerbilInput and is Gridify-filterable (isResident==true) so Kevin
can build a "nur Bestand" filter. ImportService computes it at (re-)import
and refreshes existing rows on the idempotent sweep. ResidencySummary added
to the import report.

Projected on real data: 306 loadable → 227 resident (188 rule a, +39 via
parent exception), 79 external ancestors.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-06 10:58:29 +02:00

479 lines
25 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
using System.Text.Json;
using System.Text.RegularExpressions;
using GerbilManagerWebAPI.Models;
using Microsoft.EntityFrameworkCore;
namespace GerbilManagerWebAPI.Import
{
/// <summary>
/// FEAT-8c import loader. Consumes tools/import/output (animals.json + litters.json
/// produced by extract.py) and loads conflict-free data into the database:
/// litters first (Wurfchronik = authoritative), then animals matched onto them.
///
/// Load policy:
/// - Litters: all created (idempotent by ExternalRef = source litter id).
/// - Animals: created when they have a DOB and are NOT in conflict. The birth-litter
/// link is set only for HIGH-confidence matches (litterRef.confidence == "hoch");
/// date-only/ambiguous links are quarantined (animal loads with LitterId = null).
/// - QUARANTINED (never loaded): conflicts + stubs (no DOB) — await the wife's review.
/// Idempotent: re-running matches on ExternalRef and skips existing rows.
/// Execute is gated by the endpoint; this service only acts when asked.
/// </summary>
public sealed class ImportService
{
private static readonly string[] LocusOrder = { "A", "C", "D", "E", "G", "P", "Sp", "Re" };
private static readonly JsonSerializerOptions Json = new()
{
PropertyNameCaseInsensitive = true,
};
private const string ImportSourceTag = "FEAT-8 Stammbaum/Wurfchronik";
private readonly ApplicationContext _db;
private readonly string _sourceDir;
private readonly string _photoRoot;
/// <summary>Test-friendly constructor with explicit paths.</summary>
public ImportService(ApplicationContext db, string sourceDir, string photoRoot)
{
_db = db;
_sourceDir = sourceDir;
_photoRoot = photoRoot;
}
public ImportService(ApplicationContext db, IConfiguration config, IWebHostEnvironment env)
: this(db,
config["Import:SourcePath"]
?? Path.GetFullPath(Path.Combine(env.ContentRootPath, "..", "tools", "import", "output")),
config["Photos:RootPath"] ?? Path.Combine(env.ContentRootPath, "photo-storage"))
{
}
public async Task<ImportReport> RunAsync(bool execute)
{
var notes = new List<string>();
var samples = new List<string>();
var animals = Load<List<SourceAnimal>>("animals.json") ?? new();
var litters = Load<List<SourceLitter>>("litters.json") ?? new();
if (animals.Count == 0 && litters.Count == 0)
notes.Add($"Keine Quelldaten gefunden in {_sourceDir} (animals.json/litters.json). extract.py zuerst ausführen.");
// ---- categorise animals ----
var loadable = new List<SourceAnimal>();
int conflicts = 0, stubs = 0, dateOnly = 0, ambiguous = 0;
foreach (var a in animals)
{
if (a.Conflict) { conflicts++; continue; }
if (string.IsNullOrEmpty(a.Dob)) { stubs++; continue; }
loadable.Add(a);
var conf = a.LitterRef?.Confidence;
if (a.LitterRef?.Candidates is { Count: > 0 }) ambiguous++;
else if (conf == "niedrig") dateOnly++;
}
// existing rows (idempotency). Gerbils carry ExternalRef; Litters have no such
// column, so we key litter idempotency on the stable (Name + Date) pair instead.
var existingGerbilExtRefs = await _db.Gerbils
.Where(g => g.ExternalRef != null)
.Select(g => g.ExternalRef!).ToListAsync();
var existingGerbilSet = existingGerbilExtRefs.ToHashSet();
var existingLitterKeys = await _db.Litters
.Select(l => new { l.Name, l.Date }).ToListAsync();
var existingLitterKeySet = existingLitterKeys
.Select(x => $"{x.Name}|{x.Date:yyyy-MM-dd}").ToHashSet();
// colour-variety name -> id (case-insensitive)
var varieties = await _db.ColorVarieties.Select(v => new { v.Id, v.Name }).ToListAsync();
var varietyByName = varieties
.GroupBy(v => v.Name.Trim().ToLowerInvariant())
.ToDictionary(g => g.Key, g => g.First().Id);
// gender inference from litter roles (sire -> male, dam -> female; both -> unknown)
var sireNames = litters.Select(l => Normalize(StripZucht(l.SireName))).Where(s => s.Length > 0).ToHashSet();
var damNames = litters.Select(l => Normalize(StripZucht(l.DamName))).Where(s => s.Length > 0).ToHashSet();
// ---- litters: create map source.id -> Litter (for high-confidence animal links) ----
int littersCreated = 0, littersExisting = 0;
var litterIdMap = new Dictionary<string, Guid>(); // source litter id -> Litter.Id
foreach (var sl in litters)
{
var date = ParseDate(sl.Date);
var name = $"Wurf {sl.LitterId}".Trim();
var key = $"{name}|{date:yyyy-MM-dd}";
if (existingLitterKeySet.Contains(key)) { littersExisting++; continue; }
var id = Guid.NewGuid();
litterIdMap[sl.Id] = id;
littersCreated++;
if (execute && date is DateOnly d)
{
_db.Litters.Add(new Litter
{
Id = id,
Name = name,
Date = d,
TotalBorn = sl.TotalBorn,
Notes = string.IsNullOrWhiteSpace(sl.Note) ? null : sl.Note,
PairingCode = string.IsNullOrWhiteSpace(sl.Zuchtnummer) ? null : sl.Zuchtnummer,
});
}
if (samples.Count < 8 && date is not null)
samples.Add($"Wurf: {name} ({sl.Date}) — {sl.DamName} × {sl.SireName}");
}
if (execute) await _db.SaveChangesAsync();
// ---- animals ----
int animalsCreated = 0, linked = 0, fbMatched = 0, fbUnmatched = 0, animalsExisting = 0;
int photosAttached = 0, photosMissing = 0;
var createdAnimalByName = new Dictionary<string, Guid>(); // normalized name -> gerbil id (for litter back-link)
// name+DOB -> gid index, across EXISTING rows AND this run's planned animals, so that
// chart-position parentRefs (PEDIGREE-LINK) can resolve a parent to a real gerbil id.
var existingRows = await _db.Gerbils
.Select(g => new { g.Id, g.Name, g.DateOfBirth, g.ExternalRef, g.LitterId }).ToListAsync();
var gidByNameDob = new Dictionary<string, Guid>();
foreach (var g in existingRows)
gidByNameDob[NameDobKey(g.Name, g.DateOfBirth)] = g.Id;
var existingLitterByExtRef = existingRows.Where(g => g.ExternalRef != null)
.ToDictionary(g => g.ExternalRef!, g => g.LitterId);
// PASS 1: assign ids + resolve fb/gender/Wurfchronik link (no writes yet).
var plan = new List<AnimalPlan>();
foreach (var a in loadable)
{
bool exists = existingGerbilSet.Contains(a.Id);
var gid = exists ? gidByNameDob[NameDobKey(a.Name, ParseDate(a.Dob))] : Guid.NewGuid();
Guid? wurfLitterId = null;
if (a.LitterRef?.Confidence == "hoch" && a.LitterRef.Candidates is not { Count: > 0 }
&& litterIdMap.TryGetValue(a.LitterRef.LitterId, out var lid))
wurfLitterId = lid;
Guid? colorVarietyId = null;
var fbCandidates = new[] { a.Farbschlag }.Concat(a.FarbschlagVariants)
.Where(s => !string.IsNullOrWhiteSpace(s));
foreach (var fb in fbCandidates)
if (varietyByName.TryGetValue(fb.Trim().ToLowerInvariant(), out var vid))
{ colorVarietyId = vid; break; }
var gender = InferGender(a, sireNames, damNames);
var norm = Normalize(StripZucht(a.Name));
if (norm.Length > 0) createdAnimalByName.TryAdd(norm, gid);
if (!exists) gidByNameDob.TryAdd(NameDobKey(a.Name, ParseDate(a.Dob)), gid);
var currentLitter = exists && existingLitterByExtRef.TryGetValue(a.Id, out var el) ? el : null;
plan.Add(new AnimalPlan(a, gid, exists, wurfLitterId, currentLitter, colorVarietyId, gender));
}
// PASS 1.5: PEDIGREE-LINK — synthesize/reuse a litter from chart-position parentRefs for
// any animal that has no Wurfchronik litter and isn't already litter-linked. Siblings
// (same father+mother+dob) share one derived litter. Computed for dry-run counts too.
int parentLinksAdded = 0, derivedLitters = 0;
var synthLitterForGid = new Dictionary<Guid, Guid>(); // offspring gid -> synth litter id
var synthLitters = new Dictionary<string, SynthLitter>(); // parents+date key -> synth litter
foreach (var p in plan)
{
if (p.WurfLitterId is not null || p.CurrentLitterId is not null) continue;
if (p.A.ParentRefs is not { Count: > 0 }) continue;
var father = ResolveParentGid(p.A, "father", gidByNameDob);
var mother = ResolveParentGid(p.A, "mother", gidByNameDob);
if (father is null && mother is null) continue; // nothing resolvable to link
var dob = ParseDate(p.A.Dob);
var conf = p.A.ParentRefs.FirstOrDefault()?.Confidence ?? "medium";
var key = $"{father}|{mother}|{dob:yyyy-MM-dd}";
if (!synthLitters.TryGetValue(key, out var sl))
{
sl = new SynthLitter(Guid.NewGuid(), father, mother, dob, conf);
synthLitters[key] = sl;
derivedLitters++;
}
synthLitterForGid[p.Gid] = sl.Id;
parentLinksAdded++;
}
// litter id -> (father, mother) gids, across synthesized + Wurfchronik (by name) litters.
// Used by the residency rule (b) below; augmented with existing DB litters under execute.
var litterParents = new Dictionary<Guid, (Guid? F, Guid? M)>();
foreach (var sl in synthLitters.Values)
litterParents[sl.Id] = (sl.Father, sl.Mother);
foreach (var sl in litters)
if (litterIdMap.TryGetValue(sl.Id, out var lid))
litterParents[lid] = (
createdAnimalByName.TryGetValue(Normalize(StripZucht(sl.SireName)), out var fid) ? fid : (Guid?)null,
createdAnimalByName.TryGetValue(Normalize(StripZucht(sl.DamName)), out var mid) ? mid : (Guid?)null);
// PASS 2: write (litters synthesized first so offspring FK resolves), then animals + photos.
if (execute)
{
// reuse an existing litter with the same parents+date instead of duplicating.
var existingLitterRows = await _db.Litters
.Select(l => new { l.Id, l.FatherId, l.MotherId, l.Date }).ToListAsync();
var litterByParentsDate = new Dictionary<string, Guid>();
foreach (var l in existingLitterRows)
{
litterByParentsDate[$"{l.FatherId}|{l.MotherId}|{l.Date:yyyy-MM-dd}"] = l.Id;
litterParents[l.Id] = (l.FatherId, l.MotherId);
}
foreach (var sl in synthLitters.Values.ToList())
{
var reuseKey = $"{sl.Father}|{sl.Mother}|{sl.Date:yyyy-MM-dd}";
if (litterByParentsDate.TryGetValue(reuseKey, out var existingId))
{
// remap offspring to the existing litter; don't create a duplicate.
foreach (var g in synthLitterForGid.Where(kv => kv.Value == sl.Id).Select(kv => kv.Key).ToList())
synthLitterForGid[g] = existingId;
derivedLitters--;
continue;
}
_db.Litters.Add(new Litter
{
Id = sl.Id,
Name = $"Wurf (aus Diagramm) {sl.Date:yyyy-MM-dd}".Trim(),
Date = sl.Date ?? default,
FatherId = sl.Father,
MotherId = sl.Mother,
Notes = $"aus Stammbaum-Diagramm abgeleitet (Konfidenz: {sl.Confidence})",
});
}
await _db.SaveChangesAsync();
}
// OWNERSHIP/RESIDENCY (runs AFTER litter links exist): (a) Zuchtname matches the Clan
// kennel (zuchtCanon contains 'kleinechaote'); (b) parent of a Clan offspring, even if
// the parent's own Zuchtname is foreign. See hive/agents/god/OWNERSHIP-residency.md.
static bool ClanCanon(string? zc) =>
(zc ?? "").Contains("kleinechaote", StringComparison.OrdinalIgnoreCase);
var resident = new HashSet<Guid>();
foreach (var p in plan) if (ClanCanon(p.A.ZuchtCanon)) resident.Add(p.Gid); // (a)
int residentByA = resident.Count, flippedByParentRule = 0;
foreach (var p in plan)
{
if (!ClanCanon(p.A.ZuchtCanon)) continue;
var litId = p.WurfLitterId ?? (synthLitterForGid.TryGetValue(p.Gid, out var s) ? s : (Guid?)null);
if (litId is null || !litterParents.TryGetValue(litId.Value, out var par)) continue;
if (par.F is Guid gf && resident.Add(gf)) flippedByParentRule++; // (b)
if (par.M is Guid gm && resident.Add(gm)) flippedByParentRule++;
}
int residentTotal = plan.Count(p => resident.Contains(p.Gid));
int externalTotal = plan.Count - residentTotal;
foreach (var p in plan)
{
Guid? litterId = p.WurfLitterId
?? (synthLitterForGid.TryGetValue(p.Gid, out var slid) ? slid : (Guid?)null);
if (litterId is not null) linked++;
bool isResident = resident.Contains(p.Gid);
if (p.ColorVarietyId is null) fbUnmatched++; else fbMatched++;
if (samples.Count < 16)
samples.Add($"Tier: {p.A.Name} (*{p.A.Dob}), Genotyp {ComposeGenotype(p.A.Genotype)}"
+ (litterId is not null ? (p.WurfLitterId is not null ? ", Wurf-verknüpft" : ", Eltern aus Diagramm") : "")
+ (p.ColorVarietyId is not null ? $", Farbschlag „{p.A.Farbschlag}\"" : ""));
if (p.Exists)
{
animalsExisting++;
// sweep idempotency: re-link a now-linkable animal + refresh its residency.
if (execute)
{
var row = await _db.Gerbils.FirstOrDefaultAsync(g => g.Id == p.Gid);
if (row is not null)
{
if (p.CurrentLitterId is null && litterId is not null) row.LitterId = litterId;
row.IsResident = isResident;
}
}
continue;
}
animalsCreated++;
if (execute)
{
_db.Gerbils.Add(new Gerbil
{
Id = p.Gid,
Name = p.A.Name,
Gender = p.Gender,
Status = GerbilStatus.Active,
DateOfBirth = ParseDate(p.A.Dob),
DateOfDeath = ParseDate(p.A.Death),
LitterId = litterId,
ColorVarietyId = p.ColorVarietyId,
Genotype = ComposeGenotype(p.A.Genotype),
IsDeaf = p.A.Deaf,
IsResident = isResident,
ImportSource = ImportSourceTag,
ExternalRef = p.A.Id,
OriginBreeder = string.IsNullOrWhiteSpace(p.A.Zucht) ? null : p.A.Zucht.Trim(),
RawImportData = JsonSerializer.Serialize(new
{
p.A.Genotype.RawGenotype,
p.A.Genotype.UnmappedTokens,
// GEN-3b: Sls (2nd spotting locus) preserved here until Kevin's GEN-3a
// parser adopts it into the compact Genotype contract; tags + deaf too.
Sls = p.A.Genotype.Mapped8locus.TryGetValue("Sls", out var sls) ? sls : null,
p.A.Tags,
p.A.Deaf,
p.A.Zucht,
p.A.SourceFiles,
FarbschlagRaw = p.A.Farbschlag,
}),
});
}
// photos
foreach (var rel in p.A.Photos)
{
var src = Path.Combine(_sourceDir, rel.Replace('/', Path.DirectorySeparatorChar));
if (!File.Exists(src)) { photosMissing++; continue; }
photosAttached++;
if (execute)
{
Directory.CreateDirectory(_photoRoot);
var fileName = $"{Guid.NewGuid():N}{Path.GetExtension(src)}";
File.Copy(src, Path.Combine(_photoRoot, fileName), overwrite: true);
_db.GerbilPhotos.Add(new GerbilPhoto
{
Id = Guid.NewGuid(),
GerbilId = p.Gid,
FileName = fileName,
SortOrder = 0,
CreatedAt = DateTimeOffset.UtcNow,
});
}
}
}
if (execute) await _db.SaveChangesAsync();
// ---- back-link Wurfchronik litter parents by name (best effort) ----
if (execute)
{
foreach (var sl in litters)
{
if (!litterIdMap.TryGetValue(sl.Id, out var lid)) continue;
var litter = await _db.Litters.FirstOrDefaultAsync(l => l.Id == lid);
if (litter is null) continue;
if (createdAnimalByName.TryGetValue(Normalize(StripZucht(sl.SireName)), out var fId))
litter.FatherId = fId;
if (createdAnimalByName.TryGetValue(Normalize(StripZucht(sl.DamName)), out var mId))
litter.MotherId = mId;
}
await _db.SaveChangesAsync();
}
notes.Add("Quarantäne (kein Import): Konflikte + Stubs ohne Geburtsdatum + unsichere Wurf-Zuordnungen — warten auf die Prüfung durch die Züchterin.");
if (parentLinksAdded > 0)
notes.Add($"Stammbaum-Diagramm: {parentLinksAdded} Tiere über Eltern-Verknüpfung einem (abgeleiteten) Wurf zugeordnet ({derivedLitters} abgeleitete Würfe).");
notes.Add($"Bestand/Herkunft: {residentTotal} im Bestand (Clan Kleine Chaoten), {externalTotal} externe Ahnen ({flippedByParentRule} davon über die Eltern-Regel als Bestand erkannt).");
if (!execute) notes.Add("DRY-RUN: nichts gespeichert. /import/execute lädt die konfliktfreien Daten.");
return new ImportReport(
Executed: execute,
Litters: new LitterSummary(litters.Count, littersCreated, littersExisting, derivedLitters),
Animals: new AnimalSummary(
animals.Count, animalsCreated, linked, fbMatched, fbUnmatched, animalsExisting,
new QuarantineSummary(conflicts, stubs, dateOnly, ambiguous, conflicts + stubs),
parentLinksAdded),
Photos: new PhotoSummary(photosAttached, photosMissing),
Samples: samples,
Notes: notes,
Residency: new ResidencySummary(residentTotal, externalTotal, flippedByParentRule));
}
private T? Load<T>(string file)
{
var path = Path.Combine(_sourceDir, file);
if (!File.Exists(path)) return default;
using var fs = File.OpenRead(path);
return JsonSerializer.Deserialize<T>(fs, Json);
}
public static string ComposeGenotype(SourceGenotype g)
{
var tokens = LocusOrder.Select(locus =>
{
if (g.Mapped8locus.TryGetValue(locus, out var pair) && pair.Count == 2)
return StripCaret(pair[0]) + StripCaret(pair[1]);
return "??";
}).ToList();
// GEN-3a contract (Kevin): Sls is appended LAST and ONLY for carriers — the
// wild-type sl/sl is omitted so existing 8-locus strings stay unchanged. WP het
// renders as the trailing token "Slsl"; S(l)S(l) is lethal so never appears.
if (g.Mapped8locus.TryGetValue("Sls", out var sls) && sls.Count == 2
&& !(sls[0] == "sl" && sls[1] == "sl"))
{
tokens.Add(StripCaret(sls[0]) + StripCaret(sls[1]));
}
return string.Join(' ', tokens);
}
private static string StripCaret(string allele) => allele.Replace("^", "");
private static Gender InferGender(SourceAnimal a, HashSet<string> sires, HashSet<string> dams)
{
// Box colour (blue=male, white=female) is the authoritative breeder signal — prefer it
// over sire/dam name inference (PEDIGREE-LINK, Julian 2026-06-06).
if (string.Equals(a.Gender, "male", StringComparison.OrdinalIgnoreCase)) return Gender.male;
if (string.Equals(a.Gender, "female", StringComparison.OrdinalIgnoreCase)) return Gender.female;
var n = Normalize(StripZucht(a.Name));
bool isSire = sires.Contains(n), isDam = dams.Contains(n);
if (isSire && !isDam) return Gender.male;
if (isDam && !isSire) return Gender.female;
return Gender.unknown;
}
public static DateOnly? ParseDate(string s)
{
if (string.IsNullOrWhiteSpace(s)) return null;
var m = Regex.Match(s, @"(\d{1,2})\.(\d{1,2})\.(\d{2,4})");
if (!m.Success) return null;
int d = int.Parse(m.Groups[1].Value), mo = int.Parse(m.Groups[2].Value);
int y = int.Parse(m.Groups[3].Value);
if (y < 100) y += 2000;
try { return new DateOnly(y, mo, d); }
catch { return null; }
}
private static string StripZucht(string name)
{
var n = Regex.Replace(name ?? "", @"\[.*?\]", " ");
n = Regex.Replace(n, @"\s+(?:of|von\s+den|von\s+der|v\.\s?d\.|von)\s+.*$", "", RegexOptions.IgnoreCase);
return n.Trim();
}
private static string Normalize(string name)
{
var n = (name ?? "").ToLowerInvariant();
n = Regex.Replace(n, @"\bgen\.\b", " ");
n = Regex.Replace(n, @"[^a-z0-9äöüß]", "");
return n;
}
/// <summary>Dedup identity for parent resolution: normalized call-name + DOB.</summary>
private static string NameDobKey(string name, DateOnly? dob) =>
$"{Normalize(StripZucht(name))}|{dob:yyyy-MM-dd}";
/// <summary>Resolve a chart-position parentRef (by role) to a known gerbil id, or null.</summary>
private static Guid? ResolveParentGid(SourceAnimal a, string role, Dictionary<string, Guid> gidByNameDob)
{
var pr = a.ParentRefs.FirstOrDefault(p =>
string.Equals(p.RoleGuess, role, StringComparison.OrdinalIgnoreCase));
if (pr is null || string.IsNullOrWhiteSpace(pr.Name)) return null;
return gidByNameDob.TryGetValue(NameDobKey(pr.Name, ParseDate(pr.Dob)), out var id) ? id : null;
}
/// <summary>Per-animal plan computed before any write so synthesis can run in dry-run too.</summary>
private sealed record AnimalPlan(
SourceAnimal A, Guid Gid, bool Exists, Guid? WurfLitterId,
Guid? CurrentLitterId, Guid? ColorVarietyId, Gender Gender);
/// <summary>A litter synthesized from chart-position parentRefs (PEDIGREE-LINK).</summary>
private sealed record SynthLitter(Guid Id, Guid? Father, Guid? Mother, DateOnly? Date, string Confidence);
}
}