using System.Text.Json; using System.Text.RegularExpressions; using GerbilManagerWebAPI.Models; using Microsoft.EntityFrameworkCore; namespace GerbilManagerWebAPI.Import { /// /// FEAT-8c import loader. Consumes tools/import/output (animals.json + litters.json /// produced by extract.py) and loads conflict-free data into the database: /// litters first (Wurfchronik = authoritative), then animals matched onto them. /// /// Load policy: /// - Litters: all created (idempotent by ExternalRef = source litter id). /// - Animals: created when they have a DOB and are NOT in conflict. The birth-litter /// link is set only for HIGH-confidence matches (litterRef.confidence == "hoch"); /// date-only/ambiguous links are quarantined (animal loads with LitterId = null). /// - QUARANTINED (never loaded): conflicts + stubs (no DOB) — await the wife's review. /// Idempotent: re-running matches on ExternalRef and skips existing rows. /// Execute is gated by the endpoint; this service only acts when asked. /// public sealed class ImportService { private static readonly string[] LocusOrder = { "A", "C", "D", "E", "G", "P", "Sp", "Re" }; private static readonly JsonSerializerOptions Json = new() { PropertyNameCaseInsensitive = true, }; private const string ImportSourceTag = "FEAT-8 Stammbaum/Wurfchronik"; private readonly ApplicationContext _db; private readonly string _sourceDir; private readonly string _photoRoot; /// Test-friendly constructor with explicit paths. public ImportService(ApplicationContext db, string sourceDir, string photoRoot) { _db = db; _sourceDir = sourceDir; _photoRoot = photoRoot; } public ImportService(ApplicationContext db, IConfiguration config, IWebHostEnvironment env) : this(db, config["Import:SourcePath"] ?? Path.GetFullPath(Path.Combine(env.ContentRootPath, "..", "tools", "import", "output")), config["Photos:RootPath"] ?? Path.Combine(env.ContentRootPath, "photo-storage")) { } public async Task RunAsync(bool execute) { var notes = new List(); var samples = new List(); var animals = Load>("animals.json") ?? new(); var litters = Load>("litters.json") ?? new(); if (animals.Count == 0 && litters.Count == 0) notes.Add($"Keine Quelldaten gefunden in {_sourceDir} (animals.json/litters.json). extract.py zuerst ausführen."); // ---- categorise animals ---- var loadable = new List(); int conflicts = 0, stubs = 0, dateOnly = 0, ambiguous = 0; foreach (var a in animals) { if (a.Conflict) { conflicts++; continue; } if (string.IsNullOrEmpty(a.Dob)) { stubs++; continue; } loadable.Add(a); var conf = a.LitterRef?.Confidence; if (a.LitterRef?.Candidates is { Count: > 0 }) ambiguous++; else if (conf == "niedrig") dateOnly++; } // existing rows (idempotency). Gerbils carry ExternalRef; Litters have no such // column, so we key litter idempotency on the stable (Name + Date) pair instead. var existingGerbilExtRefs = await _db.Gerbils .Where(g => g.ExternalRef != null) .Select(g => g.ExternalRef!).ToListAsync(); var existingGerbilSet = existingGerbilExtRefs.ToHashSet(); var existingLitterKeys = await _db.Litters .Select(l => new { l.Name, l.Date }).ToListAsync(); var existingLitterKeySet = existingLitterKeys .Select(x => $"{x.Name}|{x.Date:yyyy-MM-dd}").ToHashSet(); // colour-variety name -> id (case-insensitive) var varieties = await _db.ColorVarieties.Select(v => new { v.Id, v.Name }).ToListAsync(); var varietyByName = varieties .GroupBy(v => v.Name.Trim().ToLowerInvariant()) .ToDictionary(g => g.Key, g => g.First().Id); // gender inference from litter roles (sire -> male, dam -> female; both -> unknown) var sireNames = litters.Select(l => Normalize(StripZucht(l.SireName))).Where(s => s.Length > 0).ToHashSet(); var damNames = litters.Select(l => Normalize(StripZucht(l.DamName))).Where(s => s.Length > 0).ToHashSet(); // ---- litters: create map source.id -> Litter (for high-confidence animal links) ---- int littersCreated = 0, littersExisting = 0; var litterIdMap = new Dictionary(); // source litter id -> Litter.Id foreach (var sl in litters) { var date = ParseDate(sl.Date); var name = $"Wurf {sl.LitterId}".Trim(); var key = $"{name}|{date:yyyy-MM-dd}"; if (existingLitterKeySet.Contains(key)) { littersExisting++; continue; } var id = Guid.NewGuid(); litterIdMap[sl.Id] = id; littersCreated++; if (execute && date is DateOnly d) { _db.Litters.Add(new Litter { Id = id, Name = name, Date = d, TotalBorn = sl.TotalBorn, Notes = string.IsNullOrWhiteSpace(sl.Note) ? null : sl.Note, PairingCode = string.IsNullOrWhiteSpace(sl.Zuchtnummer) ? null : sl.Zuchtnummer, }); } if (samples.Count < 8 && date is not null) samples.Add($"Wurf: {name} ({sl.Date}) — {sl.DamName} × {sl.SireName}"); } if (execute) await _db.SaveChangesAsync(); // ---- animals ---- int animalsCreated = 0, linked = 0, fbMatched = 0, fbUnmatched = 0, animalsExisting = 0; int photosAttached = 0, photosMissing = 0; var createdAnimalByName = new Dictionary(); // normalized name -> gerbil id (for litter back-link) foreach (var a in loadable) { if (existingGerbilSet.Contains(a.Id)) { animalsExisting++; continue; } animalsCreated++; Guid? litterId = null; if (a.LitterRef?.Confidence == "hoch" && a.LitterRef.Candidates is not { Count: > 0 } && litterIdMap.TryGetValue(a.LitterRef.LitterId, out var lid)) { litterId = lid; linked++; } Guid? colorVarietyId = null; var fbCandidates = new[] { a.Farbschlag }.Concat(a.FarbschlagVariants) .Where(s => !string.IsNullOrWhiteSpace(s)); foreach (var fb in fbCandidates) { if (varietyByName.TryGetValue(fb.Trim().ToLowerInvariant(), out var vid)) { colorVarietyId = vid; break; } } if (colorVarietyId is null) fbUnmatched++; else fbMatched++; var gender = InferGender(a, sireNames, damNames); var gid = Guid.NewGuid(); var norm = Normalize(StripZucht(a.Name)); if (norm.Length > 0) createdAnimalByName.TryAdd(norm, gid); if (samples.Count < 16) samples.Add($"Tier: {a.Name} (*{a.Dob}), Genotyp {ComposeGenotype(a.Genotype)}" + (litterId is not null ? ", Wurf-verknüpft" : "") + (colorVarietyId is not null ? $", Farbschlag „{a.Farbschlag}\"" : "")); if (execute) { _db.Gerbils.Add(new Gerbil { Id = gid, Name = a.Name, Gender = gender, Status = GerbilStatus.Active, DateOfBirth = ParseDate(a.Dob), DateOfDeath = ParseDate(a.Death), LitterId = litterId, ColorVarietyId = colorVarietyId, Genotype = ComposeGenotype(a.Genotype), ImportSource = ImportSourceTag, ExternalRef = a.Id, RawImportData = JsonSerializer.Serialize(new { a.Genotype.RawGenotype, a.Genotype.UnmappedTokens, a.Zucht, a.SourceFiles, FarbschlagRaw = a.Farbschlag, }), }); } // photos foreach (var rel in a.Photos) { var src = Path.Combine(_sourceDir, rel.Replace('/', Path.DirectorySeparatorChar)); if (!File.Exists(src)) { photosMissing++; continue; } photosAttached++; if (execute) { Directory.CreateDirectory(_photoRoot); var fileName = $"{Guid.NewGuid():N}{Path.GetExtension(src)}"; File.Copy(src, Path.Combine(_photoRoot, fileName), overwrite: true); _db.GerbilPhotos.Add(new GerbilPhoto { Id = Guid.NewGuid(), GerbilId = gid, FileName = fileName, SortOrder = 0, CreatedAt = DateTimeOffset.UtcNow, }); } } } if (execute) await _db.SaveChangesAsync(); // ---- back-link litter parents by name (best effort) ---- if (execute) { foreach (var sl in litters) { if (!litterIdMap.TryGetValue(sl.Id, out var lid)) continue; var litter = await _db.Litters.FirstOrDefaultAsync(l => l.Id == lid); if (litter is null) continue; if (createdAnimalByName.TryGetValue(Normalize(StripZucht(sl.SireName)), out var fId)) litter.FatherId = fId; if (createdAnimalByName.TryGetValue(Normalize(StripZucht(sl.DamName)), out var mId)) litter.MotherId = mId; } await _db.SaveChangesAsync(); } notes.Add("Quarantäne (kein Import): Konflikte + Stubs ohne Geburtsdatum + unsichere Wurf-Zuordnungen — warten auf die Prüfung durch die Züchterin."); if (!execute) notes.Add("DRY-RUN: nichts gespeichert. /import/execute lädt die konfliktfreien Daten."); return new ImportReport( Executed: execute, Litters: new LitterSummary(litters.Count, littersCreated, littersExisting), Animals: new AnimalSummary( animals.Count, animalsCreated, linked, fbMatched, fbUnmatched, animalsExisting, new QuarantineSummary(conflicts, stubs, dateOnly, ambiguous, conflicts + stubs)), Photos: new PhotoSummary(photosAttached, photosMissing), Samples: samples, Notes: notes); } private T? Load(string file) { var path = Path.Combine(_sourceDir, file); if (!File.Exists(path)) return default; using var fs = File.OpenRead(path); return JsonSerializer.Deserialize(fs, Json); } public static string ComposeGenotype(SourceGenotype g) { var tokens = LocusOrder.Select(locus => { if (g.Mapped8locus.TryGetValue(locus, out var pair) && pair.Count == 2) return StripCaret(pair[0]) + StripCaret(pair[1]); return "??"; }); return string.Join(' ', tokens); } private static string StripCaret(string allele) => allele.Replace("^", ""); private static Gender InferGender(SourceAnimal a, HashSet sires, HashSet dams) { var n = Normalize(StripZucht(a.Name)); bool isSire = sires.Contains(n), isDam = dams.Contains(n); if (isSire && !isDam) return Gender.male; if (isDam && !isSire) return Gender.female; return Gender.unknown; } public static DateOnly? ParseDate(string s) { if (string.IsNullOrWhiteSpace(s)) return null; var m = Regex.Match(s, @"(\d{1,2})\.(\d{1,2})\.(\d{2,4})"); if (!m.Success) return null; int d = int.Parse(m.Groups[1].Value), mo = int.Parse(m.Groups[2].Value); int y = int.Parse(m.Groups[3].Value); if (y < 100) y += 2000; try { return new DateOnly(y, mo, d); } catch { return null; } } private static string StripZucht(string name) { var n = Regex.Replace(name ?? "", @"\[.*?\]", " "); n = Regex.Replace(n, @"\s+(?:of|von\s+den|von\s+der|v\.\s?d\.|von)\s+.*$", "", RegexOptions.IgnoreCase); return n.Trim(); } private static string Normalize(string name) { var n = (name ?? "").ToLowerInvariant(); n = Regex.Replace(n, @"\bgen\.\b", " "); n = Regex.Replace(n, @"[^a-z0-9äöüß]", ""); return n; } } }