using System.Text.Json; using GerbilManagerWebAPI.Models; using GerbilManagerWebAPI.Services; using Microsoft.EntityFrameworkCore; namespace GerbilManagerWebAPI.Import { /// /// FEAT-8d docx loader. Consumes tools/import/output/docx_litters.json + /// docx_animals.json (produced by extract_docx.py) and enriches OR CREATES animals. /// /// Load policy (IDEMPOTENT NACHZUG after main WIPE+REIMPORT): /// Match order per docx row: /// 1. ExternalRef "docx-…" (previously created by this loader) → enrich (fill-NULL-only) /// 2. normalize(name)+litterDob in Gerbils (from main import) → enrich (fill-NULL-only) /// 3. No match → CREATE: abgegebene Jungtiere that never appeared in the pedigree charts. /// /// Created animals: Status=GivenAway (or Deceased), IsResident=false, /// OriginBreeder='Zucht der Kleinen Chaoten', ImportSource="docx", /// ExternalRef = stable "docx-{ws}-{normname}-{litterDob}[-N]" key (idempotent; /// -N ordinal suffix disambiguates same-name siblings in a litter). /// /// NEVER overwrites a manually-set non-null value (fill-NULL-only for all fields). /// /// Idempotent: running multiple times is safe. Re-run finds existing rows via ExternalRef. /// Execute wraps all writes in a single transaction via CreateExecutionStrategy() so that /// providers using EnableRetryOnFailure (e.g. NpgsqlRetryingExecutionStrategy) are /// compatible. The strategy lambda resets all mutable state at the top so it is safe /// to re-run on transient-failure retry. /// Execute is gated by the endpoint; this service only acts when asked. /// public sealed class ImportDocxService { private static readonly JsonSerializerOptions Json = new() { PropertyNameCaseInsensitive = true }; private readonly ApplicationContext _db; private readonly string _sourceDir; public ImportDocxService(ApplicationContext db, IConfiguration config, IWebHostEnvironment env) : this(db, config["Import:SourcePath"] ?? Path.GetFullPath(Path.Combine(env.ContentRootPath, "..", "tools", "import", "output"))) { } public ImportDocxService(ApplicationContext db, string sourceDir) { _db = db; _sourceDir = sourceDir; } public async Task RunAsync(bool execute) { var notes = new List(); var docxLitters = Load>("docx_litters.json") ?? new(); var docxAnimals = Load>("docx_animals.json") ?? new(); var pdfLitters = Load>("pdf_litters.json") ?? new(); var pdfAnimals = Load>("pdf_animals.json") ?? new(); int docxLitCount = docxLitters.Count; int docxAnimCount = docxAnimals.Count; int pdfLitCount = pdfLitters.Count; int pdfAnimCount = pdfAnimals.Count; docxLitters.AddRange(pdfLitters); docxAnimals.AddRange(pdfAnimals); if (docxLitters.Count == 0 && docxAnimals.Count == 0) { notes.Add($"Keine Quelldaten in {_sourceDir} (docx_litters.json/docx_animals.json oder pdf_litters.json/pdf_animals.json). " + "extract_docx.py zuerst ausführen oder PDF-VLM-Daten bereitstellen."); return new ImportDocxReport(false, 0, 0, 0, 0, 0, 0, 0, notes); } // Litter lookup by birth date (DayNumber) → list of matching DB litters. // NOTE: WsCode in docx is a litter-size fraction ("4/4", "/5") — NOT a PairingCode. // Date-only lookup with uniqueness guard avoids false links (only link when // exactly one DB litter falls within ±5 days of the docx litter DOB). var littersInDb = await _db.Litters .Select(l => new { l.Id, l.Date }) .ToListAsync(); var littersByDayNumber = littersInDb .GroupBy(l => l.Date.DayNumber) .ToDictionary(g => g.Key, g => g.ToList()); // normalize(name)+litterDob → Gerbil snapshot (main-import enrich path) var gerbilsInDb = await _db.Gerbils .Select(g => new { g.Id, g.Name, g.DateOfBirth, g.LitterId, g.ReceiverContactId, g.GoHomeDate, g.DateOfDeath, g.CauseOfDeath, g.ExternalRef }) .ToListAsync(); var gerbilByKey = gerbilsInDb .Where(g => g.DateOfBirth is not null) .GroupBy(g => NameDobKey(g.Name, g.DateOfBirth!.Value)) .ToDictionary(g => g.Key, g => g.ToList()); // ExternalRef → snapshot for previously docx-created animals (idempotency across runs) var docxExternalRefs = gerbilsInDb .Where(g => g.ExternalRef?.StartsWith("docx-") == true) .ToDictionary(g => g.ExternalRef!, g => new { g.Id, g.LitterId, g.GoHomeDate, g.DateOfDeath, g.ReceiverContactId }); // Contact lookup: normalized name → existing Contact.Id var contactsInDb = await _db.Contacts .Select(c => new { c.Id, c.Name }) .ToListAsync(); var contactByNorm = contactsInDb .GroupBy(c => NormalizeName(c.Name)) .ToDictionary(g => g.Key, g => g.First().Id); // Snapshot of DB contacts before any writes. // Used to reset contactByNorm on strategy retry (rolled-back contacts vanish from DB // but would remain in the in-memory dict without this reset). var contactByNormBase = new Dictionary(contactByNorm); // ColorVariety lookup: normalized name → Id (for CREATE path Farbschlag matching) var colorVarietyByName = (await _db.ColorVarieties .Select(cv => new { cv.Id, cv.Name }) .ToListAsync()) .GroupBy(cv => NormalizeName(cv.Name)) .ToDictionary(g => g.Key, g => g.First().Id); int animalsCreated = 0, litterLinked = 0, goHomeFilled = 0, deathFilled = 0; int ownerLinked = 0, ownerCreated = 0, skipped = 0; // Ordinal counter for collision-free ExternalRef within this batch. var externalRefOrdinals = new Dictionary(); // Belt-and-suspenders: guard against adding the same ExternalRef twice in one run. var batchRefs = new HashSet(); // Inner loop — shared by dry-run and execute paths. // All local variables above are captured by reference (C# closure), so the strategy // lambda can reset them before each retry and RunLoopAsync sees the fresh state. async Task RunLoopAsync() { foreach (var da in docxAnimals) { if (string.IsNullOrWhiteSpace(da.Name)) { skipped++; continue; } // Collision-free ExternalRef: ordinal suffix for same-name siblings var baseRef = DocxExternalRefBase(da); externalRefOrdinals.TryGetValue(baseRef, out var ord); ord++; externalRefOrdinals[baseRef] = ord; var externalRef = ord == 1 ? baseRef : $"{baseRef}-{ord}"; // Resolve litter: date ±5 days, unambiguous (exactly one candidate) Guid? litterId = null; if (!string.IsNullOrWhiteSpace(da.LitterDob)) { var litterDob = ParseDate(da.LitterDob); if (litterDob is not null) { var candidates = new List(); for (int delta = -5; delta <= 5; delta++) { if (littersByDayNumber.TryGetValue(litterDob.Value.DayNumber + delta, out var cl)) candidates.AddRange(cl.Select(l => l.Id)); } if (candidates.Count == 1) litterId = candidates[0]; // If 0 or >1 candidates: no link (avoid false links) } } // Animal DOB = litter birth date (docx has no per-animal DOB) var animalDob = litterId is not null ? (await _db.Litters.Where(l => l.Id == litterId).Select(l => (DateOnly?)l.Date).FirstOrDefaultAsync()) : ParseDate(da.LitterDob); if (animalDob is null) { skipped++; continue; } var goHomeDate = ParseDate(da.AbgabeDate); var deathDate = ParseDate(da.DeathDate); // Resolve receiver contact (lookup-or-create; shared by all paths) Guid? receiverId = null; if (!string.IsNullOrWhiteSpace(da.Owner)) { var normOwner = NormalizeName(da.Owner); if (contactByNorm.TryGetValue(normOwner, out var existingContactId)) { receiverId = existingContactId; ownerLinked++; } else { ownerCreated++; if (execute) { var newContact = new Contact { Id = Guid.NewGuid(), Name = da.Owner.Trim() }; _db.Contacts.Add(newContact); await _db.SaveChangesAsync(); // flush within the outer tx receiverId = newContact.Id; contactByNorm[normOwner] = receiverId.Value; } } } // ── PATH 1: previously docx-created animal (idempotent re-run) ────────── if (docxExternalRefs.TryGetValue(externalRef, out var prevSnap)) { bool willLink = litterId is not null && prevSnap.LitterId is null; bool willHome = goHomeDate is not null && prevSnap.GoHomeDate is null; bool willDeath = deathDate is not null && prevSnap.DateOfDeath is null; if (willLink) litterLinked++; if (willHome) goHomeFilled++; if (willDeath) deathFilled++; if (execute) { var row = await _db.Gerbils.FirstOrDefaultAsync(g => g.Id == prevSnap.Id); if (row is null) continue; if (willLink) row.LitterId = litterId; if (receiverId is not null && row.ReceiverContactId is null) row.ReceiverContactId = receiverId; if (willHome) row.GoHomeDate = goHomeDate; if (willDeath) { row.DateOfDeath = deathDate; if (!string.IsNullOrWhiteSpace(da.DeathCause) && row.CauseOfDeath is null) row.CauseOfDeath = da.DeathCause.Trim(); } } continue; } // ── PATH 2: main-import animal matched by name+dob ─────────────────────── var nameKey = NameDobKey(da.Name, animalDob.Value); if (gerbilByKey.TryGetValue(nameKey, out var gerbilCands)) { var gerbilSnap = gerbilCands.FirstOrDefault(g => g.LitterId == null) ?? gerbilCands.First(); bool willLink = litterId is not null && gerbilSnap.LitterId is null; bool willHome = goHomeDate is not null && gerbilSnap.GoHomeDate is null; bool willDeath = deathDate is not null && gerbilSnap.DateOfDeath is null; if (willLink) litterLinked++; if (willHome) goHomeFilled++; if (willDeath) deathFilled++; if (execute) { var row = await _db.Gerbils.FirstOrDefaultAsync(g => g.Id == gerbilSnap.Id); if (row is null) continue; if (willLink) row.LitterId = litterId; if (receiverId is not null && row.ReceiverContactId is null) row.ReceiverContactId = receiverId; if (willHome) row.GoHomeDate = goHomeDate; if (willDeath) { row.DateOfDeath = deathDate; if (!string.IsNullOrWhiteSpace(da.DeathCause) && row.CauseOfDeath is null) row.CauseOfDeath = da.DeathCause.Trim(); } } continue; } // ── PATH 3: no match → CREATE ──────────────────────────────────────────── // Belt-and-suspenders: ordinal should ensure uniqueness, but guard anyway if (!batchRefs.Add(externalRef)) { skipped++; continue; } animalsCreated++; if (litterId is not null) litterLinked++; if (goHomeDate is not null) goHomeFilled++; if (deathDate is not null) deathFilled++; if (execute) { colorVarietyByName.TryGetValue(NormalizeName(da.Farbschlag ?? ""), out var cvId); _db.Gerbils.Add(new Gerbil { Id = Guid.NewGuid(), Name = da.Name.Trim(), DateOfBirth = animalDob, Gender = ParseGender(da.Gender), Status = GerbilStatusService.Derive(GerbilStatus.GivenAway, animalDob, deathDate, isAbgegeben: receiverId is not null, DateOnly.FromDateTime(DateTime.UtcNow)), LitterId = litterId, ReceiverContactId = receiverId, GoHomeDate = goHomeDate, DateOfDeath = deathDate, CauseOfDeath = string.IsNullOrWhiteSpace(da.DeathCause) ? null : da.DeathCause.Trim(), ColorVarietyId = cvId == default ? null : cvId, OriginBreeder = "Zucht der kleinen Chaoten", IsResident = false, ImportSource = "docx", ExternalRef = externalRef, }); } } } if (!execute) { // Dry-run: just count, no writes, no transaction needed. await RunLoopAsync(); } else { // Execute: wrap the entire transaction in the execution strategy so that providers // with EnableRetryOnFailure (NpgsqlRetryingExecutionStrategy) are compatible. // The lambda resets all mutable state at the top so retries start clean. var strategy = _db.Database.CreateExecutionStrategy(); await strategy.ExecuteAsync(async () => { // Reset mutable state — idempotent on strategy retry _db.ChangeTracker.Clear(); externalRefOrdinals.Clear(); batchRefs.Clear(); animalsCreated = 0; litterLinked = 0; goHomeFilled = 0; deathFilled = 0; ownerLinked = 0; ownerCreated = 0; skipped = 0; // Rebuild from DB snapshot: contacts added in a failed attempt were rolled back contactByNorm = new Dictionary(contactByNormBase); await using var tx = await _db.Database.BeginTransactionAsync(); await RunLoopAsync(); if ((animalsCreated + litterLinked + goHomeFilled + deathFilled + ownerCreated) > 0) await _db.SaveChangesAsync(); await tx.CommitAsync(); }); } notes.Add($"Quelle: {docxLitCount} Würfe / {docxAnimCount} Tiere aus DOCX. {pdfLitCount} Würfe / {pdfAnimCount} Tiere aus PDF."); notes.Add($"Neu angelegt: {animalsCreated} Jungtiere (abgegeben, nicht in Stammbäumen)."); notes.Add($"Litter-Links: {litterLinked} Tiere einem Wurf zugeordnet (DOB-Match ±5 Tage, eindeutig)."); notes.Add($"Abnehmer: {ownerLinked} bestehende Kontakte verknüpft, {ownerCreated} neue Kontakte angelegt."); notes.Add($"GoHomeDate: {goHomeFilled} Abgabe-Daten nachgetragen."); notes.Add($"Tod-Datum: {deathFilled} Todesdaten nachgetragen."); notes.Add($"Übersprungen: {skipped} Zeilen (kein Name oder kein Datum)."); if (!execute) notes.Add("DRY-RUN: nichts gespeichert. /import/docx/execute schreibt die Änderungen."); return new ImportDocxReport(execute, animalsCreated, litterLinked, ownerLinked + ownerCreated, goHomeFilled, deathFilled, ownerCreated, skipped, notes); } private T? Load(string file) { var path = Path.Combine(_sourceDir, file); if (!File.Exists(path)) return default; using var fs = File.OpenRead(path); return JsonSerializer.Deserialize(fs, Json); } private static DateOnly? ParseDate(string? s) { if (string.IsNullOrWhiteSpace(s)) return null; var m = System.Text.RegularExpressions.Regex.Match(s, @"(\d{1,2})\.(\d{1,2})\.(\d{2,4})"); if (!m.Success) return null; int d = int.Parse(m.Groups[1].Value), mo = int.Parse(m.Groups[2].Value); int y = int.Parse(m.Groups[3].Value); if (y < 100) y += 2000; try { return new DateOnly(y, mo, d); } catch { return null; } } private static string NameDobKey(string name, DateOnly dob) { var n = System.Text.RegularExpressions.Regex.Replace( (name ?? "").ToLowerInvariant(), @"[^a-z0-9äöüß]", ""); return $"{n}|{dob:yyyy-MM-dd}"; } private static string NormalizeName(string name) { var n = (name ?? "").ToLowerInvariant(); n = System.Text.RegularExpressions.Regex.Replace(n, @"\s+", " ").Trim(); return n; } /// /// Base ExternalRef key (before ordinal disambiguation). Two animals sharing the same /// ws+name+litterDob get this same base; the caller appends -2, -3 … for duplicates. /// internal static string DocxExternalRefBase(DocxAnimal da) { var ws = (da.WsCode ?? "").Replace(" ", "").ToLowerInvariant(); var name = System.Text.RegularExpressions.Regex.Replace( (da.Name ?? "").ToLowerInvariant(), @"[^a-z0-9äöüß]", ""); return $"docx-{ws}-{name}-{da.LitterDob}"; } private static Gender ParseGender(string? s) { if (string.IsNullOrWhiteSpace(s)) return Gender.unknown; var lower = s.ToLowerInvariant(); if (lower == "male" || lower == "m" || lower.Contains("männ")) return Gender.male; if (lower == "female" || lower == "f" || lower == "w" || lower.Contains("weibl")) return Gender.female; return Gender.unknown; } } // ---- Source shapes (from extract_docx.py output) ---- public sealed class DocxLitter { public string LitterId { get; set; } = ""; public string Dob { get; set; } = ""; public string MotherName { get; set; } = ""; public string FatherName { get; set; } = ""; public string WsCode { get; set; } = ""; public string Note { get; set; } = ""; } public sealed class DocxAnimal { public string WsCode { get; set; } = ""; public string LitterDob { get; set; } = ""; public string Name { get; set; } = ""; public string Farbschlag { get; set; } = ""; public string Gender { get; set; } = ""; public string Owner { get; set; } = ""; public string AbgabeDate { get; set; } = ""; public string AbgabeWeight { get; set; } = ""; public string DeathDate { get; set; } = ""; public string DeathCause { get; set; } = ""; public string PartnerName { get; set; } = ""; public string PartnerDob { get; set; } = ""; } // ---- Report ---- public sealed record ImportDocxReport( bool Executed, int Created, int LitterLinked, int OwnerLinked, int GoHomeFilled, int DeathFilled, int ContactsCreated, int Skipped, IReadOnlyList Notes); }