using System.Text.Json; using System.Text.Json.Serialization; using GerbilManagerWebAPI.Models; using Microsoft.EntityFrameworkCore; using GerbilManagerWebAPI.Services; namespace GerbilManagerWebAPI.Import { public sealed class IngestResolvedService { private readonly ApplicationContext _db; private readonly string _resolvedJsonPath; private readonly string _sourceDir; private readonly string _photoRoot; private static readonly JsonSerializerOptions DiffJson = new(JsonSerializerDefaults.Web); public IngestResolvedService(ApplicationContext db, IConfiguration config, IWebHostEnvironment? env) { _db = db; _sourceDir = ResolveSourceDir(config, env); _resolvedJsonPath = Path.Combine(_sourceDir, "resolved_import.json"); _photoRoot = config["Photos:RootPath"] ?? Path.Combine(env?.ContentRootPath ?? Directory.GetCurrentDirectory(), "photo-storage"); } /// /// Verzeichnis, in dem der Ingest resolved_import.json (+ referenzierte Fotos) erwartet. /// Öffentlich, damit der Upload-Endpoint die hochgeladene Datei an genau denselben Ort stagen kann /// (siehe POST /import/ingest-resolved/upload) — so lässt sich der Ingest gegen Prod fahren, /// ohne die Datei per SSH/docker cp in den Container zu kopieren. /// public static string ResolveSourceDir(IConfiguration config, IWebHostEnvironment? env) { var contentRoot = env?.ContentRootPath ?? Directory.GetCurrentDirectory(); return config["Import:SourcePath"] ?? Path.GetFullPath(Path.Combine(contentRoot, "..", "tools", "import", "output")); } public async Task RunAsync() { if (!File.Exists(_resolvedJsonPath)) { return $"Error: Resolved import file not found at {_resolvedJsonPath}. Run resolve_import.py first."; } string jsonContent = await File.ReadAllTextAsync(_resolvedJsonPath); var jsonOptions = new JsonSerializerOptions { PropertyNameCaseInsensitive = true }; jsonOptions.Converters.Add(new JsonStringEnumConverter()); var data = JsonSerializer.Deserialize(jsonContent, jsonOptions); if (data == null) { return "Error: Failed to deserialize resolved_import.json."; } // ── UPSERT-based ingest (no total wipe) ────────────────────────────────────── // The ingest NO LONGER wipes everything. Imported rows are matched by their // (deterministic) id and updated in place; manually-created rows (IsManual=true) and // user-entered sub-records (weights/health/photos) are never deleted, so the breeder's // work survives the re-import. Only STALE imported rows (IsManual=false, absent from the // new payload) are removed — after nulling any manual references to them (with a warning). // Finally, hand-curated GerbilOverrides are re-applied on top (the "freeze"). var payloadContactIds = data.Contacts.Select(c => c.Id).ToHashSet(); var payloadLitterIds = data.Litters.Select(l => l.Id).ToHashSet(); var payloadGerbilIds = data.Gerbils.Select(g => g.Id).ToHashSet(); var warnings = new List(); // 1. Upsert contacts (preserve manual rows + their IsManual flag). var existingContacts = await _db.Contacts.ToDictionaryAsync(c => c.Id); int contactsAdded = 0, contactsUpdated = 0; foreach (var c in data.Contacts) { if (existingContacts.TryGetValue(c.Id, out var ec)) { ec.Name = c.Name; ec.Email = c.Email; ec.Phone = c.Phone; ec.Address = c.Address; ec.Notes = c.Notes; ec.IsBreeder = c.IsBreeder; ec.IsReceiver = c.IsReceiver; ec.NameSuffix = c.NameSuffix; ec.Provenance = c.Provenance; contactsUpdated++; } else { c.IsManual = false; _db.Contacts.Add(c); contactsAdded++; } } await _db.SaveChangesAsync(); // 2. Upsert litters (parent FKs reset to null; set in pass 2 to avoid ordering issues). var existingLitters = await _db.Litters.ToDictionaryAsync(l => l.Id); foreach (var l in data.Litters) { if (existingLitters.TryGetValue(l.Id, out var el)) { el.Name = l.Name; el.Date = l.Date; el.TotalBorn = l.TotalBorn; el.DeathsWithin8Weeks = l.DeathsWithin8Weeks; el.Stillborn = l.Stillborn; el.ExpectedGoHomeDate = l.ExpectedGoHomeDate; el.Notes = l.Notes; el.PairingCode = l.PairingCode; el.ExternalRef = l.ExternalRef; el.LitterLetter = l.LitterLetter; el.Provenance = l.Provenance; el.ShowInChronicle = l.ShowInChronicle; el.FatherId = null; el.MotherId = null; } else { _db.Litters.Add(new Litter { Id = l.Id, Name = l.Name, Date = l.Date, TotalBorn = l.TotalBorn, DeathsWithin8Weeks = l.DeathsWithin8Weeks, Stillborn = l.Stillborn, FatherId = null, MotherId = null, ExpectedGoHomeDate = l.ExpectedGoHomeDate, Notes = l.Notes, PairingCode = l.PairingCode, ExternalRef = l.ExternalRef, LitterLetter = l.LitterLetter, Provenance = l.Provenance, ShowInChronicle = l.ShowInChronicle, IsManual = false, }); } } await _db.SaveChangesAsync(); // 3. Upsert gerbils (LitterId reset to null; set in pass 2). Manual rows keep IsManual=true // and are never matched here (their random ids never collide with deterministic import ids). var existingGerbils = await _db.Gerbils.ToDictionaryAsync(g => g.Id); foreach (var g in data.Gerbils) { Gerbil target; if (existingGerbils.TryGetValue(g.Id, out var eg)) { eg.Name = g.Name; eg.Gender = g.Gender; eg.Status = g.Status; eg.OriginContactId = g.OriginContactId; eg.ReceiverContactId = g.ReceiverContactId; eg.EnclosureId = g.EnclosureId; eg.ColorVarietyId = g.ColorVarietyId; eg.DateOfBirth = g.DateOfBirth; eg.DateOfDeath = g.DateOfDeath; eg.CauseOfDeath = g.CauseOfDeath; eg.GoHomeDate = g.GoHomeDate; eg.Genotype = g.Genotype; eg.Notes = g.Notes; eg.ImportSource = g.ImportSource; eg.ExternalRef = g.ExternalRef; eg.RawImportData = g.RawImportData; eg.Provenance = g.Provenance; eg.OriginBreeder = g.OriginBreeder; eg.CharacterTraits = g.CharacterTraits; eg.CharacterNote = g.CharacterNote; eg.IsDeaf = g.IsDeaf; eg.IsResident = g.IsResident; eg.LitterId = null; eg.BirthOrder = g.BirthOrder; target = eg; } else { var ng = new Gerbil { Id = g.Id, Name = g.Name, Gender = g.Gender, Status = g.Status, LitterId = null, OriginContactId = g.OriginContactId, ReceiverContactId = g.ReceiverContactId, EnclosureId = g.EnclosureId, ColorVarietyId = g.ColorVarietyId, DateOfBirth = g.DateOfBirth, DateOfDeath = g.DateOfDeath, CauseOfDeath = g.CauseOfDeath, GoHomeDate = g.GoHomeDate, Genotype = g.Genotype, Notes = g.Notes, ImportSource = g.ImportSource, ExternalRef = g.ExternalRef, RawImportData = g.RawImportData, Provenance = g.Provenance, OriginBreeder = g.OriginBreeder, NameSearch = g.NameSearch, CharacterTraits = g.CharacterTraits, CharacterNote = g.CharacterNote, IsDeaf = g.IsDeaf, IsResident = g.IsResident, IsManual = false, BirthOrder = g.BirthOrder, }; _db.Gerbils.Add(ng); target = ng; } GerbilStatusService.Apply(target, DateOnly.FromDateTime(DateTime.UtcNow)); } await _db.SaveChangesAsync(); // 4. Pass 2: set the payload FK relationships (only for payload rows; manual rows untouched). var littersInDb = await _db.Litters.ToDictionaryAsync(l => l.Id); foreach (var l in data.Litters) if (littersInDb.TryGetValue(l.Id, out var dl)) { dl.FatherId = l.FatherId; dl.MotherId = l.MotherId; } var gerbilsInDb = await _db.Gerbils.ToDictionaryAsync(g => g.Id); foreach (var g in data.Gerbils) if (gerbilsInDb.TryGetValue(g.Id, out var dg)) dg.LitterId = g.LitterId; await _db.SaveChangesAsync(); // 5. Remove STALE imported rows (IsManual=false, no longer in the payload). Null any // dangling references from surviving rows first; warn when a MANUAL row is affected. var allGerbils = await _db.Gerbils.ToListAsync(); var allLitters = await _db.Litters.ToListAsync(); var allContacts = await _db.Contacts.ToListAsync(); var gerbilNameById = allGerbils.ToDictionary(x => x.Id, x => x.Name); var litterNameById = allLitters.ToDictionary(x => x.Id, x => x.Name); var contactNameById = allContacts.ToDictionary(x => x.Id, x => x.Name); var staleGerbilIds = allGerbils.Where(g => !g.IsManual && !payloadGerbilIds.Contains(g.Id)).Select(g => g.Id).ToHashSet(); var staleLitterIds = allLitters.Where(l => !l.IsManual && !payloadLitterIds.Contains(l.Id)).Select(l => l.Id).ToHashSet(); var staleContactIds = allContacts.Where(c => !c.IsManual && !payloadContactIds.Contains(c.Id)).Select(c => c.Id).ToHashSet(); if (staleGerbilIds.Count > 0) { foreach (var l in allLitters) { if (l.FatherId is { } f && staleGerbilIds.Contains(f)) { if (l.IsManual) warnings.Add($"Manueller Wurf „{l.Name}\" verwies auf importiertes Elterntier „{gerbilNameById.GetValueOrDefault(f, "?")}\" (Vater), das im Import nicht mehr existiert — Elternteil entfernt."); l.FatherId = null; } if (l.MotherId is { } m && staleGerbilIds.Contains(m)) { if (l.IsManual) warnings.Add($"Manueller Wurf „{l.Name}\" verwies auf importiertes Elterntier „{gerbilNameById.GetValueOrDefault(m, "?")}\" (Mutter), das im Import nicht mehr existiert — Elternteil entfernt."); l.MotherId = null; } } } if (staleContactIds.Count > 0) { foreach (var g in allGerbils) { if (g.OriginContactId is { } o && staleContactIds.Contains(o)) { if (g.IsManual) warnings.Add($"Manuelles Tier „{g.Name}\" verwies auf Kontakt „{contactNameById.GetValueOrDefault(o, "?")}\" (Herkunft), der im Import nicht mehr existiert — Zuordnung entfernt."); g.OriginContactId = null; } if (g.ReceiverContactId is { } r && staleContactIds.Contains(r)) { if (g.IsManual) warnings.Add($"Manuelles Tier „{g.Name}\" verwies auf Kontakt „{contactNameById.GetValueOrDefault(r, "?")}\" (Abnehmer), der im Import nicht mehr existiert — Zuordnung entfernt."); g.ReceiverContactId = null; } } } if (staleLitterIds.Count > 0) { // Gerbil.LitterId FK is SetNull on delete → nulled automatically when the litter is // removed; we only surface a warning for affected MANUAL animals. foreach (var g in allGerbils) if (g.IsManual && g.LitterId is { } lid && staleLitterIds.Contains(lid)) warnings.Add($"Manuelles Tier „{g.Name}\" war im Wurf „{litterNameById.GetValueOrDefault(lid, "?")}\" geboren, der im Import nicht mehr existiert — Wurf-Zuordnung entfernt."); } await _db.SaveChangesAsync(); int gerbilsDeleted = staleGerbilIds.Count, littersDeleted = staleLitterIds.Count, contactsDeleted = staleContactIds.Count; if (staleGerbilIds.Count > 0) _db.Gerbils.RemoveRange(allGerbils.Where(g => staleGerbilIds.Contains(g.Id))); if (staleLitterIds.Count > 0) _db.Litters.RemoveRange(allLitters.Where(l => staleLitterIds.Contains(l.Id))); if (staleContactIds.Count > 0) _db.Contacts.RemoveRange(allContacts.Where(c => staleContactIds.Contains(c.Id))); await _db.SaveChangesAsync(); // 6. Photos: replace photos of IMPORTED gerbils (manual gerbils' photos survive). Copy files. var manualGerbilIds = allGerbils.Where(g => g.IsManual).Select(g => g.Id).ToHashSet(); var allPhotos = await _db.GerbilPhotos.ToListAsync(); _db.GerbilPhotos.RemoveRange(allPhotos.Where(p => !manualGerbilIds.Contains(p.GerbilId))); await _db.SaveChangesAsync(); Directory.CreateDirectory(_photoRoot); foreach (var p in data.GerbilPhotos) { if (!string.IsNullOrEmpty(p.SourcePath)) { var relPath = p.SourcePath.Replace('/', Path.DirectorySeparatorChar).Replace('\\', Path.DirectorySeparatorChar); var src = Path.Combine(_sourceDir, relPath); if (File.Exists(src)) { var dest = Path.Combine(_photoRoot, p.FileName); try { File.Copy(src, dest, overwrite: true); } catch (Exception ex) { Console.WriteLine($"Warning: Failed to copy photo {src} to {dest}: {ex.Message}"); } } else { Console.WriteLine($"Warning: Source photo file not found at {src}"); } } _db.GerbilPhotos.Add(new GerbilPhoto { Id = p.Id, GerbilId = p.GerbilId, FileName = p.FileName, Caption = p.Caption, SortOrder = p.SortOrder, CreatedAt = DateTimeOffset.UtcNow, }); } await _db.SaveChangesAsync(); // 7. Imported Abgabeverträge: recreate from payload (unchanged behaviour — the imported set // is part of the payload). Manual sale-contract protection is a separate follow-up. var scToRemove = await _db.SaleContracts.ToListAsync(); _db.SaleContracts.RemoveRange(scToRemove); await _db.SaveChangesAsync(); int contractsAdded = 0, contractAnimalsSkipped = 0; if (data.SaleContracts.Count > 0) { var contactIds = await _db.Contacts.Select(c => c.Id).ToHashSetAsync(); var gerbilIds = await _db.Gerbils.Select(g => g.Id).ToHashSetAsync(); var seenContractIds = new HashSet(); foreach (var sc in data.SaleContracts) { if (!seenContractIds.Add(sc.Id)) continue; if (!contactIds.Contains(sc.ContactId)) continue; var animals = new List(); foreach (var gid in (sc.Animals ?? new List()).Distinct()) { if (!gerbilIds.Contains(gid)) { contractAnimalsSkipped++; continue; } animals.Add(new SaleContractAnimal { SaleContractId = sc.Id, GerbilId = gid }); } _db.SaleContracts.Add(new SaleContract { Id = sc.Id, ContactId = sc.ContactId, Price = sc.Price, HandoverDate = sc.HandoverDate, ContractDate = sc.ContractDate, FileName = sc.FileName, CreatedAt = DateTimeOffset.UtcNow, Animals = animals, }); contractsAdded++; } await _db.SaveChangesAsync(); } // 8. FREEZE: capture the raw-import snapshot for verified overrides (BEFORE applying the // freeze), then re-apply every override on top of the fresh import so the breeder's // curated values win. Overrides survive re-ingest by being their own (FK-free) table. var overrides = await _db.GerbilOverrides.ToListAsync(); foreach (var ov in overrides.Where(o => o.IsVerified)) { var raw = await GerbilSnapshotService.BuildSnapshotAsync(_db, ov.GerbilId); if (raw is null) continue; // verified animal vanished from import → surfaced as "missing" in the UI ov.LastImportSnapshotJson = GerbilSnapshotService.SerializeSnapshot(raw); var golden = GerbilSnapshotService.DeserializeSnapshot(ov.SnapshotJson); var diff = GerbilSnapshotService.Diff(golden, raw); ov.LastImportDiffJson = diff.Count == 0 ? null : JsonSerializer.Serialize(diff, DiffJson); } var gerbilsById = await _db.Gerbils.ToDictionaryAsync(g => g.Id); int overridesApplied = 0; foreach (var ov in overrides) { if (gerbilsById.TryGetValue(ov.GerbilId, out var gg)) { GerbilSnapshotService.ApplyOverride(gg, ov.OverrideJson); // Der Override kann Todesdatum/Abgabe/Geburtsdatum setzen; Status neu ableiten, // damit ein verstorbenes/abgegebenes Tier nicht als 'Zucht' hängen bleibt // (Ticket 37ab228a "Gaida"). Status/Gehege sind selbst NICHT eingefroren. GerbilStatusService.Apply(gg, DateOnly.FromDateTime(DateTime.UtcNow)); overridesApplied++; } } await _db.SaveChangesAsync(); var warnPart = warnings.Count > 0 ? $" {warnings.Count} warning(s): {string.Join(" | ", warnings)}" : ""; return $"Ingestion successful! Contacts +{contactsAdded}/~{contactsUpdated} (−{contactsDeleted} stale), " + $"{data.Litters.Count} litters (−{littersDeleted} stale), {data.Gerbils.Count} gerbils (−{gerbilsDeleted} stale), " + $"{data.GerbilPhotos.Count} photos, {contractsAdded} sale contracts ({contractAnimalsSkipped} animal links skipped), " + $"{overridesApplied} override(s) applied (freeze).{warnPart}"; } } public class ResolvedImportData { public List Contacts { get; set; } = new(); public List Litters { get; set; } = new(); public List Gerbils { get; set; } = new(); public List GerbilPhotos { get; set; } = new(); public List SaleContracts { get; set; } = new(); } /// Eine importierte Abgabevertrag-Zeile aus resolved_import.json. /// Animals sind die zugeordneten Gerbil-Ids (Join-Zeilen werden beim /// Ingest gebaut). Die .docx liegt NICHT im Vertrags-Dateiroot — der /// Word-Download liefert 404, das PDF wird aus den Daten neu erzeugt. public class ResolvedSaleContract { public Guid Id { get; set; } public Guid ContactId { get; set; } public decimal Price { get; set; } public DateOnly HandoverDate { get; set; } public DateOnly ContractDate { get; set; } public required string FileName { get; set; } public List Animals { get; set; } = new(); } public class ResolvedPhoto { public Guid Id { get; set; } public Guid GerbilId { get; set; } public required string FileName { get; set; } public string? Caption { get; set; } public int SortOrder { get; set; } [JsonPropertyName("_source_path")] public string? SourcePath { get; set; } } }