feat(deploy): TrueNAS Custom-App + Auto-Deploy, plus aufgelaufene Arbeit
Deployment: - custom-app.compose.yaml: self-contained Compose fuer TrueNAS "Custom App" (absolute Host-Bind-Pfade, postgres:18, pull_policy always, Port 8090) - scripts/truenas-deploy.sh: Host-Skript create/redeploy via midclt (App bleibt unter Apps sichtbar) inkl. Image-Pull + Health-Check - ci.yml Deploy-Job: laeuft auf ubuntu-latest-Runner, kopiert Deploy-Dateien per SSH auf den NAS-Host und triggert truenas-deploy.sh (statt runs-on goldeye) - compose.yaml/.env.example: postgres:18 (Locale-Match zur Quell-DB), Port 8090 - .gitignore: .agents/, tools/rag/, deploy/truenas/.env (Secrets/Scratch) Aufgelaufene Feature-Arbeit (verified/Freeze, Migrationen, Import-Triage): - GerbilOverride/VerifiedGerbil-Endpoints + GerbilSnapshotService + Tests - EF-Migrationen (ShowInChronicle, Stillborn, BirthOrder, ManualFlag, DSGVO) - Frontend VerifizierteTierePage + verified-API + e2e-Spec - diverse Import-/Triage-Skripte und -Tests Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
307
GerbilManagerWebAPI/Import/GerbilSnapshotService.cs
Normal file
307
GerbilManagerWebAPI/Import/GerbilSnapshotService.cs
Normal file
@@ -0,0 +1,307 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using GerbilManagerWebAPI.Models;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace GerbilManagerWebAPI.Import
|
||||
{
|
||||
// ── Canonical, human-readable "Akte" snapshot ────────────────────────────
|
||||
// Names (not ids) are stored so the golden state stays human-verifiable and robust to
|
||||
// deterministic-id reseeds. Used for the drift report, the export fixture and the
|
||||
// regression test. Lists are sorted canonically by the builder so equality is stable.
|
||||
|
||||
public sealed record GerbilSnapshot(
|
||||
SnapshotSelf Self,
|
||||
SnapshotParent? Father,
|
||||
SnapshotParent? Mother,
|
||||
List<SnapshotLitter> Litters);
|
||||
|
||||
public sealed record SnapshotSelf(
|
||||
string Name, string Gender, string? DateOfBirth, string? DateOfDeath, string? CauseOfDeath,
|
||||
string? GoHomeDate, string? Genotype, string? SpottingType, string? ColorName,
|
||||
bool IsResident, bool? IsDeaf, bool IsCastrated, string? OriginBreeder,
|
||||
string? OriginContactName, string? ReceiverContactName, string? Notes,
|
||||
List<string> CharacterTraits, string? CharacterNote, string? BirthLitterName);
|
||||
|
||||
public sealed record SnapshotParent(string Name, string? DateOfBirth, string? Genotype);
|
||||
|
||||
public sealed record SnapshotLitter(
|
||||
string Name, string? Date, string Role, int? TotalBorn, bool ShowInChronicle,
|
||||
List<SnapshotChild> Children);
|
||||
|
||||
public sealed record SnapshotChild(string Name, string? DateOfBirth, string Gender);
|
||||
|
||||
/// <summary>One field-level difference between two snapshots (golden ↔ current/raw import).</summary>
|
||||
public sealed record SnapshotDiff(string Path, string? GoldenValue, string? OtherValue);
|
||||
|
||||
/// <summary>
|
||||
/// Builds the canonical Akte snapshot for a gerbil, the machine-readable freeze map (own fields
|
||||
/// that "vollständig korrekt" pins), applies an override onto a gerbil, and diffs two snapshots.
|
||||
/// SHARED by the app (endpoints + ingest freeze) and the regression test so the comparison logic
|
||||
/// can never drift between the two.
|
||||
/// </summary>
|
||||
public static class GerbilSnapshotService
|
||||
{
|
||||
private static readonly JsonSerializerOptions Json = new(JsonSerializerDefaults.Web);
|
||||
|
||||
/// <summary>Own-field keys that the "vollständig korrekt" freeze pins. Deliberately excludes
|
||||
/// Status (lifecycle, derived) and EnclosureId (current location) and LitterId/lineage
|
||||
/// (abstammung is detect-only). Keys match the JSON serialised in <see cref="BuildFreezeObject"/>.</summary>
|
||||
public static readonly IReadOnlyList<string> FreezeFields = new[]
|
||||
{
|
||||
"name", "gender", "dateOfBirth", "dateOfDeath", "causeOfDeath", "goHomeDate",
|
||||
"genotype", "spottingType", "colorVarietyId", "isResident", "isDeaf", "isCastrated",
|
||||
"originBreeder", "originContactId", "receiverContactId", "notes",
|
||||
"characterTraits", "characterNote",
|
||||
};
|
||||
|
||||
// ── Snapshot (human-readable Akte) ───────────────────────────────────
|
||||
|
||||
public static async Task<GerbilSnapshot?> BuildSnapshotAsync(ApplicationContext db, Guid gerbilId)
|
||||
{
|
||||
var g = await db.Gerbils.AsNoTracking().FirstOrDefaultAsync(x => x.Id == gerbilId);
|
||||
if (g is null) return null;
|
||||
|
||||
var colorName = g.ColorVarietyId is { } cv
|
||||
? await db.ColorVarieties.AsNoTracking().Where(c => c.Id == cv).Select(c => c.Name).FirstOrDefaultAsync()
|
||||
: null;
|
||||
var originName = g.OriginContactId is { } oc
|
||||
? await db.Contacts.AsNoTracking().Where(c => c.Id == oc).Select(c => c.Name).FirstOrDefaultAsync()
|
||||
: null;
|
||||
var receiverName = g.ReceiverContactId is { } rc
|
||||
? await db.Contacts.AsNoTracking().Where(c => c.Id == rc).Select(c => c.Name).FirstOrDefaultAsync()
|
||||
: null;
|
||||
|
||||
// Birth litter → parents.
|
||||
Litter? birthLitter = g.LitterId is { } lid
|
||||
? await db.Litters.AsNoTracking().FirstOrDefaultAsync(l => l.Id == lid)
|
||||
: null;
|
||||
var father = await BuildParentAsync(db, birthLitter?.FatherId);
|
||||
var mother = await BuildParentAsync(db, birthLitter?.MotherId);
|
||||
|
||||
// Own litters (as parent) + children.
|
||||
var ownLitters = await db.Litters.AsNoTracking()
|
||||
.Where(l => l.FatherId == gerbilId || l.MotherId == gerbilId)
|
||||
.ToListAsync();
|
||||
var ownLitterIds = ownLitters.Select(l => l.Id).ToList();
|
||||
var children = ownLitterIds.Count == 0
|
||||
? new List<Gerbil>()
|
||||
: await db.Gerbils.AsNoTracking().Where(c => c.LitterId != null && ownLitterIds.Contains(c.LitterId.Value)).ToListAsync();
|
||||
|
||||
var litterSnaps = ownLitters
|
||||
.Select(l => new SnapshotLitter(
|
||||
l.Name,
|
||||
Iso(l.Date),
|
||||
l.FatherId == gerbilId ? "father" : "mother",
|
||||
l.TotalBorn,
|
||||
l.ShowInChronicle,
|
||||
children.Where(c => c.LitterId == l.Id)
|
||||
.Select(c => new SnapshotChild(c.Name, Iso(c.DateOfBirth), c.Gender.ToString()))
|
||||
.OrderBy(c => c.DateOfBirth ?? "", StringComparer.Ordinal)
|
||||
.ThenBy(c => c.Name, StringComparer.Ordinal)
|
||||
.ThenBy(c => c.Gender, StringComparer.Ordinal)
|
||||
.ToList()))
|
||||
.OrderBy(l => l.Date ?? "", StringComparer.Ordinal)
|
||||
.ThenBy(l => l.Name, StringComparer.Ordinal)
|
||||
.ToList();
|
||||
|
||||
var self = new SnapshotSelf(
|
||||
g.Name, g.Gender.ToString(), Iso(g.DateOfBirth), Iso(g.DateOfDeath), g.CauseOfDeath,
|
||||
Iso(g.GoHomeDate), g.Genotype, g.SpottingType, colorName,
|
||||
g.IsResident, g.IsDeaf, g.IsCastrated, g.OriginBreeder,
|
||||
originName, receiverName, g.Notes,
|
||||
(g.CharacterTraits ?? new List<string>()).OrderBy(t => t, StringComparer.Ordinal).ToList(),
|
||||
g.CharacterNote, birthLitter?.Name);
|
||||
|
||||
return new GerbilSnapshot(self, father, mother, litterSnaps);
|
||||
}
|
||||
|
||||
private static async Task<SnapshotParent?> BuildParentAsync(ApplicationContext db, Guid? parentId)
|
||||
{
|
||||
if (parentId is not { } pid) return null;
|
||||
var p = await db.Gerbils.AsNoTracking().Where(x => x.Id == pid)
|
||||
.Select(x => new { x.Name, x.DateOfBirth, x.Genotype }).FirstOrDefaultAsync();
|
||||
return p is null ? null : new SnapshotParent(p.Name, Iso(p.DateOfBirth), p.Genotype);
|
||||
}
|
||||
|
||||
public static string SerializeSnapshot(GerbilSnapshot snap) => JsonSerializer.Serialize(snap, Json);
|
||||
|
||||
public static GerbilSnapshot? DeserializeSnapshot(string? json) =>
|
||||
string.IsNullOrWhiteSpace(json) ? null : JsonSerializer.Deserialize<GerbilSnapshot>(json, Json);
|
||||
|
||||
// ── Diff (golden ↔ other) ────────────────────────────────────────────
|
||||
|
||||
/// <summary>Flat, human-readable field diff between a golden snapshot and another (current
|
||||
/// DB or raw import) snapshot. Empty = identical.</summary>
|
||||
public static List<SnapshotDiff> Diff(GerbilSnapshot? golden, GerbilSnapshot? other)
|
||||
{
|
||||
var g = golden is null ? new Dictionary<string, string?>() : Flatten(golden);
|
||||
var o = other is null ? new Dictionary<string, string?>() : Flatten(other);
|
||||
var diffs = new List<SnapshotDiff>();
|
||||
foreach (var key in g.Keys.Union(o.Keys).OrderBy(k => k, StringComparer.Ordinal))
|
||||
{
|
||||
var gv = g.GetValueOrDefault(key);
|
||||
var ov = o.GetValueOrDefault(key);
|
||||
if (!string.Equals(gv, ov, StringComparison.Ordinal))
|
||||
diffs.Add(new SnapshotDiff(key, gv, ov));
|
||||
}
|
||||
return diffs;
|
||||
}
|
||||
|
||||
private static Dictionary<string, string?> Flatten(GerbilSnapshot s)
|
||||
{
|
||||
var d = new Dictionary<string, string?>(StringComparer.Ordinal)
|
||||
{
|
||||
["name"] = s.Self.Name,
|
||||
["gender"] = s.Self.Gender,
|
||||
["dateOfBirth"] = s.Self.DateOfBirth,
|
||||
["dateOfDeath"] = s.Self.DateOfDeath,
|
||||
["causeOfDeath"] = s.Self.CauseOfDeath,
|
||||
["goHomeDate"] = s.Self.GoHomeDate,
|
||||
["genotype"] = s.Self.Genotype,
|
||||
["spottingType"] = s.Self.SpottingType,
|
||||
["colorName"] = s.Self.ColorName,
|
||||
["isResident"] = s.Self.IsResident.ToString(),
|
||||
["isDeaf"] = s.Self.IsDeaf?.ToString(),
|
||||
["isCastrated"] = s.Self.IsCastrated.ToString(),
|
||||
["originBreeder"] = s.Self.OriginBreeder,
|
||||
["originContact"] = s.Self.OriginContactName,
|
||||
["receiverContact"] = s.Self.ReceiverContactName,
|
||||
["notes"] = s.Self.Notes,
|
||||
["characterTraits"] = string.Join(", ", s.Self.CharacterTraits),
|
||||
["characterNote"] = s.Self.CharacterNote,
|
||||
["birthLitter"] = s.Self.BirthLitterName,
|
||||
["father"] = FormatParent(s.Father),
|
||||
["mother"] = FormatParent(s.Mother),
|
||||
};
|
||||
foreach (var l in s.Litters)
|
||||
{
|
||||
var lk = $"wurf[{l.Name} {l.Date}]";
|
||||
d[$"{lk}.rolle"] = l.Role;
|
||||
d[$"{lk}.geboren"] = l.TotalBorn?.ToString();
|
||||
d[$"{lk}.inChronik"] = l.ShowInChronicle.ToString();
|
||||
foreach (var c in l.Children)
|
||||
d[$"{lk}.kind[{c.Name} {c.DateOfBirth}]"] = c.Gender;
|
||||
}
|
||||
return d;
|
||||
}
|
||||
|
||||
private static string? FormatParent(SnapshotParent? p) =>
|
||||
p is null ? null : $"{p.Name} (*{p.DateOfBirth}) {p.Genotype}".Trim();
|
||||
|
||||
// ── Freeze map (machine-readable own-field override) ─────────────────
|
||||
|
||||
/// <summary>Builds the full own-field freeze object (all <see cref="FreezeFields"/>) from a
|
||||
/// gerbil's current values. Used by verify (store as OverrideJson) and by the per-field
|
||||
/// edit-protection diff.</summary>
|
||||
public static JsonObject BuildFreezeObject(Gerbil g)
|
||||
{
|
||||
var o = new JsonObject
|
||||
{
|
||||
["name"] = g.Name,
|
||||
["gender"] = g.Gender.ToString(),
|
||||
["dateOfBirth"] = Iso(g.DateOfBirth),
|
||||
["dateOfDeath"] = Iso(g.DateOfDeath),
|
||||
["causeOfDeath"] = g.CauseOfDeath,
|
||||
["goHomeDate"] = Iso(g.GoHomeDate),
|
||||
["genotype"] = g.Genotype,
|
||||
["spottingType"] = g.SpottingType,
|
||||
["colorVarietyId"] = g.ColorVarietyId?.ToString(),
|
||||
["isResident"] = g.IsResident,
|
||||
["isDeaf"] = g.IsDeaf,
|
||||
["isCastrated"] = g.IsCastrated,
|
||||
["originBreeder"] = g.OriginBreeder,
|
||||
["originContactId"] = g.OriginContactId?.ToString(),
|
||||
["receiverContactId"] = g.ReceiverContactId?.ToString(),
|
||||
["notes"] = g.Notes,
|
||||
["characterTraits"] = new JsonArray((g.CharacterTraits ?? new List<string>()).Select(t => (JsonNode?)JsonValue.Create(t)).ToArray()),
|
||||
["characterNote"] = g.CharacterNote,
|
||||
};
|
||||
return o;
|
||||
}
|
||||
|
||||
public static string BuildFreezeJson(Gerbil g) => BuildFreezeObject(g).ToJsonString(Json);
|
||||
|
||||
/// <summary>Per-field change detection for the PUT edit-protection path. Returns the subset of
|
||||
/// freeze keys whose value differs between <paramref name="before"/> and <paramref name="after"/>,
|
||||
/// as a JsonObject carrying the AFTER values (the ones to pin). Empty if nothing relevant changed.</summary>
|
||||
public static JsonObject ChangedFields(JsonObject before, JsonObject after)
|
||||
{
|
||||
var changed = new JsonObject();
|
||||
foreach (var key in FreezeFields)
|
||||
{
|
||||
var b = before[key]?.ToJsonString() ?? "null";
|
||||
var a = after[key]?.ToJsonString() ?? "null";
|
||||
if (!string.Equals(b, a, StringComparison.Ordinal))
|
||||
changed[key] = after[key] is { } node ? node.DeepClone() : null;
|
||||
}
|
||||
return changed;
|
||||
}
|
||||
|
||||
/// <summary>Merge <paramref name="incoming"/> field values into an existing override JSON
|
||||
/// object (incoming wins). Returns the merged JSON string.</summary>
|
||||
public static string MergeOverrideJson(string? existingJson, JsonObject incoming)
|
||||
{
|
||||
var baseObj = ParseObject(existingJson) ?? new JsonObject();
|
||||
foreach (var kv in incoming)
|
||||
baseObj[kv.Key] = kv.Value is { } n ? n.DeepClone() : null;
|
||||
return baseObj.ToJsonString(Json);
|
||||
}
|
||||
|
||||
/// <summary>Applies a stored override JSON object onto a gerbil, writing only the keys present
|
||||
/// (the "freeze"). Unknown keys are ignored.</summary>
|
||||
public static void ApplyOverride(Gerbil g, string? overrideJson)
|
||||
{
|
||||
var o = ParseObject(overrideJson);
|
||||
if (o is null) return;
|
||||
foreach (var kv in o)
|
||||
{
|
||||
var v = kv.Value;
|
||||
switch (kv.Key)
|
||||
{
|
||||
case "name": if (Str(v) is { Length: > 0 } nm) g.Name = nm; break;
|
||||
case "gender": if (Str(v) is { } gd && Enum.TryParse<Gender>(gd, out var gender)) g.Gender = gender; break;
|
||||
case "dateOfBirth": g.DateOfBirth = Date(v); break;
|
||||
case "dateOfDeath": g.DateOfDeath = Date(v); break;
|
||||
case "causeOfDeath": g.CauseOfDeath = Str(v); break;
|
||||
case "goHomeDate": g.GoHomeDate = Date(v); break;
|
||||
case "genotype": g.Genotype = Str(v); break;
|
||||
case "spottingType": g.SpottingType = Str(v); break;
|
||||
case "colorVarietyId": g.ColorVarietyId = Guid_(v); break;
|
||||
case "isResident": g.IsResident = Bool(v) ?? g.IsResident; break;
|
||||
case "isDeaf": g.IsDeaf = Bool(v); break;
|
||||
case "isCastrated": g.IsCastrated = Bool(v) ?? g.IsCastrated; break;
|
||||
case "originBreeder": g.OriginBreeder = Str(v); break;
|
||||
case "originContactId": g.OriginContactId = Guid_(v); break;
|
||||
case "receiverContactId": g.ReceiverContactId = Guid_(v); break;
|
||||
case "notes": g.Notes = Str(v); break;
|
||||
case "characterTraits": g.CharacterTraits = StrList(v); break;
|
||||
case "characterNote": g.CharacterNote = Str(v); break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── JSON helpers ─────────────────────────────────────────────────────
|
||||
|
||||
private static JsonObject? ParseObject(string? json)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(json)) return null;
|
||||
try { return JsonNode.Parse(json) as JsonObject; }
|
||||
catch (JsonException) { return null; }
|
||||
}
|
||||
|
||||
private static string? Str(JsonNode? n) => n?.GetValueKind() == JsonValueKind.String ? n.GetValue<string>() : null;
|
||||
private static bool? Bool(JsonNode? n) => n?.GetValueKind() is JsonValueKind.True or JsonValueKind.False ? n!.GetValue<bool>() : null;
|
||||
private static Guid? Guid_(JsonNode? n) => Guid.TryParse(Str(n), out var id) ? id : null;
|
||||
private static DateOnly? Date(JsonNode? n) => DateOnly.TryParse(Str(n), out var d) ? d : null;
|
||||
|
||||
private static List<string> StrList(JsonNode? n)
|
||||
{
|
||||
if (n is JsonArray arr)
|
||||
return arr.Where(x => x is not null).Select(x => x!.GetValue<string>()).ToList();
|
||||
return new List<string>();
|
||||
}
|
||||
|
||||
private static string? Iso(DateOnly? d) => d?.ToString("yyyy-MM-dd");
|
||||
}
|
||||
}
|
||||
@@ -13,6 +13,8 @@ namespace GerbilManagerWebAPI.Import
|
||||
private readonly string _sourceDir;
|
||||
private readonly string _photoRoot;
|
||||
|
||||
private static readonly JsonSerializerOptions DiffJson = new(JsonSerializerDefaults.Web);
|
||||
|
||||
public IngestResolvedService(ApplicationContext db, IConfiguration config, IWebHostEnvironment? env)
|
||||
{
|
||||
_db = db;
|
||||
@@ -32,11 +34,8 @@ namespace GerbilManagerWebAPI.Import
|
||||
}
|
||||
|
||||
string jsonContent = await File.ReadAllTextAsync(_resolvedJsonPath);
|
||||
var jsonOptions = new JsonSerializerOptions
|
||||
{
|
||||
PropertyNameCaseInsensitive = true
|
||||
};
|
||||
jsonOptions.Converters.Add(new System.Text.Json.Serialization.JsonStringEnumConverter());
|
||||
var jsonOptions = new JsonSerializerOptions { PropertyNameCaseInsensitive = true };
|
||||
jsonOptions.Converters.Add(new JsonStringEnumConverter());
|
||||
var data = JsonSerializer.Deserialize<ResolvedImportData>(jsonContent, jsonOptions);
|
||||
|
||||
if (data == null)
|
||||
@@ -44,163 +43,185 @@ namespace GerbilManagerWebAPI.Import
|
||||
return "Error: Failed to deserialize resolved_import.json.";
|
||||
}
|
||||
|
||||
// 1. Provider-agnostic clean wipe of dependent tables to prevent duplicate keys
|
||||
if (_db.Database.ProviderName == "Microsoft.EntityFrameworkCore.InMemory")
|
||||
{
|
||||
// Break circular references first
|
||||
foreach (var l in _db.Litters)
|
||||
{
|
||||
l.FatherId = null;
|
||||
l.MotherId = null;
|
||||
}
|
||||
foreach (var g in _db.Gerbils)
|
||||
{
|
||||
g.LitterId = null;
|
||||
}
|
||||
await _db.SaveChangesAsync();
|
||||
// ── UPSERT-based ingest (no total wipe) ──────────────────────────────────────
|
||||
// The ingest NO LONGER wipes everything. Imported rows are matched by their
|
||||
// (deterministic) id and updated in place; manually-created rows (IsManual=true) and
|
||||
// user-entered sub-records (weights/health/photos) are never deleted, so the breeder's
|
||||
// work survives the re-import. Only STALE imported rows (IsManual=false, absent from the
|
||||
// new payload) are removed — after nulling any manual references to them (with a warning).
|
||||
// Finally, hand-curated GerbilOverrides are re-applied on top (the "freeze").
|
||||
var payloadContactIds = data.Contacts.Select(c => c.Id).ToHashSet();
|
||||
var payloadLitterIds = data.Litters.Select(l => l.Id).ToHashSet();
|
||||
var payloadGerbilIds = data.Gerbils.Select(g => g.Id).ToHashSet();
|
||||
var warnings = new List<string>();
|
||||
|
||||
_db.GerbilPhotos.RemoveRange(_db.GerbilPhotos);
|
||||
_db.WeightRecords.RemoveRange(_db.WeightRecords);
|
||||
_db.HealthRecords.RemoveRange(_db.HealthRecords);
|
||||
_db.SaleContracts.RemoveRange(_db.SaleContracts);
|
||||
_db.Gerbils.RemoveRange(_db.Gerbils);
|
||||
_db.Litters.RemoveRange(_db.Litters);
|
||||
_db.Contacts.RemoveRange(_db.Contacts);
|
||||
await _db.SaveChangesAsync();
|
||||
}
|
||||
else
|
||||
{
|
||||
// Break circular references first using ExecuteUpdateAsync
|
||||
await _db.Litters.ExecuteUpdateAsync(s => s
|
||||
.SetProperty(l => l.FatherId, (Guid?)null)
|
||||
.SetProperty(l => l.MotherId, (Guid?)null));
|
||||
|
||||
await _db.Gerbils.ExecuteUpdateAsync(s => s
|
||||
.SetProperty(g => g.LitterId, (Guid?)null));
|
||||
|
||||
await _db.GerbilPhotos.ExecuteDeleteAsync();
|
||||
await _db.WeightRecords.ExecuteDeleteAsync();
|
||||
await _db.HealthRecords.ExecuteDeleteAsync();
|
||||
await _db.SaleContracts.ExecuteDeleteAsync();
|
||||
await _db.Gerbils.ExecuteDeleteAsync();
|
||||
await _db.Litters.ExecuteDeleteAsync();
|
||||
await _db.Contacts.ExecuteDeleteAsync();
|
||||
}
|
||||
|
||||
// 2. Import Contacts (Add new ones, and update roles/details of existing ones)
|
||||
// 1. Upsert contacts (preserve manual rows + their IsManual flag).
|
||||
var existingContacts = await _db.Contacts.ToDictionaryAsync(c => c.Id);
|
||||
var addedContactIds = new HashSet<Guid>();
|
||||
int contactsAdded = 0;
|
||||
int contactsUpdated = 0;
|
||||
int contactsAdded = 0, contactsUpdated = 0;
|
||||
foreach (var c in data.Contacts)
|
||||
{
|
||||
if (existingContacts.TryGetValue(c.Id, out var existingContact))
|
||||
if (existingContacts.TryGetValue(c.Id, out var ec))
|
||||
{
|
||||
existingContact.Name = c.Name;
|
||||
existingContact.Email = c.Email;
|
||||
existingContact.Phone = c.Phone;
|
||||
existingContact.Address = c.Address;
|
||||
existingContact.Notes = c.Notes;
|
||||
existingContact.IsBreeder = c.IsBreeder;
|
||||
existingContact.IsReceiver = c.IsReceiver;
|
||||
existingContact.NameSuffix = c.NameSuffix;
|
||||
existingContact.Provenance = c.Provenance;
|
||||
ec.Name = c.Name; ec.Email = c.Email; ec.Phone = c.Phone; ec.Address = c.Address;
|
||||
ec.Notes = c.Notes; ec.IsBreeder = c.IsBreeder; ec.IsReceiver = c.IsReceiver;
|
||||
ec.NameSuffix = c.NameSuffix; ec.Provenance = c.Provenance;
|
||||
contactsUpdated++;
|
||||
}
|
||||
else if (addedContactIds.Add(c.Id))
|
||||
else
|
||||
{
|
||||
c.IsManual = false;
|
||||
_db.Contacts.Add(c);
|
||||
contactsAdded++;
|
||||
}
|
||||
}
|
||||
await _db.SaveChangesAsync();
|
||||
|
||||
// 3. Pass 1: Insert all Litters with parent references set to null to avoid FK errors
|
||||
var littersToInsert = new List<Litter>();
|
||||
// 2. Upsert litters (parent FKs reset to null; set in pass 2 to avoid ordering issues).
|
||||
var existingLitters = await _db.Litters.ToDictionaryAsync(l => l.Id);
|
||||
foreach (var l in data.Litters)
|
||||
{
|
||||
littersToInsert.Add(new Litter
|
||||
if (existingLitters.TryGetValue(l.Id, out var el))
|
||||
{
|
||||
Id = l.Id,
|
||||
Name = l.Name,
|
||||
Date = l.Date,
|
||||
TotalBorn = l.TotalBorn,
|
||||
DeathsWithin8Weeks = l.DeathsWithin8Weeks,
|
||||
FatherId = null, // Set in Pass 2
|
||||
MotherId = null, // Set in Pass 2
|
||||
ExpectedGoHomeDate = l.ExpectedGoHomeDate,
|
||||
Notes = l.Notes,
|
||||
PairingCode = l.PairingCode,
|
||||
ExternalRef = l.ExternalRef,
|
||||
LitterLetter = l.LitterLetter,
|
||||
Provenance = l.Provenance
|
||||
});
|
||||
el.Name = l.Name; el.Date = l.Date; el.TotalBorn = l.TotalBorn;
|
||||
el.DeathsWithin8Weeks = l.DeathsWithin8Weeks; el.Stillborn = l.Stillborn;
|
||||
el.ExpectedGoHomeDate = l.ExpectedGoHomeDate; el.Notes = l.Notes;
|
||||
el.PairingCode = l.PairingCode; el.ExternalRef = l.ExternalRef;
|
||||
el.LitterLetter = l.LitterLetter; el.Provenance = l.Provenance;
|
||||
el.ShowInChronicle = l.ShowInChronicle;
|
||||
el.FatherId = null; el.MotherId = null;
|
||||
}
|
||||
else
|
||||
{
|
||||
_db.Litters.Add(new Litter
|
||||
{
|
||||
Id = l.Id, Name = l.Name, Date = l.Date, TotalBorn = l.TotalBorn,
|
||||
DeathsWithin8Weeks = l.DeathsWithin8Weeks, Stillborn = l.Stillborn, FatherId = null, MotherId = null,
|
||||
ExpectedGoHomeDate = l.ExpectedGoHomeDate, Notes = l.Notes, PairingCode = l.PairingCode,
|
||||
ExternalRef = l.ExternalRef, LitterLetter = l.LitterLetter, Provenance = l.Provenance,
|
||||
ShowInChronicle = l.ShowInChronicle, IsManual = false,
|
||||
});
|
||||
}
|
||||
}
|
||||
_db.Litters.AddRange(littersToInsert);
|
||||
await _db.SaveChangesAsync();
|
||||
|
||||
// 4. Pass 1: Insert all Gerbils with LitterId set to null to avoid FK errors
|
||||
var gerbilsToInsert = new List<Gerbil>();
|
||||
// 3. Upsert gerbils (LitterId reset to null; set in pass 2). Manual rows keep IsManual=true
|
||||
// and are never matched here (their random ids never collide with deterministic import ids).
|
||||
var existingGerbils = await _db.Gerbils.ToDictionaryAsync(g => g.Id);
|
||||
foreach (var g in data.Gerbils)
|
||||
{
|
||||
var importedGerbil = new Gerbil
|
||||
Gerbil target;
|
||||
if (existingGerbils.TryGetValue(g.Id, out var eg))
|
||||
{
|
||||
Id = g.Id,
|
||||
Name = g.Name,
|
||||
Gender = g.Gender,
|
||||
Status = g.Status,
|
||||
LitterId = null, // Set in Pass 2
|
||||
OriginContactId = g.OriginContactId,
|
||||
ReceiverContactId = g.ReceiverContactId,
|
||||
EnclosureId = g.EnclosureId,
|
||||
ColorVarietyId = g.ColorVarietyId,
|
||||
DateOfBirth = g.DateOfBirth,
|
||||
DateOfDeath = g.DateOfDeath,
|
||||
CauseOfDeath = g.CauseOfDeath,
|
||||
GoHomeDate = g.GoHomeDate,
|
||||
Genotype = g.Genotype,
|
||||
Notes = g.Notes,
|
||||
ImportSource = g.ImportSource,
|
||||
ExternalRef = g.ExternalRef,
|
||||
RawImportData = g.RawImportData,
|
||||
Provenance = g.Provenance,
|
||||
OriginBreeder = g.OriginBreeder,
|
||||
NameSearch = g.NameSearch,
|
||||
CharacterTraits = g.CharacterTraits,
|
||||
CharacterNote = g.CharacterNote,
|
||||
IsDeaf = g.IsDeaf,
|
||||
IsResident = g.IsResident
|
||||
};
|
||||
GerbilStatusService.Apply(importedGerbil, DateOnly.FromDateTime(DateTime.UtcNow));
|
||||
gerbilsToInsert.Add(importedGerbil);
|
||||
eg.Name = g.Name; eg.Gender = g.Gender; eg.Status = g.Status;
|
||||
eg.OriginContactId = g.OriginContactId; eg.ReceiverContactId = g.ReceiverContactId;
|
||||
eg.EnclosureId = g.EnclosureId; eg.ColorVarietyId = g.ColorVarietyId;
|
||||
eg.DateOfBirth = g.DateOfBirth; eg.DateOfDeath = g.DateOfDeath; eg.CauseOfDeath = g.CauseOfDeath;
|
||||
eg.GoHomeDate = g.GoHomeDate; eg.Genotype = g.Genotype; eg.Notes = g.Notes;
|
||||
eg.ImportSource = g.ImportSource; eg.ExternalRef = g.ExternalRef; eg.RawImportData = g.RawImportData;
|
||||
eg.Provenance = g.Provenance; eg.OriginBreeder = g.OriginBreeder;
|
||||
eg.CharacterTraits = g.CharacterTraits; eg.CharacterNote = g.CharacterNote;
|
||||
eg.IsDeaf = g.IsDeaf; eg.IsResident = g.IsResident; eg.LitterId = null;
|
||||
eg.BirthOrder = g.BirthOrder;
|
||||
target = eg;
|
||||
}
|
||||
else
|
||||
{
|
||||
var ng = new Gerbil
|
||||
{
|
||||
Id = g.Id, Name = g.Name, Gender = g.Gender, Status = g.Status, LitterId = null,
|
||||
OriginContactId = g.OriginContactId, ReceiverContactId = g.ReceiverContactId,
|
||||
EnclosureId = g.EnclosureId, ColorVarietyId = g.ColorVarietyId,
|
||||
DateOfBirth = g.DateOfBirth, DateOfDeath = g.DateOfDeath, CauseOfDeath = g.CauseOfDeath,
|
||||
GoHomeDate = g.GoHomeDate, Genotype = g.Genotype, Notes = g.Notes,
|
||||
ImportSource = g.ImportSource, ExternalRef = g.ExternalRef, RawImportData = g.RawImportData,
|
||||
Provenance = g.Provenance, OriginBreeder = g.OriginBreeder, NameSearch = g.NameSearch,
|
||||
CharacterTraits = g.CharacterTraits, CharacterNote = g.CharacterNote,
|
||||
IsDeaf = g.IsDeaf, IsResident = g.IsResident, IsManual = false,
|
||||
BirthOrder = g.BirthOrder,
|
||||
};
|
||||
_db.Gerbils.Add(ng);
|
||||
target = ng;
|
||||
}
|
||||
GerbilStatusService.Apply(target, DateOnly.FromDateTime(DateTime.UtcNow));
|
||||
}
|
||||
_db.Gerbils.AddRange(gerbilsToInsert);
|
||||
await _db.SaveChangesAsync();
|
||||
|
||||
// 5. Pass 2: Set the actual foreign key relationships
|
||||
// Update Litters with their parent animal IDs
|
||||
// 4. Pass 2: set the payload FK relationships (only for payload rows; manual rows untouched).
|
||||
var littersInDb = await _db.Litters.ToDictionaryAsync(l => l.Id);
|
||||
foreach (var l in data.Litters)
|
||||
{
|
||||
if (littersInDb.TryGetValue(l.Id, out var dbLitter))
|
||||
{
|
||||
dbLitter.FatherId = l.FatherId;
|
||||
dbLitter.MotherId = l.MotherId;
|
||||
}
|
||||
}
|
||||
|
||||
// Update Gerbils with their actual LitterId
|
||||
if (littersInDb.TryGetValue(l.Id, out var dl)) { dl.FatherId = l.FatherId; dl.MotherId = l.MotherId; }
|
||||
var gerbilsInDb = await _db.Gerbils.ToDictionaryAsync(g => g.Id);
|
||||
foreach (var g in data.Gerbils)
|
||||
if (gerbilsInDb.TryGetValue(g.Id, out var dg)) dg.LitterId = g.LitterId;
|
||||
await _db.SaveChangesAsync();
|
||||
|
||||
// 5. Remove STALE imported rows (IsManual=false, no longer in the payload). Null any
|
||||
// dangling references from surviving rows first; warn when a MANUAL row is affected.
|
||||
var allGerbils = await _db.Gerbils.ToListAsync();
|
||||
var allLitters = await _db.Litters.ToListAsync();
|
||||
var allContacts = await _db.Contacts.ToListAsync();
|
||||
var gerbilNameById = allGerbils.ToDictionary(x => x.Id, x => x.Name);
|
||||
var litterNameById = allLitters.ToDictionary(x => x.Id, x => x.Name);
|
||||
var contactNameById = allContacts.ToDictionary(x => x.Id, x => x.Name);
|
||||
|
||||
var staleGerbilIds = allGerbils.Where(g => !g.IsManual && !payloadGerbilIds.Contains(g.Id)).Select(g => g.Id).ToHashSet();
|
||||
var staleLitterIds = allLitters.Where(l => !l.IsManual && !payloadLitterIds.Contains(l.Id)).Select(l => l.Id).ToHashSet();
|
||||
var staleContactIds = allContacts.Where(c => !c.IsManual && !payloadContactIds.Contains(c.Id)).Select(c => c.Id).ToHashSet();
|
||||
|
||||
if (staleGerbilIds.Count > 0)
|
||||
{
|
||||
if (gerbilsInDb.TryGetValue(g.Id, out var dbGerbil))
|
||||
foreach (var l in allLitters)
|
||||
{
|
||||
dbGerbil.LitterId = g.LitterId;
|
||||
if (l.FatherId is { } f && staleGerbilIds.Contains(f))
|
||||
{
|
||||
if (l.IsManual) warnings.Add($"Manueller Wurf „{l.Name}\" verwies auf importiertes Elterntier „{gerbilNameById.GetValueOrDefault(f, "?")}\" (Vater), das im Import nicht mehr existiert — Elternteil entfernt.");
|
||||
l.FatherId = null;
|
||||
}
|
||||
if (l.MotherId is { } m && staleGerbilIds.Contains(m))
|
||||
{
|
||||
if (l.IsManual) warnings.Add($"Manueller Wurf „{l.Name}\" verwies auf importiertes Elterntier „{gerbilNameById.GetValueOrDefault(m, "?")}\" (Mutter), das im Import nicht mehr existiert — Elternteil entfernt.");
|
||||
l.MotherId = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (staleContactIds.Count > 0)
|
||||
{
|
||||
foreach (var g in allGerbils)
|
||||
{
|
||||
if (g.OriginContactId is { } o && staleContactIds.Contains(o))
|
||||
{
|
||||
if (g.IsManual) warnings.Add($"Manuelles Tier „{g.Name}\" verwies auf Kontakt „{contactNameById.GetValueOrDefault(o, "?")}\" (Herkunft), der im Import nicht mehr existiert — Zuordnung entfernt.");
|
||||
g.OriginContactId = null;
|
||||
}
|
||||
if (g.ReceiverContactId is { } r && staleContactIds.Contains(r))
|
||||
{
|
||||
if (g.IsManual) warnings.Add($"Manuelles Tier „{g.Name}\" verwies auf Kontakt „{contactNameById.GetValueOrDefault(r, "?")}\" (Abnehmer), der im Import nicht mehr existiert — Zuordnung entfernt.");
|
||||
g.ReceiverContactId = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (staleLitterIds.Count > 0)
|
||||
{
|
||||
// Gerbil.LitterId FK is SetNull on delete → nulled automatically when the litter is
|
||||
// removed; we only surface a warning for affected MANUAL animals.
|
||||
foreach (var g in allGerbils)
|
||||
if (g.IsManual && g.LitterId is { } lid && staleLitterIds.Contains(lid))
|
||||
warnings.Add($"Manuelles Tier „{g.Name}\" war im Wurf „{litterNameById.GetValueOrDefault(lid, "?")}\" geboren, der im Import nicht mehr existiert — Wurf-Zuordnung entfernt.");
|
||||
}
|
||||
await _db.SaveChangesAsync();
|
||||
|
||||
int gerbilsDeleted = staleGerbilIds.Count, littersDeleted = staleLitterIds.Count, contactsDeleted = staleContactIds.Count;
|
||||
if (staleGerbilIds.Count > 0) _db.Gerbils.RemoveRange(allGerbils.Where(g => staleGerbilIds.Contains(g.Id)));
|
||||
if (staleLitterIds.Count > 0) _db.Litters.RemoveRange(allLitters.Where(l => staleLitterIds.Contains(l.Id)));
|
||||
if (staleContactIds.Count > 0) _db.Contacts.RemoveRange(allContacts.Where(c => staleContactIds.Contains(c.Id)));
|
||||
await _db.SaveChangesAsync();
|
||||
|
||||
// 6. Photos: replace photos of IMPORTED gerbils (manual gerbils' photos survive). Copy files.
|
||||
var manualGerbilIds = allGerbils.Where(g => g.IsManual).Select(g => g.Id).ToHashSet();
|
||||
var allPhotos = await _db.GerbilPhotos.ToListAsync();
|
||||
_db.GerbilPhotos.RemoveRange(allPhotos.Where(p => !manualGerbilIds.Contains(p.GerbilId)));
|
||||
await _db.SaveChangesAsync();
|
||||
|
||||
// 6. Insert all Photos & copy physical files
|
||||
Directory.CreateDirectory(_photoRoot);
|
||||
foreach (var p in data.GerbilPhotos)
|
||||
{
|
||||
@@ -211,14 +232,8 @@ namespace GerbilManagerWebAPI.Import
|
||||
if (File.Exists(src))
|
||||
{
|
||||
var dest = Path.Combine(_photoRoot, p.FileName);
|
||||
try
|
||||
{
|
||||
File.Copy(src, dest, overwrite: true);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($"Warning: Failed to copy photo {src} to {dest}: {ex.Message}");
|
||||
}
|
||||
try { File.Copy(src, dest, overwrite: true); }
|
||||
catch (Exception ex) { Console.WriteLine($"Warning: Failed to copy photo {src} to {dest}: {ex.Message}"); }
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -233,24 +248,18 @@ namespace GerbilManagerWebAPI.Import
|
||||
FileName = p.FileName,
|
||||
Caption = p.Caption,
|
||||
SortOrder = p.SortOrder,
|
||||
CreatedAt = DateTimeOffset.UtcNow
|
||||
CreatedAt = DateTimeOffset.UtcNow,
|
||||
});
|
||||
}
|
||||
await _db.SaveChangesAsync();
|
||||
|
||||
// 7. Insert imported Abgabeverträge (SaleContract + join rows).
|
||||
// The table was wiped above, so these are recreated on every ingest
|
||||
// (they survive only by being part of the payload). FK-safe: contacts
|
||||
// and gerbils are already inserted. We skip rows whose buyer contact
|
||||
// is unknown and skip individual animal links to unknown gerbils
|
||||
// (defensive — the resolver should never emit those).
|
||||
// No .docx is copied: the source files live on the network share and
|
||||
// are not staged into Contracts:RootPath, so the Word download 404s
|
||||
// gracefully (the endpoint already returns NotFound) while the PDF is
|
||||
// regenerated from the data. The DTO exposes HasFile=false for these
|
||||
// so the frontend hides the Word button.
|
||||
int contractsAdded = 0;
|
||||
int contractAnimalsSkipped = 0;
|
||||
// 7. Imported Abgabeverträge: recreate from payload (unchanged behaviour — the imported set
|
||||
// is part of the payload). Manual sale-contract protection is a separate follow-up.
|
||||
var scToRemove = await _db.SaleContracts.ToListAsync();
|
||||
_db.SaleContracts.RemoveRange(scToRemove);
|
||||
await _db.SaveChangesAsync();
|
||||
|
||||
int contractsAdded = 0, contractAnimalsSkipped = 0;
|
||||
if (data.SaleContracts.Count > 0)
|
||||
{
|
||||
var contactIds = await _db.Contacts.Select(c => c.Id).ToHashSetAsync();
|
||||
@@ -258,8 +267,8 @@ namespace GerbilManagerWebAPI.Import
|
||||
var seenContractIds = new HashSet<Guid>();
|
||||
foreach (var sc in data.SaleContracts)
|
||||
{
|
||||
if (!seenContractIds.Add(sc.Id)) continue; // dedupe within payload
|
||||
if (!contactIds.Contains(sc.ContactId)) continue; // unknown buyer
|
||||
if (!seenContractIds.Add(sc.Id)) continue;
|
||||
if (!contactIds.Contains(sc.ContactId)) continue;
|
||||
|
||||
var animals = new List<SaleContractAnimal>();
|
||||
foreach (var gid in (sc.Animals ?? new List<Guid>()).Distinct())
|
||||
@@ -284,7 +293,36 @@ namespace GerbilManagerWebAPI.Import
|
||||
await _db.SaveChangesAsync();
|
||||
}
|
||||
|
||||
return $"Ingestion successful! Imported {contactsAdded} contacts ({contactsUpdated} updated), {data.Litters.Count} litters, {data.Gerbils.Count} gerbils, {data.GerbilPhotos.Count} photos, and {contractsAdded} sale contracts ({contractAnimalsSkipped} animal links to unknown gerbils skipped).";
|
||||
// 8. FREEZE: capture the raw-import snapshot for verified overrides (BEFORE applying the
|
||||
// freeze), then re-apply every override on top of the fresh import so the breeder's
|
||||
// curated values win. Overrides survive re-ingest by being their own (FK-free) table.
|
||||
var overrides = await _db.GerbilOverrides.ToListAsync();
|
||||
foreach (var ov in overrides.Where(o => o.IsVerified))
|
||||
{
|
||||
var raw = await GerbilSnapshotService.BuildSnapshotAsync(_db, ov.GerbilId);
|
||||
if (raw is null) continue; // verified animal vanished from import → surfaced as "missing" in the UI
|
||||
ov.LastImportSnapshotJson = GerbilSnapshotService.SerializeSnapshot(raw);
|
||||
var golden = GerbilSnapshotService.DeserializeSnapshot(ov.SnapshotJson);
|
||||
var diff = GerbilSnapshotService.Diff(golden, raw);
|
||||
ov.LastImportDiffJson = diff.Count == 0 ? null : JsonSerializer.Serialize(diff, DiffJson);
|
||||
}
|
||||
var gerbilsById = await _db.Gerbils.ToDictionaryAsync(g => g.Id);
|
||||
int overridesApplied = 0;
|
||||
foreach (var ov in overrides)
|
||||
{
|
||||
if (gerbilsById.TryGetValue(ov.GerbilId, out var gg))
|
||||
{
|
||||
GerbilSnapshotService.ApplyOverride(gg, ov.OverrideJson);
|
||||
overridesApplied++;
|
||||
}
|
||||
}
|
||||
await _db.SaveChangesAsync();
|
||||
|
||||
var warnPart = warnings.Count > 0 ? $" {warnings.Count} warning(s): {string.Join(" | ", warnings)}" : "";
|
||||
return $"Ingestion successful! Contacts +{contactsAdded}/~{contactsUpdated} (−{contactsDeleted} stale), " +
|
||||
$"{data.Litters.Count} litters (−{littersDeleted} stale), {data.Gerbils.Count} gerbils (−{gerbilsDeleted} stale), " +
|
||||
$"{data.GerbilPhotos.Count} photos, {contractsAdded} sale contracts ({contractAnimalsSkipped} animal links skipped), " +
|
||||
$"{overridesApplied} override(s) applied (freeze).{warnPart}";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -319,7 +357,7 @@ namespace GerbilManagerWebAPI.Import
|
||||
public required string FileName { get; set; }
|
||||
public string? Caption { get; set; }
|
||||
public int SortOrder { get; set; }
|
||||
|
||||
|
||||
[JsonPropertyName("_source_path")]
|
||||
public string? SourcePath { get; set; }
|
||||
}
|
||||
|
||||
@@ -105,6 +105,11 @@ namespace GerbilManagerWebAPI.Import.Rpro3
|
||||
public string? Color { get; set; }
|
||||
public string? Dob { get; set; } // ISO yyyy-MM-dd
|
||||
public string? Origin { get; set; }
|
||||
/// <summary>Roher Gencode (Fcode, z. B. „aa Cc[h] DD ee Gg P- spsp"). Überschreibt den
|
||||
/// automatisch aus den Cluster-Mitgliedern gewählten Gencode — nötig, wenn sich die
|
||||
/// Varianten im Gencode unterscheiden (z. B. C-Locus c[h] vs. c[chm]) und die Züchterin
|
||||
/// den korrekten Genotyp vorgibt.</summary>
|
||||
public string? Genotype { get; set; }
|
||||
public bool? Resident { get; set; }
|
||||
/// <summary>Zusatz, der an die Notizen des Tiers angehängt wird (z. B. abweichendes DOB).</summary>
|
||||
public string? Note { get; set; }
|
||||
|
||||
@@ -427,6 +427,7 @@ namespace GerbilManagerWebAPI.Import.Rpro3
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(ov.Color)) farbe = ov.Color;
|
||||
if (!string.IsNullOrWhiteSpace(ov.Origin)) origin = ov.Origin;
|
||||
if (!string.IsNullOrWhiteSpace(ov.Genotype)) fcode = ov.Genotype;
|
||||
if (!string.IsNullOrWhiteSpace(ov.Dob) && DateOnly.TryParse(ov.Dob, out var od)) dob = od;
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user