Gulum/GerbilManager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

FIX-8D-DEDUP: P0 duplicate ExternalRef + WsCode→litter mismatch

Browse Source 
FIX-1 DEDUP (P0): Two same-name siblings in one litter produced identical
ExternalRefs ('docx-{ws}-{name}-{dob}'), crashing execute on IX_Gerbils_ExternalRef.
Fix: ordinal counter per base-ref within the batch → first occurrence keeps the
base ref, subsequent ones get -2, -3 … suffix. Deterministic (JSON-order) → idempotent
re-runs find existing rows via ExternalRef path (PATH 1). HashSet guard added as a
belt-and-suspenders check.

FIX-2 TRANSACTION: execute now opens a single BeginTransactionAsync before the loop
and commits after the final SaveChangesAsync. Eager contact saves (within the tx) and
the gerbil batch save are fully atomic — crash → full rollback, no partial state.

FIX-3 LITTER LOOKUP: docx WsCode is a litter-size fraction ('4/4', '/5') — NOT a
PairingCode like 'G01/ZdkC'. The previous WsCode→PairingCode lookup was always a
no-op (hence litterLinked=0 in every dry-run). Fix: look up DB litters by LitterDob
(±5 days); link only when exactly ONE candidate exists (unambiguous, no false links).

P0 REGRESSION TEST (Test 6): SQLite + EnsureCreated → unique index enforced.
Two animals same name+litter → execute succeeds, both created, distinct ExternalRefs,
re-run = 0 new. This test would have caught the live crash.
Gate: 164/164 tests, has-pending=No, no schema change.

Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Gulum
2026-06-06 23:31:46 +02:00
parent e367657254
commit d5e8946d7d
2 changed files with 239 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

310
GerbilManagerWebAPI/Import/ImportDocxService.cs
View File

@@ -16,11 +16,13 @@ namespace GerbilManagerWebAPI.Import
///
/// Created animals: Status=GivenAway (or Deceased), IsResident=false,
/// OriginBreeder='Zucht der Kleinen Chaoten', ImportSource="docx",
/// ExternalRef = stable "docx-{ws}-{normname}-{litterDob}" key (idempotent).
/// ExternalRef = stable "docx-{ws}-{normname}-{litterDob}[-N]" key (idempotent;
/// -N ordinal suffix disambiguates same-name siblings in a litter).
///
/// NEVER overwrites a manually-set non-null value (fill-NULL-only for all fields).
///
/// Idempotent: running multiple times is safe. Each run resolves whatever is still null.
/// Idempotent: running multiple times is safe. Re-run finds existing rows via ExternalRef.
/// Execute wraps all writes in a single transaction (atomic: crash → full rollback).
/// Execute is gated by the endpoint; this service only acts when asked.
/// </summary>
public sealed class ImportDocxService
@@ -56,13 +58,15 @@ namespace GerbilManagerWebAPI.Import
return new ImportDocxReport(false, 0, 0, 0, 0, 0, 0, 0, notes);
}
// PairingCode → Litters (WS-code normalised: spaces removed)
// Litter lookup by birth date (DayNumber) → list of matching DB litters.
// NOTE: WsCode in docx is a litter-size fraction ("4/4", "/5") — NOT a PairingCode.
// Date-only lookup with uniqueness guard avoids false links (only link when
// exactly one DB litter falls within ±5 days of the docx litter DOB).
var littersInDb = await _db.Litters
.Where(l => l.PairingCode != null)
.Select(l => new { l.Id, l.Date, l.PairingCode })
.Select(l => new { l.Id, l.Date })
.ToListAsync();
var litterByWs = littersInDb
.GroupBy(l => l.PairingCode!.Replace(" ", ""))
var littersByDayNumber = littersInDb
.GroupBy(l => l.Date.DayNumber)
.ToDictionary(g => g.Key, g => g.ToList());
// normalize(name)+litterDob → Gerbil snapshot (main-import enrich path)
@@ -100,156 +104,198 @@ namespace GerbilManagerWebAPI.Import
int animalsCreated = 0, litterLinked = 0, goHomeFilled = 0, deathFilled = 0;
int ownerLinked = 0, ownerCreated = 0, skipped = 0;
foreach (var da in docxAnimals)
// Ordinal counter for collision-free ExternalRef within this batch.
// Two animals with the same base ref (same ws+name+litterDob) get -2, -3 suffixes.
var externalRefOrdinals = new Dictionary<string, int>();
// Belt-and-suspenders: guard against adding the same ExternalRef twice in one run.
var batchRefs = new HashSet<string>();
// --- Planning pass (dry-run counts + execute writes) ---
// Execute path is wrapped in a single transaction for atomicity.
Microsoft.EntityFrameworkCore.Storage.IDbContextTransaction? tx = null;
if (execute)
tx = await _db.Database.BeginTransactionAsync();
try
{
if (string.IsNullOrWhiteSpace(da.Name)) { skipped++; continue; }
var externalRef = DocxExternalRef(da);
// Resolve litter by WS-code + approximate birth date
Guid? litterId = null;
if (!string.IsNullOrWhiteSpace(da.WsCode) && !string.IsNullOrWhiteSpace(da.LitterDob))
foreach (var da in docxAnimals)
{
var litterDob = ParseDate(da.LitterDob);
if (litterDob is not null && litterByWs.TryGetValue(da.WsCode.Replace(" ", ""), out var cands))
if (string.IsNullOrWhiteSpace(da.Name)) { skipped++; continue; }
// Collision-free ExternalRef: ordinal suffix for same-name siblings
var baseRef = DocxExternalRefBase(da);
externalRefOrdinals.TryGetValue(baseRef, out var ord);
ord++;
externalRefOrdinals[baseRef] = ord;
var externalRef = ord == 1 ? baseRef : $"{baseRef}-{ord}";
// Resolve litter: date ±5 days, unambiguous (exactly one candidate)
Guid? litterId = null;
if (!string.IsNullOrWhiteSpace(da.LitterDob))
{
var match = cands.FirstOrDefault(l =>
Math.Abs(l.Date.DayNumber - litterDob.Value.DayNumber) <= 5);
litterId = match?.Id;
var litterDob = ParseDate(da.LitterDob);
if (litterDob is not null)
{
var candidates = new List<Guid>();
for (int delta = -5; delta <= 5; delta++)
{
if (littersByDayNumber.TryGetValue(litterDob.Value.DayNumber + delta, out var cl))
candidates.AddRange(cl.Select(l => l.Id));
}
if (candidates.Count == 1)
litterId = candidates[0];
// If 0 or >1 candidates: no link (avoid false links)
}
}
}
// Animal DOB = litter birth date (docx has no per-animal DOB)
var animalDob = litterId is not null
? (await _db.Litters.Where(l => l.Id == litterId).Select(l => (DateOnly?)l.Date).FirstOrDefaultAsync())
: ParseDate(da.LitterDob);
// Animal DOB = litter birth date (docx has no per-animal DOB)
var animalDob = litterId is not null
? (await _db.Litters.Where(l => l.Id == litterId).Select(l => (DateOnly?)l.Date).FirstOrDefaultAsync())
: ParseDate(da.LitterDob);
if (animalDob is null) { skipped++; continue; }
if (animalDob is null) { skipped++; continue; }
var goHomeDate = ParseDate(da.AbgabeDate);
var deathDate = ParseDate(da.DeathDate);
var goHomeDate = ParseDate(da.AbgabeDate);
var deathDate = ParseDate(da.DeathDate);
// Resolve receiver contact (lookup-or-create; shared by all paths)
Guid? receiverId = null;
if (!string.IsNullOrWhiteSpace(da.Owner))
{
var normOwner = NormalizeName(da.Owner);
if (contactByNorm.TryGetValue(normOwner, out var existingContactId))
// Resolve receiver contact (lookup-or-create; shared by all paths)
Guid? receiverId = null;
if (!string.IsNullOrWhiteSpace(da.Owner))
{
receiverId = existingContactId;
ownerLinked++;
var normOwner = NormalizeName(da.Owner);
if (contactByNorm.TryGetValue(normOwner, out var existingContactId))
{
receiverId = existingContactId;
ownerLinked++;
}
else
{
ownerCreated++;
if (execute)
{
var newContact = new Contact { Id = Guid.NewGuid(), Name = da.Owner.Trim() };
_db.Contacts.Add(newContact);
await _db.SaveChangesAsync(); // flush within the outer tx
receiverId = newContact.Id;
contactByNorm[normOwner] = receiverId.Value;
}
}
}
else
// ── PATH 1: previously docx-created animal (idempotent re-run) ──────────
if (docxExternalRefs.TryGetValue(externalRef, out var prevSnap))
{
ownerCreated++;
bool willLink = litterId is not null && prevSnap.LitterId is null;
bool willHome = goHomeDate is not null && prevSnap.GoHomeDate is null;
bool willDeath = deathDate is not null && prevSnap.DateOfDeath is null;
if (willLink) litterLinked++;
if (willHome) goHomeFilled++;
if (willDeath) deathFilled++;
if (execute)
{
var newContact = new Contact { Id = Guid.NewGuid(), Name = da.Owner.Trim() };
_db.Contacts.Add(newContact);
await _db.SaveChangesAsync(); // persist before any Gerbil FK reference
receiverId = newContact.Id;
contactByNorm[normOwner] = receiverId.Value;
var row = await _db.Gerbils.FirstOrDefaultAsync(g => g.Id == prevSnap.Id);
if (row is null) continue;
if (willLink) row.LitterId = litterId;
if (receiverId is not null && row.ReceiverContactId is null) row.ReceiverContactId = receiverId;
if (willHome) row.GoHomeDate = goHomeDate;
if (willDeath)
{
row.DateOfDeath = deathDate;
if (!string.IsNullOrWhiteSpace(da.DeathCause) && row.CauseOfDeath is null)
row.CauseOfDeath = da.DeathCause.Trim();
}
}
continue;
}
}
// ── PATH 1: previously docx-created animal (idempotent re-run) ──────────
if (docxExternalRefs.TryGetValue(externalRef, out var prevSnap))
{
bool willLink = litterId is not null && prevSnap.LitterId is null;
bool willHome = goHomeDate is not null && prevSnap.GoHomeDate is null;
bool willDeath = deathDate is not null && prevSnap.DateOfDeath is null;
// ── PATH 2: main-import animal matched by name+dob ───────────────────────
var nameKey = NameDobKey(da.Name, animalDob.Value);
if (gerbilByKey.TryGetValue(nameKey, out var gerbilCands))
{
var gerbilSnap = gerbilCands.FirstOrDefault(g => g.LitterId == null)
?? gerbilCands.First();
if (willLink) litterLinked++;
if (willHome) goHomeFilled++;
if (willDeath) deathFilled++;
bool willLink = litterId is not null && gerbilSnap.LitterId is null;
bool willHome = goHomeDate is not null && gerbilSnap.GoHomeDate is null;
bool willDeath = deathDate is not null && gerbilSnap.DateOfDeath is null;
if (willLink) litterLinked++;
if (willHome) goHomeFilled++;
if (willDeath) deathFilled++;
if (execute)
{
var row = await _db.Gerbils.FirstOrDefaultAsync(g => g.Id == gerbilSnap.Id);
if (row is null) continue;
if (willLink) row.LitterId = litterId;
if (receiverId is not null && row.ReceiverContactId is null) row.ReceiverContactId = receiverId;
if (willHome) row.GoHomeDate = goHomeDate;
if (willDeath)
{
row.DateOfDeath = deathDate;
if (!string.IsNullOrWhiteSpace(da.DeathCause) && row.CauseOfDeath is null)
row.CauseOfDeath = da.DeathCause.Trim();
}
}
continue;
}
// ── PATH 3: no match → CREATE ────────────────────────────────────────────
// Belt-and-suspenders: ordinal should ensure uniqueness, but guard anyway
if (!batchRefs.Add(externalRef)) { skipped++; continue; }
animalsCreated++;
if (litterId is not null) litterLinked++;
if (goHomeDate is not null) goHomeFilled++;
if (deathDate is not null) deathFilled++;
if (execute)
{
var row = await _db.Gerbils.FirstOrDefaultAsync(g => g.Id == prevSnap.Id);
if (row is null) continue;
if (willLink) row.LitterId = litterId;
if (receiverId is not null && row.ReceiverContactId is null) row.ReceiverContactId = receiverId;
if (willHome) row.GoHomeDate = goHomeDate;
if (willDeath)
colorVarietyByName.TryGetValue(NormalizeName(da.Farbschlag ?? ""), out var cvId);
_db.Gerbils.Add(new Gerbil
{
row.DateOfDeath = deathDate;
if (!string.IsNullOrWhiteSpace(da.DeathCause) && row.CauseOfDeath is null)
row.CauseOfDeath = da.DeathCause.Trim();
}
Id = Guid.NewGuid(),
Name = da.Name.Trim(),
DateOfBirth = animalDob,
Gender = ParseGender(da.Gender),
Status = deathDate is not null ? GerbilStatus.Deceased : GerbilStatus.GivenAway,
LitterId = litterId,
ReceiverContactId = receiverId,
GoHomeDate = goHomeDate,
DateOfDeath = deathDate,
CauseOfDeath = string.IsNullOrWhiteSpace(da.DeathCause) ? null : da.DeathCause.Trim(),
ColorVarietyId = cvId == default ? null : cvId,
OriginBreeder = "Zucht der Kleinen Chaoten",
IsResident = false,
ImportSource = "docx",
ExternalRef = externalRef,
});
}
continue;
}
// ── PATH 2: main-import animal matched by name+dob ───────────────────────
var nameKey = NameDobKey(da.Name, animalDob.Value);
if (gerbilByKey.TryGetValue(nameKey, out var gerbilCands))
{
var gerbilSnap = gerbilCands.FirstOrDefault(g => g.LitterId == null)
?? gerbilCands.First();
// Flush all gerbil inserts + enrich updates in one shot (within the tx)
if (execute && (animalsCreated + litterLinked + goHomeFilled + deathFilled + ownerCreated) > 0)
await _db.SaveChangesAsync();
bool willLink = litterId is not null && gerbilSnap.LitterId is null;
bool willHome = goHomeDate is not null && gerbilSnap.GoHomeDate is null;
bool willDeath = deathDate is not null && gerbilSnap.DateOfDeath is null;
if (willLink) litterLinked++;
if (willHome) goHomeFilled++;
if (willDeath) deathFilled++;
if (execute)
{
var row = await _db.Gerbils.FirstOrDefaultAsync(g => g.Id == gerbilSnap.Id);
if (row is null) continue;
if (willLink) row.LitterId = litterId;
if (receiverId is not null && row.ReceiverContactId is null) row.ReceiverContactId = receiverId;
if (willHome) row.GoHomeDate = goHomeDate;
if (willDeath)
{
row.DateOfDeath = deathDate;
if (!string.IsNullOrWhiteSpace(da.DeathCause) && row.CauseOfDeath is null)
row.CauseOfDeath = da.DeathCause.Trim();
}
}
continue;
}
// ── PATH 3: no match → CREATE ────────────────────────────────────────────
animalsCreated++;
if (litterId is not null) litterLinked++;
if (goHomeDate is not null) goHomeFilled++;
if (deathDate is not null) deathFilled++;
if (execute)
{
colorVarietyByName.TryGetValue(NormalizeName(da.Farbschlag ?? ""), out var cvId);
_db.Gerbils.Add(new Gerbil
{
Id = Guid.NewGuid(),
Name = da.Name.Trim(),
DateOfBirth = animalDob,
Gender = ParseGender(da.Gender),
Status = deathDate is not null ? GerbilStatus.Deceased : GerbilStatus.GivenAway,
LitterId = litterId,
ReceiverContactId = receiverId,
GoHomeDate = goHomeDate,
DateOfDeath = deathDate,
CauseOfDeath = string.IsNullOrWhiteSpace(da.DeathCause) ? null : da.DeathCause.Trim(),
ColorVarietyId = cvId == default ? null : cvId,
OriginBreeder = "Zucht der Kleinen Chaoten",
IsResident = false,
ImportSource = "docx",
ExternalRef = externalRef,
});
}
if (tx is not null) await tx.CommitAsync();
}
catch
{
// tx.DisposeAsync (in finally) rolls back if not committed
throw;
}
finally
{
if (tx is not null) await tx.DisposeAsync();
}
if (execute && (animalsCreated + litterLinked + goHomeFilled + deathFilled + ownerCreated) > 0)
await _db.SaveChangesAsync();
notes.Add($"Quelle: {docxLitters.Count} Würfe, {docxAnimals.Count} Tier-Zeilen aus der docx.");
notes.Add($"Neu angelegt: {animalsCreated} Jungtiere (abgegeben, nicht in Stammbäumen).");
notes.Add($"Litter-Links: {litterLinked} Tiere einem Wurf zugeordnet.");
notes.Add($"Litter-Links: {litterLinked} Tiere einem Wurf zugeordnet (DOB-Match ±5 Tage, eindeutig).");
notes.Add($"Abnehmer: {ownerLinked} bestehende Kontakte verknüpft, {ownerCreated} neue Kontakte angelegt.");
notes.Add($"GoHomeDate: {goHomeFilled} Abgabe-Daten nachgetragen.");
notes.Add($"Tod-Datum: {deathFilled} Todesdaten nachgetragen.");
@@ -295,12 +341,12 @@ namespace GerbilManagerWebAPI.Import
}
/// <summary>
/// Stable ExternalRef key for a docx animal row — used for idempotency across re-runs.
/// Format: "docx-{wsCode}-{normname}-{litterDob}" (lowercased, spaces/punctuation stripped).
/// Base ExternalRef key (before ordinal disambiguation). Two animals sharing the same
/// ws+name+litterDob get this same base; the caller appends -2, -3 … for duplicates.
/// </summary>
internal static string DocxExternalRef(DocxAnimal da)
internal static string DocxExternalRefBase(DocxAnimal da)
{
var ws = da.WsCode.Replace(" ", "").ToLowerInvariant();
var ws = (da.WsCode ?? "").Replace(" ", "").ToLowerInvariant();
var name = System.Text.RegularExpressions.Regex.Replace(
(da.Name ?? "").ToLowerInvariant(), @"[^a-z0-9äöüß]", "");
return $"docx-{ws}-{name}-{da.LitterDob}";