Gulum/GerbilManager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

REVIEW-FIXES-2: DB-4 (Deutsche Collation ä/ö/ü) + DB-5 (Litter.ExternalRef)

Browse Source 
DB-4 (medium): ICU German collation de-x-icu auf deutschen Textspalten
  Gerbil.Name, Gerbil.NameSearch, Gerbil.OriginBreeder, ColorVariety.Name,
  Contact.Name. Korrekte ä/ö/ü-Sortierung + locale-aware lower() in Postgres.
  Nur bei Npgsql (Database.ProviderName check) — SQLite-Testhost kennt keine
  eigenen Collation-Namen und wuerde EnsureCreated fail lassen.
  Migration AddGermanCollation (ALTER COLUMN ... TYPE text COLLATE de-x-icu).

DB-5 (low): Litter.ExternalRef + filtered unique index + Importer-Anbindung
  Litter.ExternalRef (string?, nullable) = stable import source id (sl.Id).
  Filtered unique index WHERE ExternalRef IS NOT NULL (wie DB-1 fuer Gerbil).
  Migration AddLitterExternalRef.
  Importer (ImportService.cs): setzt ExternalRef = sl.Id beim Litter-Insert;
  prueft existingLitterExtRefSet als PRIMAEREN Idempotenz-Key (ExternalRef),
  Name+Date-Key als Fallback fuer Wuerfe ohne ExternalRef aus frueheren Laeufen.
  +Test DB5_Litter_ExternalRef_set_and_used_for_idempotency (mutiert den Namen
  nach erstem Import, prueft dass Re-Import via ExternalRef erkennt).

GATE: 140/140 C#, has-pending=No.
This commit is contained in:
Gulum
2026-06-06 19:49:50 +02:00
parent e6b513ed5d
commit ea3596ca0d
9 changed files with 3047 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
GerbilManagerWebAPI/Import/ImportService.cs
View File

@@ -71,17 +71,21 @@ namespace GerbilManagerWebAPI.Import
else if (conf == "niedrig") dateOnly++;
}
// existing rows (idempotency). Gerbils carry ExternalRef; Litters have no such
// column, so we key litter idempotency on the stable (Name + Date) pair instead.
// existing rows (idempotency).
var existingGerbilExtRefs = await _db.Gerbils
.Where(g => g.ExternalRef != null)
.Select(g => g.ExternalRef!).ToListAsync();
var existingGerbilSet = existingGerbilExtRefs.ToHashSet();
var existingLitterKeys = await _db.Litters
.Select(l => new { l.Name, l.Date }).ToListAsync();
var existingLitterKeySet = existingLitterKeys
// DB-5: Litters now carry ExternalRef (= source litter id from extract.py).
// Primary idempotency: ExternalRef. Fallback: Name+Date for litters created before DB-5.
var existingLitterData = await _db.Litters
.Select(l => new { l.Name, l.Date, l.ExternalRef }).ToListAsync();
var existingLitterKeySet = existingLitterData
.Select(x => $"{x.Name}|{x.Date:yyyy-MM-dd}").ToHashSet();
var existingLitterExtRefSet = existingLitterData
.Where(x => x.ExternalRef != null)
.Select(x => x.ExternalRef!).ToHashSet();
// colour-variety name -> id (case-insensitive)
var varieties = await _db.ColorVarieties.Select(v => new { v.Id, v.Name }).ToListAsync();
@@ -94,10 +98,10 @@ namespace GerbilManagerWebAPI.Import
var damNames = litters.Select(l => Normalize(StripZucht(l.DamName))).Where(s => s.Length > 0).ToHashSet();
// ---- litters: create map source.id -> Litter (for high-confidence animal links) ----
// Idempotency (DB-5): ExternalRef-match is primary (stable source id); Name+Date is the
// fallback for litters created before DB-5 (those have ExternalRef=null in the DB).
// COUNTER-BUG FIX: undated litters (31 in the Wurfchronik) have no parseable date,
// so their existingLitterKeySet key was always "" → they were always counted as
// "created" even though the execute block skipped them (date is DateOnly d = false).
// Fix: skip undated litters early — they can never be created or linked to animals.
// so skip them early — they can never be created or linked to animals.
int littersCreated = 0, littersExisting = 0, littersWithoutDate = 0;
var litterIdMap = new Dictionary<string, Guid>(); // source litter id -> Litter.Id
foreach (var sl in litters)
@@ -107,7 +111,10 @@ namespace GerbilManagerWebAPI.Import
var name = $"Wurf {sl.LitterId}".Trim();
var key = $"{name}|{date:yyyy-MM-dd}";
if (existingLitterKeySet.Contains(key)) { littersExisting++; continue; }
// DB-5: check ExternalRef first (stable, source-id-based); fall back to Name+Date
// for litters imported before ExternalRef existed (those have ExternalRef = null).
if (existingLitterExtRefSet.Contains(sl.Id) || existingLitterKeySet.Contains(key))
{ littersExisting++; continue; }
var id = Guid.NewGuid();
litterIdMap[sl.Id] = id;
@@ -122,6 +129,7 @@ namespace GerbilManagerWebAPI.Import
TotalBorn = sl.TotalBorn,
Notes = string.IsNullOrWhiteSpace(sl.Note) ? null : sl.Note,
PairingCode = string.IsNullOrWhiteSpace(sl.Zuchtnummer) ? null : sl.Zuchtnummer,
ExternalRef = sl.Id, // DB-5: stable import key for future re-imports
});
}
if (samples.Count < 8)