REVIEW-FIXES-2: DB-4 (Deutsche Collation ä/ö/ü) + DB-5 (Litter.ExternalRef)

DB-4 (medium): ICU German collation de-x-icu auf deutschen Textspalten
  Gerbil.Name, Gerbil.NameSearch, Gerbil.OriginBreeder, ColorVariety.Name,
  Contact.Name. Korrekte ä/ö/ü-Sortierung + locale-aware lower() in Postgres.
  Nur bei Npgsql (Database.ProviderName check) — SQLite-Testhost kennt keine
  eigenen Collation-Namen und wuerde EnsureCreated fail lassen.
  Migration AddGermanCollation (ALTER COLUMN ... TYPE text COLLATE de-x-icu).

DB-5 (low): Litter.ExternalRef + filtered unique index + Importer-Anbindung
  Litter.ExternalRef (string?, nullable) = stable import source id (sl.Id).
  Filtered unique index WHERE ExternalRef IS NOT NULL (wie DB-1 fuer Gerbil).
  Migration AddLitterExternalRef.
  Importer (ImportService.cs): setzt ExternalRef = sl.Id beim Litter-Insert;
  prueft existingLitterExtRefSet als PRIMAEREN Idempotenz-Key (ExternalRef),
  Name+Date-Key als Fallback fuer Wuerfe ohne ExternalRef aus frueheren Laeufen.
  +Test DB5_Litter_ExternalRef_set_and_used_for_idempotency (mutiert den Namen
  nach erstem Import, prueft dass Re-Import via ExternalRef erkennt).

GATE: 140/140 C#, has-pending=No.

GerbilManager.Tests/ImportServiceTests.cs

@@ -103,6 +103,31 @@ namespace GerbilManager.Tests
             Assert.Equal(2, await db.Litters.CountAsync());
         }
 
+        [Fact]
+        public async Task DB5_Litter_ExternalRef_set_and_used_for_idempotency()
+        {
+            // DB-5: litters must have ExternalRef set to the source litter id on first import,
+            // and subsequent runs must detect them via ExternalRef (not just Name+Date).
+            using var db = NewDb();
+            await new ImportService(db, _dir, _dir).RunAsync(execute: true);
+
+            // ExternalRef is set on created litters
+            var litters = await db.Litters.ToListAsync();
+            Assert.All(litters, l => Assert.NotNull(l.ExternalRef));
+            Assert.Contains(litters, l => l.ExternalRef == "L1");
+            Assert.Contains(litters, l => l.ExternalRef == "L2");
+
+            // Simulate the "Name+Date lookup would still work, but ExternalRef is now primary":
+            // mutate Name to something different — Name+Date fallback would fail, ExternalRef must catch it.
+            foreach (var l in litters) l.Name = "Geänderter Name";
+            await db.SaveChangesAsync();
+
+            // Re-import: litters detected as existing via ExternalRef even though Name changed
+            var second = await new ImportService(db, _dir, _dir).RunAsync(execute: true);
+            Assert.Equal(0, second.Litters.Created);
+            Assert.Equal(2, await db.Litters.CountAsync());
+        }
+
         [Fact]
         public async Task Execute_persists_deaf_flag_and_preserves_sls_and_tags()
         {

GerbilManagerWebAPI/ApplicationContext.cs

@@ -49,6 +49,13 @@ public class ApplicationContext : DbContext
 
     protected override void OnModelCreating(ModelBuilder modelBuilder)
     {
+        // DB-4: ICU German collation on user-visible / searched text columns.
+        // "de-x-icu" gives correct German sort order (ä between a and b) and locale-aware
+        // case-folding (lower('Ä')='ä'). Applied only on Npgsql (Postgres); SQLite does not
+        // support custom collation names and would fail EnsureCreated on the test host.
+        const string DeIcu = "de-x-icu";
+        bool isNpgsql = Database.ProviderName?.Contains("Npgsql", StringComparison.OrdinalIgnoreCase) ?? false;
+
         modelBuilder.Entity<Gerbil>(e =>
         {
             // Enums persisted as their string names (readable, Gridify-friendly).
@@ -58,6 +65,14 @@ public class ApplicationContext : DbContext
             // Residency defaults to true (own stock unless explicitly marked external).
             e.Property(g => g.IsResident).HasDefaultValue(true);
 
+            // DB-4: German collation on searched/sorted name fields (Npgsql-only).
+            if (isNpgsql)
+            {
+                e.Property(g => g.Name).UseCollation(DeIcu);
+                e.Property(g => g.NameSearch).UseCollation(DeIcu);
+                e.Property(g => g.OriginBreeder).UseCollation(DeIcu);
+            }
+
             // FEAT-14: character traits stored as a JSON text column (works on both
             // Npgsql and the SQLite test host; opaque labels, no backend vocabulary).
             var traitsConverter = new Microsoft.EntityFrameworkCore.Storage.ValueConversion.ValueConverter<List<string>, string>(
@@ -95,6 +110,12 @@ public class ApplicationContext : DbContext
                 .HasForeignKey(l => l.FatherId).OnDelete(DeleteBehavior.Restrict);
             e.HasOne(l => l.Mother).WithMany()
                 .HasForeignKey(l => l.MotherId).OnDelete(DeleteBehavior.Restrict);
+
+            // DB-5: ExternalRef = source litter id from extract.py.
+            // Unique (filtered, nulls allowed for manually-entered litters).
+            e.HasIndex(l => l.ExternalRef)
+                .IsUnique()
+                .HasFilter("\"ExternalRef\" IS NOT NULL");
         });
 
         modelBuilder.Entity<HealthRecord>(e =>
@@ -176,6 +197,15 @@ public class ApplicationContext : DbContext
         modelBuilder.Entity<MailSettings>()
             .HasData(new MailSettings { Id = GerbilManagerWebAPI.Models.MailSettings.SingletonId });
 
+        // DB-4: German collation on remaining searched/sorted text columns (Npgsql-only).
+        if (isNpgsql)
+        {
+            modelBuilder.Entity<ColorVariety>()
+                .Property(v => v.Name).UseCollation(DeIcu);
+            modelBuilder.Entity<Contact>()
+                .Property(c => c.Name).UseCollation(DeIcu);
+        }
+
         SeedColorVarieties(modelBuilder);
     }
 

GerbilManagerWebAPI/Import/ImportService.cs

@@ -71,17 +71,21 @@ namespace GerbilManagerWebAPI.Import
                 else if (conf == "niedrig") dateOnly++;
             }
 
-            // existing rows (idempotency). Gerbils carry ExternalRef; Litters have no such
-            // column, so we key litter idempotency on the stable (Name + Date) pair instead.
+            // existing rows (idempotency).
             var existingGerbilExtRefs = await _db.Gerbils
                 .Where(g => g.ExternalRef != null)
                 .Select(g => g.ExternalRef!).ToListAsync();
             var existingGerbilSet = existingGerbilExtRefs.ToHashSet();
 
-            var existingLitterKeys = await _db.Litters
-                .Select(l => new { l.Name, l.Date }).ToListAsync();
-            var existingLitterKeySet = existingLitterKeys
+            // DB-5: Litters now carry ExternalRef (= source litter id from extract.py).
+            // Primary idempotency: ExternalRef. Fallback: Name+Date for litters created before DB-5.
+            var existingLitterData = await _db.Litters
+                .Select(l => new { l.Name, l.Date, l.ExternalRef }).ToListAsync();
+            var existingLitterKeySet = existingLitterData
                 .Select(x => $"{x.Name}|{x.Date:yyyy-MM-dd}").ToHashSet();
+            var existingLitterExtRefSet = existingLitterData
+                .Where(x => x.ExternalRef != null)
+                .Select(x => x.ExternalRef!).ToHashSet();
 
             // colour-variety name -> id (case-insensitive)
             var varieties = await _db.ColorVarieties.Select(v => new { v.Id, v.Name }).ToListAsync();
@@ -94,10 +98,10 @@ namespace GerbilManagerWebAPI.Import
             var damNames = litters.Select(l => Normalize(StripZucht(l.DamName))).Where(s => s.Length > 0).ToHashSet();
 
             // ---- litters: create map source.id -> Litter (for high-confidence animal links) ----
+            // Idempotency (DB-5): ExternalRef-match is primary (stable source id); Name+Date is the
+            // fallback for litters created before DB-5 (those have ExternalRef=null in the DB).
             // COUNTER-BUG FIX: undated litters (31 in the Wurfchronik) have no parseable date,
-            // so their existingLitterKeySet key was always "" → they were always counted as
-            // "created" even though the execute block skipped them (date is DateOnly d = false).
-            // Fix: skip undated litters early — they can never be created or linked to animals.
+            // so skip them early — they can never be created or linked to animals.
             int littersCreated = 0, littersExisting = 0, littersWithoutDate = 0;
             var litterIdMap = new Dictionary<string, Guid>();    // source litter id -> Litter.Id
             foreach (var sl in litters)
@@ -107,7 +111,10 @@ namespace GerbilManagerWebAPI.Import
 
                 var name = $"Wurf {sl.LitterId}".Trim();
                 var key = $"{name}|{date:yyyy-MM-dd}";
-                if (existingLitterKeySet.Contains(key)) { littersExisting++; continue; }
+                // DB-5: check ExternalRef first (stable, source-id-based); fall back to Name+Date
+                // for litters imported before ExternalRef existed (those have ExternalRef = null).
+                if (existingLitterExtRefSet.Contains(sl.Id) || existingLitterKeySet.Contains(key))
+                { littersExisting++; continue; }
 
                 var id = Guid.NewGuid();
                 litterIdMap[sl.Id] = id;
@@ -122,6 +129,7 @@ namespace GerbilManagerWebAPI.Import
                         TotalBorn = sl.TotalBorn,
                         Notes = string.IsNullOrWhiteSpace(sl.Note) ? null : sl.Note,
                         PairingCode = string.IsNullOrWhiteSpace(sl.Zuchtnummer) ? null : sl.Zuchtnummer,
+                        ExternalRef = sl.Id,   // DB-5: stable import key for future re-imports
                     });
                 }
                 if (samples.Count < 8)

GerbilManagerWebAPI/Migrations/20260606174843_AddGermanCollation.cs

@@ -0,0 +1,133 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace GerbilManagerWebAPI.Migrations
+{
+    /// <inheritdoc />
+    public partial class AddGermanCollation : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.AddColumn<string>(
+                name: "ExternalRef",
+                table: "Litters",
+                type: "text",
+                nullable: true);
+
+            migrationBuilder.AlterColumn<string>(
+                name: "OriginBreeder",
+                table: "Gerbils",
+                type: "text",
+                nullable: true,
+                collation: "de-x-icu",
+                oldClrType: typeof(string),
+                oldType: "text",
+                oldNullable: true);
+
+            migrationBuilder.AlterColumn<string>(
+                name: "NameSearch",
+                table: "Gerbils",
+                type: "text",
+                nullable: true,
+                collation: "de-x-icu",
+                oldClrType: typeof(string),
+                oldType: "text",
+                oldNullable: true);
+
+            migrationBuilder.AlterColumn<string>(
+                name: "Name",
+                table: "Gerbils",
+                type: "text",
+                nullable: false,
+                collation: "de-x-icu",
+                oldClrType: typeof(string),
+                oldType: "text");
+
+            migrationBuilder.AlterColumn<string>(
+                name: "Name",
+                table: "Contacts",
+                type: "text",
+                nullable: false,
+                collation: "de-x-icu",
+                oldClrType: typeof(string),
+                oldType: "text");
+
+            migrationBuilder.AlterColumn<string>(
+                name: "Name",
+                table: "ColorVarieties",
+                type: "text",
+                nullable: false,
+                collation: "de-x-icu",
+                oldClrType: typeof(string),
+                oldType: "text");
+
+            migrationBuilder.CreateIndex(
+                name: "IX_Litters_ExternalRef",
+                table: "Litters",
+                column: "ExternalRef",
+                unique: true,
+                filter: "\"ExternalRef\" IS NOT NULL");
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropIndex(
+                name: "IX_Litters_ExternalRef",
+                table: "Litters");
+
+            migrationBuilder.DropColumn(
+                name: "ExternalRef",
+                table: "Litters");
+
+            migrationBuilder.AlterColumn<string>(
+                name: "OriginBreeder",
+                table: "Gerbils",
+                type: "text",
+                nullable: true,
+                oldClrType: typeof(string),
+                oldType: "text",
+                oldNullable: true,
+                oldCollation: "de-x-icu");
+
+            migrationBuilder.AlterColumn<string>(
+                name: "NameSearch",
+                table: "Gerbils",
+                type: "text",
+                nullable: true,
+                oldClrType: typeof(string),
+                oldType: "text",
+                oldNullable: true,
+                oldCollation: "de-x-icu");
+
+            migrationBuilder.AlterColumn<string>(
+                name: "Name",
+                table: "Gerbils",
+                type: "text",
+                nullable: false,
+                oldClrType: typeof(string),
+                oldType: "text",
+                oldCollation: "de-x-icu");
+
+            migrationBuilder.AlterColumn<string>(
+                name: "Name",
+                table: "Contacts",
+                type: "text",
+                nullable: false,
+                oldClrType: typeof(string),
+                oldType: "text",
+                oldCollation: "de-x-icu");
+
+            migrationBuilder.AlterColumn<string>(
+                name: "Name",
+                table: "ColorVarieties",
+                type: "text",
+                nullable: false,
+                oldClrType: typeof(string),
+                oldType: "text",
+                oldCollation: "de-x-icu");
+        }
+    }
+}

GerbilManagerWebAPI/Migrations/20260606174848_AddLitterExternalRef.cs

@@ -0,0 +1,22 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace GerbilManagerWebAPI.Migrations
+{
+    /// <inheritdoc />
+    public partial class AddLitterExternalRef : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+
+        }
+    }
+}

GerbilManagerWebAPI/Migrations/ApplicationContextModelSnapshot.cs

@@ -201,7 +201,8 @@ namespace GerbilManagerWebAPI.Migrations
 
                     b.Property<string>("Name")
                         .IsRequired()
-                        .HasColumnType("text");
+                        .HasColumnType("text")
+                        .UseCollation("de-x-icu");
 
                     b.Property<int>("SortOrder")
                         .HasColumnType("integer");
@@ -689,7 +690,8 @@ namespace GerbilManagerWebAPI.Migrations
 
                     b.Property<string>("Name")
                         .IsRequired()
-                        .HasColumnType("text");
+                        .HasColumnType("text")
+                        .UseCollation("de-x-icu");
 
                     b.Property<string>("Notes")
                         .HasColumnType("text");
@@ -777,16 +779,19 @@ namespace GerbilManagerWebAPI.Migrations
 
                     b.Property<string>("Name")
                         .IsRequired()
-                        .HasColumnType("text");
+                        .HasColumnType("text")
+                        .UseCollation("de-x-icu");
 
                     b.Property<string>("NameSearch")
-                        .HasColumnType("text");
+                        .HasColumnType("text")
+                        .UseCollation("de-x-icu");
 
                     b.Property<string>("Notes")
                         .HasColumnType("text");
 
                     b.Property<string>("OriginBreeder")
-                        .HasColumnType("text");
+                        .HasColumnType("text")
+                        .UseCollation("de-x-icu");
 
                     b.Property<Guid?>("OriginContactId")
                         .HasColumnType("uuid");
@@ -894,6 +899,9 @@ namespace GerbilManagerWebAPI.Migrations
                     b.Property<DateOnly?>("ExpectedGoHomeDate")
                         .HasColumnType("date");
 
+                    b.Property<string>("ExternalRef")
+                        .HasColumnType("text");
+
                     b.Property<Guid?>("FatherId")
                         .HasColumnType("uuid");
 
@@ -915,6 +923,10 @@ namespace GerbilManagerWebAPI.Migrations
 
                     b.HasKey("Id");
 
+                    b.HasIndex("ExternalRef")
+                        .IsUnique()
+                        .HasFilter("\"ExternalRef\" IS NOT NULL");
+
                     b.HasIndex("FatherId");
 
                     b.HasIndex("MotherId");

GerbilManagerWebAPI/Models/Litter.cs

@@ -25,5 +25,10 @@ namespace GerbilManagerWebAPI.Models
         /// <summary>Zuchtnummer der Verpaarung (Wurfchronik col H) — pairing-level code;
         /// litters sharing it are the same Zuchtpaar. Set by the FEAT-8 import.</summary>
         public string? PairingCode { get; set; }
+
+        /// <summary>DB-5: stable import source id (extract.py litter Id). Unique (filtered,
+        /// nulls allowed for manually-entered litters). Primary idempotency key for re-imports;
+        /// Name+Date is the fallback for litters created before this column existed.</summary>
+        public string? ExternalRef { get; set; }
     }
 }