using GerbilManagerWebAPI.Import; using GerbilManagerWebAPI.Models; using Microsoft.Data.Sqlite; using Microsoft.EntityFrameworkCore; namespace GerbilManager.Tests { /// /// FEAT-8c loader tests against a small JSON fixture + an in-memory DB (never Julian's /// instance). Covers categorisation, the quarantine policy, genotype composition, /// Farbschlag matching, high-confidence linking, and idempotency. /// public class ImportServiceTests : IDisposable { private readonly string _dir; public ImportServiceTests() { _dir = Path.Combine(Path.GetTempPath(), "feat8c-" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(_dir); File.WriteAllText(Path.Combine(_dir, "litters.json"), LittersJson); File.WriteAllText(Path.Combine(_dir, "animals.json"), AnimalsJson); } public void Dispose() { try { Directory.Delete(_dir, recursive: true); } catch { } } private ApplicationContext NewDb() { var opts = new DbContextOptionsBuilder() .UseInMemoryDatabase("feat8c-" + Guid.NewGuid().ToString("N")) .Options; var db = new ApplicationContext(opts); db.Database.EnsureCreated(); // applies the 73-variety HasData seed return db; } [Fact] public async Task DryRun_categorises_without_writing() { using var db = NewDb(); var report = await new ImportService(db, _dir, _dir).RunAsync(execute: false); Assert.False(report.Executed); Assert.Equal(2, report.Litters.InSource); Assert.Equal(2, report.Litters.Created); // 4 animals: a1 (hoch) + a4 (date-only) loadable; a2 conflict, a3 stub quarantined Assert.Equal(4, report.Animals.InSource); Assert.Equal(2, report.Animals.Created); Assert.Equal(1, report.Animals.LinkedToLitter); // only a1 (hoch) Assert.Equal(1, report.Animals.FarbschlagMatched); // a1 -> Agouti Assert.Equal(1, report.Animals.Quarantined.Conflicts); Assert.Equal(1, report.Animals.Quarantined.Stubs); Assert.Equal(1, report.Animals.Quarantined.DateOnlyLinks); // nothing written in dry-run Assert.Equal(0, await db.Gerbils.CountAsync()); Assert.Equal(0, await db.Litters.CountAsync()); } [Fact] public async Task Execute_loads_conflict_free_and_links_high_confidence() { using var db = NewDb(); var report = await new ImportService(db, _dir, _dir).RunAsync(execute: true); Assert.True(report.Executed); Assert.Equal(2, await db.Litters.CountAsync()); Assert.Equal(2, await db.Gerbils.CountAsync()); // a1, a4 only // conflict + stub never loaded Assert.False(await db.Gerbils.AnyAsync(g => g.ExternalRef == "a2")); Assert.False(await db.Gerbils.AnyAsync(g => g.ExternalRef == "a3")); var a1 = await db.Gerbils.SingleAsync(g => g.ExternalRef == "a1"); Assert.NotNull(a1.LitterId); // high-confidence link Assert.Equal("FEAT-8 Stammbaum/Wurfchronik", a1.ImportSource); Assert.NotNull(a1.ColorVarietyId); // Agouti matched Assert.Contains("aa CC", a1.Genotype); // composed from mapped8locus Assert.Contains("??", a1.Genotype!); // missing loci -> ?? Assert.Contains("eef", a1.Genotype!); // e^f caret stripped Assert.Contains("RawGenotype", a1.RawImportData!); // raw verbatim preserved var a4 = await db.Gerbils.SingleAsync(g => g.ExternalRef == "a4"); Assert.Null(a4.LitterId); // date-only link quarantined // PairingCode carried through Assert.True(await db.Litters.AnyAsync(l => l.PairingCode == "G01/ZdkC")); } [Fact] public async Task Execute_is_idempotent() { using var db = NewDb(); await new ImportService(db, _dir, _dir).RunAsync(execute: true); var second = await new ImportService(db, _dir, _dir).RunAsync(execute: true); Assert.Equal(0, second.Animals.Created); Assert.Equal(0, second.Litters.Created); Assert.Equal(2, await db.Gerbils.CountAsync()); Assert.Equal(2, await db.Litters.CountAsync()); } [Fact] public async Task Execute_persists_deaf_flag_and_preserves_sls_and_tags() { using var db = NewDb(); await new ImportService(db, _dir, _dir).RunAsync(execute: true); var a1 = await db.Gerbils.SingleAsync(g => g.ExternalRef == "a1"); // GEN-3b: deafness is a persisted phenotype flag (NOT a genotype locus). Assert.True(a1.IsDeaf); // Sls (2nd spotting locus) + provenance tags are preserved in RawImportData // (kept out of the 8-locus compact Genotype contract until GEN-3a adopts them). Assert.Contains("Sls", a1.RawImportData!); Assert.Contains("WFNZ", a1.RawImportData!); // GEN-3a contract: a WP/Sls carrier appends the trailing "Slsl" token (Kevin). Assert.EndsWith("Slsl", a1.Genotype!); } [Fact] public void ComposeGenotype_appends_Slsl_only_for_carriers() { // wild-type sl/sl is omitted -> plain 8-locus string var wild = new SourceGenotype { Mapped8locus = new() { ["A"] = new() { "a", "a" }, ["Sls"] = new() { "sl", "sl" } } }; Assert.DoesNotContain("Sl", ImportService.ComposeGenotype(wild)); // WP heterozygote -> trailing Slsl var wp = new SourceGenotype { Mapped8locus = new() { ["A"] = new() { "a", "a" }, ["Sls"] = new() { "Sl", "sl" } } }; Assert.EndsWith("Slsl", ImportService.ComposeGenotype(wp)); } [Fact] public async Task Synthesizes_litter_from_chart_parentRefs_links_offspring_and_parents() { // Offspring 'C' has chart-position parentRefs to father 'Papa' (loaded) and mother // 'Mama' (loaded), but NO Wurfchronik litterRef -> the loader must synthesize a derived // litter, link C to it, and set the litter's Father/Mother (PEDIGREE-LINK structural fix). var dir = Path.Combine(Path.GetTempPath(), "pedlink-" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(dir); try { File.WriteAllText(Path.Combine(dir, "litters.json"), "[]"); File.WriteAllText(Path.Combine(dir, "animals.json"), """ [ {"id":"papa","name":"Papa v.d. Test","dob":"01.01.2022","death":"","farbschlag":"","gender":"male","zuchtCanon":"test", "genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":false}, {"id":"mama","name":"Mama v.d. Test","dob":"02.02.2022","death":"","farbschlag":"","gender":"female","zuchtCanon":"test", "genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":false}, {"id":"ext","name":"Fremd of Foreign","dob":"03.03.2022","death":"","farbschlag":"","zuchtCanon":"foreign", "genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":false}, {"id":"c","name":"C","dob":"29.04.2024","death":"","farbschlag":"","zuchtCanon":"kleinechaote", "genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":false, "parentRefs":[ {"name":"Papa v.d. Test","dob":"01.01.2022","roleGuess":"father","method":"chart-position","confidence":"medium"}, {"name":"Mama v.d. Test","dob":"02.02.2022","roleGuess":"mother","method":"chart-position","confidence":"medium"} ]} ] """); using var db = NewDb(); var report = await new ImportService(db, dir, dir).RunAsync(execute: true); Assert.Equal(1, report.Animals.ParentLinksFromChart); Assert.Equal(1, report.Litters.DerivedFromChart); var papa = await db.Gerbils.SingleAsync(g => g.ExternalRef == "papa"); var mama = await db.Gerbils.SingleAsync(g => g.ExternalRef == "mama"); var c = await db.Gerbils.SingleAsync(g => g.ExternalRef == "c"); Assert.NotNull(c.LitterId); // C no longer "unbekannt" // box-colour sex flows through (blue=male, white=female) Assert.Equal(Gender.male, papa.Gender); Assert.Equal(Gender.female, mama.Gender); var litter = await db.Litters.SingleAsync(l => l.Id == c.LitterId); Assert.Equal(papa.Id, litter.FatherId); Assert.Equal(mama.Id, litter.MotherId); Assert.Contains("Diagramm", litter.Notes!); // transparent + reversible // OWNERSHIP/RESIDENCY: C is Clan (rule a); its foreign-Zucht parents flip to // resident (rule b); the unrelated foreign animal stays external. Assert.True(c.IsResident); // rule (a) Assert.True(papa.IsResident); // rule (b) parent exception Assert.True(mama.IsResident); // rule (b) Assert.False((await db.Gerbils.SingleAsync(g => g.ExternalRef == "ext")).IsResident); Assert.NotNull(report.Residency); Assert.Equal(3, report.Residency!.Resident); Assert.Equal(1, report.Residency.External); Assert.Equal(2, report.Residency.FlippedByParentRule); } finally { try { Directory.Delete(dir, recursive: true); } catch { } } } [Fact] public async Task Execute_on_relational_db_with_new_chart_parents_does_not_FK_throw() { // Regression for FK_Litters_Gerbils_FatherId: a derived litter references parent gerbils // created in the SAME run, so they must be inserted before the litter. The EF in-memory // provider does NOT enforce FKs (which masked the bug), so this uses SQLite — which does. var dir = Path.Combine(Path.GetTempPath(), "fkfix-" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(dir); using var conn = new SqliteConnection("DataSource=:memory:"); conn.Open(); try { File.WriteAllText(Path.Combine(dir, "litters.json"), "[]"); File.WriteAllText(Path.Combine(dir, "animals.json"), """ [ {"id":"papa","name":"Papa v.d. Test","dob":"01.01.2022","death":"","farbschlag":"", "genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":false}, {"id":"mama","name":"Mama v.d. Test","dob":"02.02.2022","death":"","farbschlag":"", "genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":false}, {"id":"c","name":"C","dob":"29.04.2024","death":"","farbschlag":"", "genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":false, "parentRefs":[ {"name":"Papa v.d. Test","dob":"01.01.2022","roleGuess":"father","method":"chart-position","confidence":"medium"}, {"name":"Mama v.d. Test","dob":"02.02.2022","roleGuess":"mother","method":"chart-position","confidence":"medium"} ]} ] """); var opts = new DbContextOptionsBuilder().UseSqlite(conn).Options; using var db = new ApplicationContext(opts); await db.Database.EnsureCreatedAsync(); // schema WITH enforced FK constraints // must not throw FK_Litters_Gerbils_FatherId (parents inserted before the litter) var report = await new ImportService(db, dir, dir).RunAsync(execute: true); var c = await db.Gerbils.SingleAsync(g => g.ExternalRef == "c"); Assert.NotNull(c.LitterId); var litter = await db.Litters.SingleAsync(l => l.Id == c.LitterId); Assert.Equal((await db.Gerbils.SingleAsync(g => g.ExternalRef == "papa")).Id, litter.FatherId); Assert.Equal(0, report.Litters.ParentFksDropped); Assert.Equal(0, report.Litters.DerivedSkipped); } finally { try { Directory.Delete(dir, recursive: true); } catch { } } } [Fact] public async Task Derived_litter_with_quarantined_parent_leaves_FK_null_no_throw() { // A chart parentRef pointing to a QUARANTINED (conflict) animal must not become an FK — // the derived litter keeps that side null; if both sides are unresolvable, no litter. var dir = Path.Combine(Path.GetTempPath(), "fkq-" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(dir); using var conn = new SqliteConnection("DataSource=:memory:"); conn.Open(); try { File.WriteAllText(Path.Combine(dir, "litters.json"), "[]"); File.WriteAllText(Path.Combine(dir, "animals.json"), """ [ {"id":"badpa","name":"BadPapa v.d. Test","dob":"01.01.2022","death":"","farbschlag":"", "genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":true}, {"id":"goodma","name":"GoodMama v.d. Test","dob":"02.02.2022","death":"","farbschlag":"", "genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":false}, {"id":"c2","name":"C2","dob":"29.04.2024","death":"","farbschlag":"", "genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":false, "parentRefs":[ {"name":"BadPapa v.d. Test","dob":"01.01.2022","roleGuess":"father","method":"chart-position","confidence":"medium"}, {"name":"GoodMama v.d. Test","dob":"02.02.2022","roleGuess":"mother","method":"chart-position","confidence":"medium"} ]} ] """); var opts = new DbContextOptionsBuilder().UseSqlite(conn).Options; using var db = new ApplicationContext(opts); await db.Database.EnsureCreatedAsync(); await new ImportService(db, dir, dir).RunAsync(execute: true); // must not throw Assert.False(await db.Gerbils.AnyAsync(g => g.ExternalRef == "badpa")); // quarantined var c2 = await db.Gerbils.SingleAsync(g => g.ExternalRef == "c2"); Assert.NotNull(c2.LitterId); // still linked (via mother) var litter = await db.Litters.SingleAsync(l => l.Id == c2.LitterId); Assert.Null(litter.FatherId); // quarantined father -> null FK Assert.Equal((await db.Gerbils.SingleAsync(g => g.ExternalRef == "goodma")).Id, litter.MotherId); } finally { try { Directory.Delete(dir, recursive: true); } catch { } } } [Fact] public async Task Decision_resolved_animal_loads_and_is_counted() { // extract.py clears the conflict + sets resolvedByDecision when a human conflict-decision // un-quarantines an animal; the loader must then LOAD it and surface the count. var dir = Path.Combine(Path.GetTempPath(), "dec-" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(dir); try { File.WriteAllText(Path.Combine(dir, "litters.json"), "[]"); File.WriteAllText(Path.Combine(dir, "animals.json"), """ [ {"id":"firefly","name":"Firefly","dob":"18.12.2019","death":"","farbschlag":"Agouti", "genotype":{"mapped8locus":{"A":["A","a"],"D":["D","?"]},"rawGenotype":"Aa D-","unmappedTokens":[]}, "conflict":false,"resolvedByDecision":true} ] """); using var db = NewDb(); var report = await new ImportService(db, dir, dir).RunAsync(execute: true); Assert.Equal(1, report.Animals.ConflictsResolvedByDecision); Assert.True(await db.Gerbils.AnyAsync(g => g.ExternalRef == "firefly")); // loaded, not quarantined Assert.Equal(0, report.Animals.Quarantined.Conflicts); } finally { try { Directory.Delete(dir, recursive: true); } catch { } } } [Fact] public void ComposeGenotype_strips_carets_and_fills_missing_loci() { var g = new SourceGenotype { Mapped8locus = new() { ["A"] = new() { "a", "a" }, ["C"] = new() { "C", "c^chm" }, ["E"] = new() { "e", "e^f" }, }, }; // order A C D E G P Sp Re ; missing -> ?? Assert.Equal("aa Ccchm ?? eef ?? ?? ?? ??", ImportService.ComposeGenotype(g)); } [Fact] public async Task ParentFkBackfill_fills_null_litter_parent_on_reimport() { // Run 1: litter "Wurf A" has sire "Vater" (conflict=true — not loaded) and dam "Mutter" // (conflict=false — loaded). After run 1: litter.FatherId = null. // Run 2: sire "Vater" is no longer in conflict. Backfill must set litter.FatherId. var dir = Path.Combine(Path.GetTempPath(), "backfill-" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(dir); using var conn = new SqliteConnection("DataSource=:memory:"); conn.Open(); try { var littersJson = """ [{"id":"L-A","litterId":"A","date":"01.05.2023","damName":"Mutter [ZdkC]","sireName":"Vater [ZdkC]","totalBorn":3,"zuchtnummer":"","note":""}] """; // Run 1: Vater is in conflict -> not loaded var animals1 = """ [ {"id":"mutter","name":"Mutter [ZdkC]","dob":"01.01.2021","death":"","farbschlag":"","gender":"female","zuchtCanon":"kleinechaote", "genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":false}, {"id":"vater","name":"Vater [ZdkC]","dob":"02.02.2021","death":"","farbschlag":"","gender":"male","zuchtCanon":"kleinechaote", "genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":true}, {"id":"kind","name":"Kind [ZdkC]","dob":"01.05.2023","death":"","farbschlag":"","gender":null,"zuchtCanon":"kleinechaote", "genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":false, "litterRef":{"litterId":"L-A","method":"geburtsdatum+eltern","confidence":"hoch"}} ] """; File.WriteAllText(Path.Combine(dir, "litters.json"), littersJson); File.WriteAllText(Path.Combine(dir, "animals.json"), animals1); var opts = new DbContextOptionsBuilder().UseSqlite(conn).Options; using var db = new ApplicationContext(opts); await db.Database.EnsureCreatedAsync(); var report1 = await new ImportService(db, dir, dir).RunAsync(execute: true); Assert.Equal(0, report1.Litters.ParentFksBackfilled); var litter1 = await db.Litters.SingleAsync(l => l.Name == "Wurf A"); Assert.Null(litter1.FatherId); // Vater was quarantined -> null FK Assert.NotNull(litter1.MotherId); // Mutter was loaded -> set // Run 2: Vater is now conflict=false var animals2 = """ [ {"id":"mutter","name":"Mutter [ZdkC]","dob":"01.01.2021","death":"","farbschlag":"","gender":"female","zuchtCanon":"kleinechaote", "genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":false}, {"id":"vater","name":"Vater [ZdkC]","dob":"02.02.2021","death":"","farbschlag":"","gender":"male","zuchtCanon":"kleinechaote", "genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":false}, {"id":"kind","name":"Kind [ZdkC]","dob":"01.05.2023","death":"","farbschlag":"","gender":null,"zuchtCanon":"kleinechaote", "genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":false, "litterRef":{"litterId":"L-A","method":"geburtsdatum+eltern","confidence":"hoch"}} ] """; File.WriteAllText(Path.Combine(dir, "animals.json"), animals2); var report2 = await new ImportService(db, dir, dir).RunAsync(execute: true); Assert.Equal(1, report2.Litters.ParentFksBackfilled); // backfill happened var vater = await db.Gerbils.SingleAsync(g => g.ExternalRef == "vater"); var litter2 = await db.Litters.SingleAsync(l => l.Name == "Wurf A"); Assert.Equal(vater.Id, litter2.FatherId); // FK now set } finally { try { Directory.Delete(dir, recursive: true); } catch { } } } [Fact] public async Task ParentFkBackfill_dry_run_counts_without_writing() { // Dry-run on a DB with an existing null-parent litter should predict the backfill count. var dir = Path.Combine(Path.GetTempPath(), "backfill-dr-" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(dir); using var conn = new SqliteConnection("DataSource=:memory:"); conn.Open(); try { var littersJson = """ [{"id":"L-B","litterId":"B","date":"15.06.2023","damName":"Mami [ZdkC]","sireName":"Papi [ZdkC]","totalBorn":2,"zuchtnummer":"","note":""}] """; var animals1 = """ [ {"id":"mami","name":"Mami [ZdkC]","dob":"03.03.2021","death":"","farbschlag":"","gender":"female","zuchtCanon":"kleinechaote", "genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":false}, {"id":"papi","name":"Papi [ZdkC]","dob":"04.04.2021","death":"","farbschlag":"","gender":"male","zuchtCanon":"kleinechaote", "genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]},"conflict":true} ] """; File.WriteAllText(Path.Combine(dir, "litters.json"), littersJson); File.WriteAllText(Path.Combine(dir, "animals.json"), animals1); var opts = new DbContextOptionsBuilder().UseSqlite(conn).Options; using var db = new ApplicationContext(opts); await db.Database.EnsureCreatedAsync(); await new ImportService(db, dir, dir).RunAsync(execute: true); // run 1 // Run 2 dry-run with papi un-quarantined var animals2 = animals1.Replace("\"conflict\":true", "\"conflict\":false"); File.WriteAllText(Path.Combine(dir, "animals.json"), animals2); var dry = await new ImportService(db, dir, dir).RunAsync(execute: false); Assert.Equal(1, dry.Litters.ParentFksBackfilled); // predicted but not written var litter = await db.Litters.SingleAsync(l => l.Name == "Wurf B"); Assert.Null(litter.FatherId); // not written in dry-run } finally { try { Directory.Delete(dir, recursive: true); } catch { } } } [Theory] [InlineData("01.02.2020", 2020, 2, 1)] [InlineData("5.3.21", 2021, 3, 5)] public void ParseDate_handles_german_dates(string s, int y, int m, int d) { var date = ImportService.ParseDate(s); Assert.Equal(new DateOnly(y, m, d), date); } // ---- fixtures ---- private const string LittersJson = """ [ {"id":"L1","litterId":"A","date":"01.02.2020","damName":"Mama [X]","sireName":"Papa of Y","totalBorn":4,"zuchtnummer":"G01/ZdkC","note":"erster Wurf"}, {"id":"L2","litterId":"B","date":"05.03.2021","damName":"Oma [Z]","sireName":"Opa of W","totalBorn":2,"zuchtnummer":"G02/ZdkC","note":""} ] """; private const string AnimalsJson = """ [ {"id":"a1","name":"Kind Eins","dob":"01.02.2020","death":"","gender":null, "farbschlag":"Agouti","farbschlagVariants":["Agouti"], "genotype":{"mapped8locus":{"A":["a","a"],"C":["C","C"],"D":["D","?"],"E":["e","e^f"],"Sls":["Sl","sl"]},"rawGenotype":"aa CC D- ee[f] WP dea WFNZ","unmappedTokens":[]}, "deaf":true,"tags":["WFNZ"], "zucht":"","parentRefs":[],"photos":[],"sourceFiles":["f1"],"conflict":false, "litterRef":{"litterId":"L1","method":"geburtsdatum+eltern","confidence":"hoch"}}, {"id":"a2","name":"Streit","dob":"01.01.2019","death":"","gender":null, "farbschlag":"Schwarz","farbschlagVariants":["Schwarz"], "genotype":{"mapped8locus":{"A":["a","a"]},"rawGenotype":"aa","unmappedTokens":[]}, "zucht":"","parentRefs":[],"photos":[],"sourceFiles":["f1","f2"],"conflict":true}, {"id":"a3","name":"Namenlos","dob":"","death":"","gender":null, "farbschlag":"","farbschlagVariants":[], "genotype":{"mapped8locus":{},"rawGenotype":"","unmappedTokens":[]}, "zucht":"","parentRefs":[],"photos":[],"sourceFiles":["f1"],"conflict":false}, {"id":"a4","name":"Unsicher","dob":"05.03.2021","death":"","gender":null, "farbschlag":"","farbschlagVariants":[], "genotype":{"mapped8locus":{"A":["A","a"]},"rawGenotype":"Aa","unmappedTokens":[]}, "zucht":"","parentRefs":[],"photos":[],"sourceFiles":["f1"],"conflict":false, "litterRef":{"litterId":"L2","method":"geburtsdatum","confidence":"niedrig"}} ] """; } }