Gulum
09f11cb4e3
- Gerbil.NameSearch (lowercase, strip whitespace/.-_) auto-synced in ApplicationContext .SaveChanges; Gridify-filterable so 'clan kleine chaoten' matches 'Clan-Kleine-Chaoten' (client strips separators the same way). GerbilSearch.Normalize shared helper. - Gerbil.OriginBreeder (free-text Herkunft) on DTO+Input, set by the FEAT-8 import from the source Zucht (imported animals have no OriginContact). Gridify-filterable. - GET /gerbils/breeders: distinct non-empty OriginBreeder values for the Herkunft dropdown. - Migration AddSearchFields (+ NameSearch backfill SQL for existing rows).
303 lines
14 KiB
C#
303 lines
14 KiB
C#
using System.Text.Json;
|
||
using System.Text.RegularExpressions;
|
||
using GerbilManagerWebAPI.Models;
|
||
using Microsoft.EntityFrameworkCore;
|
||
|
||
namespace GerbilManagerWebAPI.Import
|
||
{
|
||
/// <summary>
|
||
/// FEAT-8c import loader. Consumes tools/import/output (animals.json + litters.json
|
||
/// produced by extract.py) and loads conflict-free data into the database:
|
||
/// litters first (Wurfchronik = authoritative), then animals matched onto them.
|
||
///
|
||
/// Load policy:
|
||
/// - Litters: all created (idempotent by ExternalRef = source litter id).
|
||
/// - Animals: created when they have a DOB and are NOT in conflict. The birth-litter
|
||
/// link is set only for HIGH-confidence matches (litterRef.confidence == "hoch");
|
||
/// date-only/ambiguous links are quarantined (animal loads with LitterId = null).
|
||
/// - QUARANTINED (never loaded): conflicts + stubs (no DOB) — await the wife's review.
|
||
/// Idempotent: re-running matches on ExternalRef and skips existing rows.
|
||
/// Execute is gated by the endpoint; this service only acts when asked.
|
||
/// </summary>
|
||
public sealed class ImportService
|
||
{
|
||
private static readonly string[] LocusOrder = { "A", "C", "D", "E", "G", "P", "Sp", "Re" };
|
||
private static readonly JsonSerializerOptions Json = new()
|
||
{
|
||
PropertyNameCaseInsensitive = true,
|
||
};
|
||
private const string ImportSourceTag = "FEAT-8 Stammbaum/Wurfchronik";
|
||
|
||
private readonly ApplicationContext _db;
|
||
private readonly string _sourceDir;
|
||
private readonly string _photoRoot;
|
||
|
||
/// <summary>Test-friendly constructor with explicit paths.</summary>
|
||
public ImportService(ApplicationContext db, string sourceDir, string photoRoot)
|
||
{
|
||
_db = db;
|
||
_sourceDir = sourceDir;
|
||
_photoRoot = photoRoot;
|
||
}
|
||
|
||
public ImportService(ApplicationContext db, IConfiguration config, IWebHostEnvironment env)
|
||
: this(db,
|
||
config["Import:SourcePath"]
|
||
?? Path.GetFullPath(Path.Combine(env.ContentRootPath, "..", "tools", "import", "output")),
|
||
config["Photos:RootPath"] ?? Path.Combine(env.ContentRootPath, "photo-storage"))
|
||
{
|
||
}
|
||
|
||
public async Task<ImportReport> RunAsync(bool execute)
|
||
{
|
||
var notes = new List<string>();
|
||
var samples = new List<string>();
|
||
|
||
var animals = Load<List<SourceAnimal>>("animals.json") ?? new();
|
||
var litters = Load<List<SourceLitter>>("litters.json") ?? new();
|
||
if (animals.Count == 0 && litters.Count == 0)
|
||
notes.Add($"Keine Quelldaten gefunden in {_sourceDir} (animals.json/litters.json). extract.py zuerst ausführen.");
|
||
|
||
// ---- categorise animals ----
|
||
var loadable = new List<SourceAnimal>();
|
||
int conflicts = 0, stubs = 0, dateOnly = 0, ambiguous = 0;
|
||
foreach (var a in animals)
|
||
{
|
||
if (a.Conflict) { conflicts++; continue; }
|
||
if (string.IsNullOrEmpty(a.Dob)) { stubs++; continue; }
|
||
loadable.Add(a);
|
||
var conf = a.LitterRef?.Confidence;
|
||
if (a.LitterRef?.Candidates is { Count: > 0 }) ambiguous++;
|
||
else if (conf == "niedrig") dateOnly++;
|
||
}
|
||
|
||
// existing rows (idempotency). Gerbils carry ExternalRef; Litters have no such
|
||
// column, so we key litter idempotency on the stable (Name + Date) pair instead.
|
||
var existingGerbilExtRefs = await _db.Gerbils
|
||
.Where(g => g.ExternalRef != null)
|
||
.Select(g => g.ExternalRef!).ToListAsync();
|
||
var existingGerbilSet = existingGerbilExtRefs.ToHashSet();
|
||
|
||
var existingLitterKeys = await _db.Litters
|
||
.Select(l => new { l.Name, l.Date }).ToListAsync();
|
||
var existingLitterKeySet = existingLitterKeys
|
||
.Select(x => $"{x.Name}|{x.Date:yyyy-MM-dd}").ToHashSet();
|
||
|
||
// colour-variety name -> id (case-insensitive)
|
||
var varieties = await _db.ColorVarieties.Select(v => new { v.Id, v.Name }).ToListAsync();
|
||
var varietyByName = varieties
|
||
.GroupBy(v => v.Name.Trim().ToLowerInvariant())
|
||
.ToDictionary(g => g.Key, g => g.First().Id);
|
||
|
||
// gender inference from litter roles (sire -> male, dam -> female; both -> unknown)
|
||
var sireNames = litters.Select(l => Normalize(StripZucht(l.SireName))).Where(s => s.Length > 0).ToHashSet();
|
||
var damNames = litters.Select(l => Normalize(StripZucht(l.DamName))).Where(s => s.Length > 0).ToHashSet();
|
||
|
||
// ---- litters: create map source.id -> Litter (for high-confidence animal links) ----
|
||
int littersCreated = 0, littersExisting = 0;
|
||
var litterIdMap = new Dictionary<string, Guid>(); // source litter id -> Litter.Id
|
||
foreach (var sl in litters)
|
||
{
|
||
var date = ParseDate(sl.Date);
|
||
var name = $"Wurf {sl.LitterId}".Trim();
|
||
var key = $"{name}|{date:yyyy-MM-dd}";
|
||
if (existingLitterKeySet.Contains(key)) { littersExisting++; continue; }
|
||
|
||
var id = Guid.NewGuid();
|
||
litterIdMap[sl.Id] = id;
|
||
littersCreated++;
|
||
if (execute && date is DateOnly d)
|
||
{
|
||
_db.Litters.Add(new Litter
|
||
{
|
||
Id = id,
|
||
Name = name,
|
||
Date = d,
|
||
TotalBorn = sl.TotalBorn,
|
||
Notes = string.IsNullOrWhiteSpace(sl.Note) ? null : sl.Note,
|
||
PairingCode = string.IsNullOrWhiteSpace(sl.Zuchtnummer) ? null : sl.Zuchtnummer,
|
||
});
|
||
}
|
||
if (samples.Count < 8 && date is not null)
|
||
samples.Add($"Wurf: {name} ({sl.Date}) — {sl.DamName} × {sl.SireName}");
|
||
}
|
||
if (execute) await _db.SaveChangesAsync();
|
||
|
||
// ---- animals ----
|
||
int animalsCreated = 0, linked = 0, fbMatched = 0, fbUnmatched = 0, animalsExisting = 0;
|
||
int photosAttached = 0, photosMissing = 0;
|
||
var createdAnimalByName = new Dictionary<string, Guid>(); // normalized name -> gerbil id (for litter back-link)
|
||
|
||
foreach (var a in loadable)
|
||
{
|
||
if (existingGerbilSet.Contains(a.Id)) { animalsExisting++; continue; }
|
||
animalsCreated++;
|
||
|
||
Guid? litterId = null;
|
||
if (a.LitterRef?.Confidence == "hoch" && a.LitterRef.Candidates is not { Count: > 0 }
|
||
&& litterIdMap.TryGetValue(a.LitterRef.LitterId, out var lid))
|
||
{
|
||
litterId = lid;
|
||
linked++;
|
||
}
|
||
|
||
Guid? colorVarietyId = null;
|
||
var fbCandidates = new[] { a.Farbschlag }.Concat(a.FarbschlagVariants)
|
||
.Where(s => !string.IsNullOrWhiteSpace(s));
|
||
foreach (var fb in fbCandidates)
|
||
{
|
||
if (varietyByName.TryGetValue(fb.Trim().ToLowerInvariant(), out var vid))
|
||
{ colorVarietyId = vid; break; }
|
||
}
|
||
if (colorVarietyId is null) fbUnmatched++; else fbMatched++;
|
||
|
||
var gender = InferGender(a, sireNames, damNames);
|
||
var gid = Guid.NewGuid();
|
||
var norm = Normalize(StripZucht(a.Name));
|
||
if (norm.Length > 0) createdAnimalByName.TryAdd(norm, gid);
|
||
|
||
if (samples.Count < 16)
|
||
samples.Add($"Tier: {a.Name} (*{a.Dob}), Genotyp {ComposeGenotype(a.Genotype)}"
|
||
+ (litterId is not null ? ", Wurf-verknüpft" : "")
|
||
+ (colorVarietyId is not null ? $", Farbschlag „{a.Farbschlag}\"" : ""));
|
||
|
||
if (execute)
|
||
{
|
||
_db.Gerbils.Add(new Gerbil
|
||
{
|
||
Id = gid,
|
||
Name = a.Name,
|
||
Gender = gender,
|
||
Status = GerbilStatus.Active,
|
||
DateOfBirth = ParseDate(a.Dob),
|
||
DateOfDeath = ParseDate(a.Death),
|
||
LitterId = litterId,
|
||
ColorVarietyId = colorVarietyId,
|
||
Genotype = ComposeGenotype(a.Genotype),
|
||
ImportSource = ImportSourceTag,
|
||
ExternalRef = a.Id,
|
||
OriginBreeder = string.IsNullOrWhiteSpace(a.Zucht) ? null : a.Zucht.Trim(),
|
||
RawImportData = JsonSerializer.Serialize(new
|
||
{
|
||
a.Genotype.RawGenotype,
|
||
a.Genotype.UnmappedTokens,
|
||
a.Zucht,
|
||
a.SourceFiles,
|
||
FarbschlagRaw = a.Farbschlag,
|
||
}),
|
||
});
|
||
}
|
||
|
||
// photos
|
||
foreach (var rel in a.Photos)
|
||
{
|
||
var src = Path.Combine(_sourceDir, rel.Replace('/', Path.DirectorySeparatorChar));
|
||
if (!File.Exists(src)) { photosMissing++; continue; }
|
||
photosAttached++;
|
||
if (execute)
|
||
{
|
||
Directory.CreateDirectory(_photoRoot);
|
||
var fileName = $"{Guid.NewGuid():N}{Path.GetExtension(src)}";
|
||
File.Copy(src, Path.Combine(_photoRoot, fileName), overwrite: true);
|
||
_db.GerbilPhotos.Add(new GerbilPhoto
|
||
{
|
||
Id = Guid.NewGuid(),
|
||
GerbilId = gid,
|
||
FileName = fileName,
|
||
SortOrder = 0,
|
||
CreatedAt = DateTimeOffset.UtcNow,
|
||
});
|
||
}
|
||
}
|
||
}
|
||
if (execute) await _db.SaveChangesAsync();
|
||
|
||
// ---- back-link litter parents by name (best effort) ----
|
||
if (execute)
|
||
{
|
||
foreach (var sl in litters)
|
||
{
|
||
if (!litterIdMap.TryGetValue(sl.Id, out var lid)) continue;
|
||
var litter = await _db.Litters.FirstOrDefaultAsync(l => l.Id == lid);
|
||
if (litter is null) continue;
|
||
if (createdAnimalByName.TryGetValue(Normalize(StripZucht(sl.SireName)), out var fId))
|
||
litter.FatherId = fId;
|
||
if (createdAnimalByName.TryGetValue(Normalize(StripZucht(sl.DamName)), out var mId))
|
||
litter.MotherId = mId;
|
||
}
|
||
await _db.SaveChangesAsync();
|
||
}
|
||
|
||
notes.Add("Quarantäne (kein Import): Konflikte + Stubs ohne Geburtsdatum + unsichere Wurf-Zuordnungen — warten auf die Prüfung durch die Züchterin.");
|
||
if (!execute) notes.Add("DRY-RUN: nichts gespeichert. /import/execute lädt die konfliktfreien Daten.");
|
||
|
||
return new ImportReport(
|
||
Executed: execute,
|
||
Litters: new LitterSummary(litters.Count, littersCreated, littersExisting),
|
||
Animals: new AnimalSummary(
|
||
animals.Count, animalsCreated, linked, fbMatched, fbUnmatched, animalsExisting,
|
||
new QuarantineSummary(conflicts, stubs, dateOnly, ambiguous, conflicts + stubs)),
|
||
Photos: new PhotoSummary(photosAttached, photosMissing),
|
||
Samples: samples,
|
||
Notes: notes);
|
||
}
|
||
|
||
private T? Load<T>(string file)
|
||
{
|
||
var path = Path.Combine(_sourceDir, file);
|
||
if (!File.Exists(path)) return default;
|
||
using var fs = File.OpenRead(path);
|
||
return JsonSerializer.Deserialize<T>(fs, Json);
|
||
}
|
||
|
||
public static string ComposeGenotype(SourceGenotype g)
|
||
{
|
||
var tokens = LocusOrder.Select(locus =>
|
||
{
|
||
if (g.Mapped8locus.TryGetValue(locus, out var pair) && pair.Count == 2)
|
||
return StripCaret(pair[0]) + StripCaret(pair[1]);
|
||
return "??";
|
||
});
|
||
return string.Join(' ', tokens);
|
||
}
|
||
|
||
private static string StripCaret(string allele) => allele.Replace("^", "");
|
||
|
||
private static Gender InferGender(SourceAnimal a, HashSet<string> sires, HashSet<string> dams)
|
||
{
|
||
var n = Normalize(StripZucht(a.Name));
|
||
bool isSire = sires.Contains(n), isDam = dams.Contains(n);
|
||
if (isSire && !isDam) return Gender.male;
|
||
if (isDam && !isSire) return Gender.female;
|
||
return Gender.unknown;
|
||
}
|
||
|
||
public static DateOnly? ParseDate(string s)
|
||
{
|
||
if (string.IsNullOrWhiteSpace(s)) return null;
|
||
var m = Regex.Match(s, @"(\d{1,2})\.(\d{1,2})\.(\d{2,4})");
|
||
if (!m.Success) return null;
|
||
int d = int.Parse(m.Groups[1].Value), mo = int.Parse(m.Groups[2].Value);
|
||
int y = int.Parse(m.Groups[3].Value);
|
||
if (y < 100) y += 2000;
|
||
try { return new DateOnly(y, mo, d); }
|
||
catch { return null; }
|
||
}
|
||
|
||
private static string StripZucht(string name)
|
||
{
|
||
var n = Regex.Replace(name ?? "", @"\[.*?\]", " ");
|
||
n = Regex.Replace(n, @"\s+(?:of|von\s+den|von\s+der|v\.\s?d\.|von)\s+.*$", "", RegexOptions.IgnoreCase);
|
||
return n.Trim();
|
||
}
|
||
|
||
private static string Normalize(string name)
|
||
{
|
||
var n = (name ?? "").ToLowerInvariant();
|
||
n = Regex.Replace(n, @"\bgen\.\b", " ");
|
||
n = Regex.Replace(n, @"[^a-z0-9äöüß]", "");
|
||
return n;
|
||
}
|
||
}
|
||
}
|