438 lines
22 KiB
C#
438 lines
22 KiB
C#
using System.Text.Json;
|
|
using GerbilManagerWebAPI.Models;
|
|
using GerbilManagerWebAPI.Services;
|
|
using Microsoft.EntityFrameworkCore;
|
|
|
|
namespace GerbilManagerWebAPI.Import
|
|
{
|
|
/// <summary>
|
|
/// FEAT-8d docx loader. Consumes tools/import/output/docx_litters.json +
|
|
/// docx_animals.json (produced by extract_docx.py) and enriches OR CREATES animals.
|
|
///
|
|
/// Load policy (IDEMPOTENT NACHZUG after main WIPE+REIMPORT):
|
|
/// Match order per docx row:
|
|
/// 1. ExternalRef "docx-…" (previously created by this loader) → enrich (fill-NULL-only)
|
|
/// 2. normalize(name)+litterDob in Gerbils (from main import) → enrich (fill-NULL-only)
|
|
/// 3. No match → CREATE: abgegebene Jungtiere that never appeared in the pedigree charts.
|
|
///
|
|
/// Created animals: Status=GivenAway (or Deceased), IsResident=false,
|
|
/// OriginBreeder='Zucht der Kleinen Chaoten', ImportSource="docx",
|
|
/// ExternalRef = stable "docx-{ws}-{normname}-{litterDob}[-N]" key (idempotent;
|
|
/// -N ordinal suffix disambiguates same-name siblings in a litter).
|
|
///
|
|
/// NEVER overwrites a manually-set non-null value (fill-NULL-only for all fields).
|
|
///
|
|
/// Idempotent: running multiple times is safe. Re-run finds existing rows via ExternalRef.
|
|
/// Execute wraps all writes in a single transaction via CreateExecutionStrategy() so that
|
|
/// providers using EnableRetryOnFailure (e.g. NpgsqlRetryingExecutionStrategy) are
|
|
/// compatible. The strategy lambda resets all mutable state at the top so it is safe
|
|
/// to re-run on transient-failure retry.
|
|
/// Execute is gated by the endpoint; this service only acts when asked.
|
|
/// </summary>
|
|
public sealed class ImportDocxService
|
|
{
|
|
private static readonly JsonSerializerOptions Json = new() { PropertyNameCaseInsensitive = true };
|
|
|
|
private readonly ApplicationContext _db;
|
|
private readonly string _sourceDir;
|
|
|
|
public ImportDocxService(ApplicationContext db, IConfiguration config, IWebHostEnvironment env)
|
|
: this(db,
|
|
config["Import:SourcePath"]
|
|
?? Path.GetFullPath(Path.Combine(env.ContentRootPath, "..", "tools", "import", "output")))
|
|
{ }
|
|
|
|
public ImportDocxService(ApplicationContext db, string sourceDir)
|
|
{
|
|
_db = db;
|
|
_sourceDir = sourceDir;
|
|
}
|
|
|
|
public async Task<ImportDocxReport> RunAsync(bool execute)
|
|
{
|
|
var notes = new List<string>();
|
|
|
|
var docxLitters = Load<List<DocxLitter>>("docx_litters.json") ?? new();
|
|
var docxAnimals = Load<List<DocxAnimal>>("docx_animals.json") ?? new();
|
|
|
|
var pdfLitters = Load<List<DocxLitter>>("pdf_litters.json") ?? new();
|
|
var pdfAnimals = Load<List<DocxAnimal>>("pdf_animals.json") ?? new();
|
|
|
|
int docxLitCount = docxLitters.Count;
|
|
int docxAnimCount = docxAnimals.Count;
|
|
int pdfLitCount = pdfLitters.Count;
|
|
int pdfAnimCount = pdfAnimals.Count;
|
|
|
|
docxLitters.AddRange(pdfLitters);
|
|
docxAnimals.AddRange(pdfAnimals);
|
|
|
|
if (docxLitters.Count == 0 && docxAnimals.Count == 0)
|
|
{
|
|
notes.Add($"Keine Quelldaten in {_sourceDir} (docx_litters.json/docx_animals.json oder pdf_litters.json/pdf_animals.json). " +
|
|
"extract_docx.py zuerst ausführen oder PDF-VLM-Daten bereitstellen.");
|
|
return new ImportDocxReport(false, 0, 0, 0, 0, 0, 0, 0, notes);
|
|
}
|
|
|
|
// Litter lookup by birth date (DayNumber) → list of matching DB litters.
|
|
// NOTE: WsCode in docx is a litter-size fraction ("4/4", "/5") — NOT a PairingCode.
|
|
// Date-only lookup with uniqueness guard avoids false links (only link when
|
|
// exactly one DB litter falls within ±5 days of the docx litter DOB).
|
|
var littersInDb = await _db.Litters
|
|
.Select(l => new { l.Id, l.Date })
|
|
.ToListAsync();
|
|
var littersByDayNumber = littersInDb
|
|
.Where(l => l.Date.HasValue)
|
|
.GroupBy(l => l.Date!.Value.DayNumber)
|
|
.ToDictionary(g => g.Key, g => g.ToList());
|
|
|
|
// normalize(name)+litterDob → Gerbil snapshot (main-import enrich path)
|
|
var gerbilsInDb = await _db.Gerbils
|
|
.Select(g => new { g.Id, g.Name, g.DateOfBirth, g.LitterId,
|
|
g.ReceiverContactId, g.GoHomeDate, g.DateOfDeath, g.CauseOfDeath,
|
|
g.ExternalRef })
|
|
.ToListAsync();
|
|
var gerbilByKey = gerbilsInDb
|
|
.Where(g => g.DateOfBirth is not null)
|
|
.GroupBy(g => NameDobKey(g.Name, g.DateOfBirth!.Value))
|
|
.ToDictionary(g => g.Key, g => g.ToList());
|
|
|
|
// ExternalRef → snapshot for previously docx-created animals (idempotency across runs)
|
|
var docxExternalRefs = gerbilsInDb
|
|
.Where(g => g.ExternalRef?.StartsWith("docx-") == true)
|
|
.ToDictionary(g => g.ExternalRef!,
|
|
g => new { g.Id, g.LitterId, g.GoHomeDate, g.DateOfDeath, g.ReceiverContactId });
|
|
|
|
// Contact lookup: normalized name → existing Contact.Id
|
|
var contactsInDb = await _db.Contacts
|
|
.Select(c => new { c.Id, c.Name })
|
|
.ToListAsync();
|
|
var contactByNorm = contactsInDb
|
|
.GroupBy(c => NormalizeName(c.Name))
|
|
.ToDictionary(g => g.Key, g => g.First().Id);
|
|
|
|
// Snapshot of DB contacts before any writes.
|
|
// Used to reset contactByNorm on strategy retry (rolled-back contacts vanish from DB
|
|
// but would remain in the in-memory dict without this reset).
|
|
var contactByNormBase = new Dictionary<string, Guid>(contactByNorm);
|
|
|
|
// ColorVariety lookup: normalized name → Id (for CREATE path Farbschlag matching)
|
|
var colorVarietyByName = (await _db.ColorVarieties
|
|
.Select(cv => new { cv.Id, cv.Name })
|
|
.ToListAsync())
|
|
.GroupBy(cv => NormalizeName(cv.Name))
|
|
.ToDictionary(g => g.Key, g => g.First().Id);
|
|
|
|
int animalsCreated = 0, litterLinked = 0, goHomeFilled = 0, deathFilled = 0;
|
|
int ownerLinked = 0, ownerCreated = 0, skipped = 0;
|
|
|
|
// Ordinal counter for collision-free ExternalRef within this batch.
|
|
var externalRefOrdinals = new Dictionary<string, int>();
|
|
|
|
// Belt-and-suspenders: guard against adding the same ExternalRef twice in one run.
|
|
var batchRefs = new HashSet<string>();
|
|
|
|
// Inner loop — shared by dry-run and execute paths.
|
|
// All local variables above are captured by reference (C# closure), so the strategy
|
|
// lambda can reset them before each retry and RunLoopAsync sees the fresh state.
|
|
async Task RunLoopAsync()
|
|
{
|
|
foreach (var da in docxAnimals)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(da.Name)) { skipped++; continue; }
|
|
|
|
// Collision-free ExternalRef: ordinal suffix for same-name siblings
|
|
var baseRef = DocxExternalRefBase(da);
|
|
externalRefOrdinals.TryGetValue(baseRef, out var ord);
|
|
ord++;
|
|
externalRefOrdinals[baseRef] = ord;
|
|
var externalRef = ord == 1 ? baseRef : $"{baseRef}-{ord}";
|
|
|
|
// Resolve litter: date ±5 days, unambiguous (exactly one candidate)
|
|
Guid? litterId = null;
|
|
if (!string.IsNullOrWhiteSpace(da.LitterDob))
|
|
{
|
|
var litterDob = ParseDate(da.LitterDob);
|
|
if (litterDob is not null)
|
|
{
|
|
var candidates = new List<Guid>();
|
|
for (int delta = -5; delta <= 5; delta++)
|
|
{
|
|
if (littersByDayNumber.TryGetValue(litterDob.Value.DayNumber + delta, out var cl))
|
|
candidates.AddRange(cl.Select(l => l.Id));
|
|
}
|
|
if (candidates.Count == 1)
|
|
litterId = candidates[0];
|
|
// If 0 or >1 candidates: no link (avoid false links)
|
|
}
|
|
}
|
|
|
|
// Animal DOB = litter birth date (docx has no per-animal DOB)
|
|
var animalDob = litterId is not null
|
|
? (await _db.Litters.Where(l => l.Id == litterId).Select(l => (DateOnly?)l.Date).FirstOrDefaultAsync())
|
|
: ParseDate(da.LitterDob);
|
|
|
|
if (animalDob is null) { skipped++; continue; }
|
|
|
|
var goHomeDate = ParseDate(da.AbgabeDate);
|
|
var deathDate = ParseDate(da.DeathDate);
|
|
|
|
// Resolve receiver contact (lookup-or-create; shared by all paths)
|
|
Guid? receiverId = null;
|
|
if (!string.IsNullOrWhiteSpace(da.Owner))
|
|
{
|
|
var normOwner = NormalizeName(da.Owner);
|
|
if (contactByNorm.TryGetValue(normOwner, out var existingContactId))
|
|
{
|
|
receiverId = existingContactId;
|
|
ownerLinked++;
|
|
}
|
|
else
|
|
{
|
|
ownerCreated++;
|
|
if (execute)
|
|
{
|
|
var newContact = new Contact { Id = Guid.NewGuid(), Name = da.Owner.Trim() };
|
|
_db.Contacts.Add(newContact);
|
|
await _db.SaveChangesAsync(); // flush within the outer tx
|
|
receiverId = newContact.Id;
|
|
contactByNorm[normOwner] = receiverId.Value;
|
|
}
|
|
}
|
|
}
|
|
|
|
// ── PATH 1: previously docx-created animal (idempotent re-run) ──────────
|
|
if (docxExternalRefs.TryGetValue(externalRef, out var prevSnap))
|
|
{
|
|
bool willLink = litterId is not null && prevSnap.LitterId is null;
|
|
bool willHome = goHomeDate is not null && prevSnap.GoHomeDate is null;
|
|
bool willDeath = deathDate is not null && prevSnap.DateOfDeath is null;
|
|
|
|
if (willLink) litterLinked++;
|
|
if (willHome) goHomeFilled++;
|
|
if (willDeath) deathFilled++;
|
|
|
|
if (execute)
|
|
{
|
|
var row = await _db.Gerbils.FirstOrDefaultAsync(g => g.Id == prevSnap.Id);
|
|
if (row is null) continue;
|
|
if (willLink) row.LitterId = litterId;
|
|
if (receiverId is not null && row.ReceiverContactId is null) row.ReceiverContactId = receiverId;
|
|
if (willHome) row.GoHomeDate = goHomeDate;
|
|
if (willDeath)
|
|
{
|
|
row.DateOfDeath = deathDate;
|
|
if (!string.IsNullOrWhiteSpace(da.DeathCause) && row.CauseOfDeath is null)
|
|
row.CauseOfDeath = da.DeathCause.Trim();
|
|
}
|
|
}
|
|
continue;
|
|
}
|
|
|
|
// ── PATH 2: main-import animal matched by name+dob ───────────────────────
|
|
var nameKey = NameDobKey(da.Name, animalDob.Value);
|
|
if (gerbilByKey.TryGetValue(nameKey, out var gerbilCands))
|
|
{
|
|
var gerbilSnap = gerbilCands.FirstOrDefault(g => g.LitterId == null)
|
|
?? gerbilCands.First();
|
|
|
|
bool willLink = litterId is not null && gerbilSnap.LitterId is null;
|
|
bool willHome = goHomeDate is not null && gerbilSnap.GoHomeDate is null;
|
|
bool willDeath = deathDate is not null && gerbilSnap.DateOfDeath is null;
|
|
|
|
if (willLink) litterLinked++;
|
|
if (willHome) goHomeFilled++;
|
|
if (willDeath) deathFilled++;
|
|
|
|
if (execute)
|
|
{
|
|
var row = await _db.Gerbils.FirstOrDefaultAsync(g => g.Id == gerbilSnap.Id);
|
|
if (row is null) continue;
|
|
if (willLink) row.LitterId = litterId;
|
|
if (receiverId is not null && row.ReceiverContactId is null) row.ReceiverContactId = receiverId;
|
|
if (willHome) row.GoHomeDate = goHomeDate;
|
|
if (willDeath)
|
|
{
|
|
row.DateOfDeath = deathDate;
|
|
if (!string.IsNullOrWhiteSpace(da.DeathCause) && row.CauseOfDeath is null)
|
|
row.CauseOfDeath = da.DeathCause.Trim();
|
|
}
|
|
}
|
|
continue;
|
|
}
|
|
|
|
// ── PATH 3: no match → CREATE ────────────────────────────────────────────
|
|
// Belt-and-suspenders: ordinal should ensure uniqueness, but guard anyway
|
|
if (!batchRefs.Add(externalRef)) { skipped++; continue; }
|
|
|
|
animalsCreated++;
|
|
if (litterId is not null) litterLinked++;
|
|
if (goHomeDate is not null) goHomeFilled++;
|
|
if (deathDate is not null) deathFilled++;
|
|
|
|
if (execute)
|
|
{
|
|
colorVarietyByName.TryGetValue(NormalizeName(da.Farbschlag ?? ""), out var cvId);
|
|
|
|
_db.Gerbils.Add(new Gerbil
|
|
{
|
|
Id = Guid.NewGuid(),
|
|
Name = da.Name.Trim(),
|
|
DateOfBirth = animalDob,
|
|
Gender = ParseGender(da.Gender),
|
|
Status = GerbilStatusService.Derive(GerbilStatus.GivenAway, animalDob, deathDate, isAbgegeben: receiverId is not null, DateOnly.FromDateTime(DateTime.UtcNow)),
|
|
LitterId = litterId,
|
|
ReceiverContactId = receiverId,
|
|
GoHomeDate = goHomeDate,
|
|
DateOfDeath = deathDate,
|
|
CauseOfDeath = string.IsNullOrWhiteSpace(da.DeathCause) ? null : da.DeathCause.Trim(),
|
|
ColorVarietyId = cvId == default ? null : cvId,
|
|
OriginBreeder = "Zucht der kleinen Chaoten",
|
|
IsResident = false,
|
|
ImportSource = "docx",
|
|
ExternalRef = externalRef,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!execute)
|
|
{
|
|
// Dry-run: just count, no writes, no transaction needed.
|
|
await RunLoopAsync();
|
|
}
|
|
else
|
|
{
|
|
// Execute: wrap the entire transaction in the execution strategy so that providers
|
|
// with EnableRetryOnFailure (NpgsqlRetryingExecutionStrategy) are compatible.
|
|
// The lambda resets all mutable state at the top so retries start clean.
|
|
var strategy = _db.Database.CreateExecutionStrategy();
|
|
await strategy.ExecuteAsync(async () =>
|
|
{
|
|
// Reset mutable state — idempotent on strategy retry
|
|
_db.ChangeTracker.Clear();
|
|
externalRefOrdinals.Clear();
|
|
batchRefs.Clear();
|
|
animalsCreated = 0; litterLinked = 0; goHomeFilled = 0; deathFilled = 0;
|
|
ownerLinked = 0; ownerCreated = 0; skipped = 0;
|
|
// Rebuild from DB snapshot: contacts added in a failed attempt were rolled back
|
|
contactByNorm = new Dictionary<string, Guid>(contactByNormBase);
|
|
|
|
await using var tx = await _db.Database.BeginTransactionAsync();
|
|
await RunLoopAsync();
|
|
if ((animalsCreated + litterLinked + goHomeFilled + deathFilled + ownerCreated) > 0)
|
|
await _db.SaveChangesAsync();
|
|
await tx.CommitAsync();
|
|
});
|
|
}
|
|
|
|
notes.Add($"Quelle: {docxLitCount} Würfe / {docxAnimCount} Tiere aus DOCX. {pdfLitCount} Würfe / {pdfAnimCount} Tiere aus PDF.");
|
|
notes.Add($"Neu angelegt: {animalsCreated} Jungtiere (abgegeben, nicht in Stammbäumen).");
|
|
notes.Add($"Litter-Links: {litterLinked} Tiere einem Wurf zugeordnet (DOB-Match ±5 Tage, eindeutig).");
|
|
notes.Add($"Abnehmer: {ownerLinked} bestehende Kontakte verknüpft, {ownerCreated} neue Kontakte angelegt.");
|
|
notes.Add($"GoHomeDate: {goHomeFilled} Abgabe-Daten nachgetragen.");
|
|
notes.Add($"Tod-Datum: {deathFilled} Todesdaten nachgetragen.");
|
|
notes.Add($"Übersprungen: {skipped} Zeilen (kein Name oder kein Datum).");
|
|
if (!execute) notes.Add("DRY-RUN: nichts gespeichert. /import/docx/execute schreibt die Änderungen.");
|
|
|
|
return new ImportDocxReport(execute, animalsCreated, litterLinked, ownerLinked + ownerCreated,
|
|
goHomeFilled, deathFilled, ownerCreated, skipped, notes);
|
|
}
|
|
|
|
private T? Load<T>(string file)
|
|
{
|
|
var path = Path.Combine(_sourceDir, file);
|
|
if (!File.Exists(path)) return default;
|
|
using var fs = File.OpenRead(path);
|
|
return JsonSerializer.Deserialize<T>(fs, Json);
|
|
}
|
|
|
|
private static DateOnly? ParseDate(string? s)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(s)) return null;
|
|
var m = System.Text.RegularExpressions.Regex.Match(s,
|
|
@"(\d{1,2})\.(\d{1,2})\.(\d{2,4})");
|
|
if (!m.Success) return null;
|
|
int d = int.Parse(m.Groups[1].Value), mo = int.Parse(m.Groups[2].Value);
|
|
int y = int.Parse(m.Groups[3].Value);
|
|
if (y < 100) y += 2000;
|
|
try { return new DateOnly(y, mo, d); } catch { return null; }
|
|
}
|
|
|
|
private static string NameDobKey(string name, DateOnly dob)
|
|
{
|
|
var n = System.Text.RegularExpressions.Regex.Replace(
|
|
(name ?? "").ToLowerInvariant(), @"[^a-z0-9äöüß]", "");
|
|
return $"{n}|{dob:yyyy-MM-dd}";
|
|
}
|
|
|
|
private static string NormalizeName(string name)
|
|
{
|
|
var n = (name ?? "").ToLowerInvariant();
|
|
n = System.Text.RegularExpressions.Regex.Replace(n, @"\s+", " ").Trim();
|
|
return n;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Base ExternalRef key (before ordinal disambiguation). Two animals sharing the same
|
|
/// ws+name+litterDob get this same base; the caller appends -2, -3 … for duplicates.
|
|
/// </summary>
|
|
internal static string DocxExternalRefBase(DocxAnimal da)
|
|
{
|
|
var ws = (da.WsCode ?? "").Replace(" ", "").ToLowerInvariant();
|
|
var name = System.Text.RegularExpressions.Regex.Replace(
|
|
(da.Name ?? "").ToLowerInvariant(), @"[^a-z0-9äöüß]", "");
|
|
return $"docx-{ws}-{name}-{da.LitterDob}";
|
|
}
|
|
|
|
private static Gender ParseGender(string? s)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(s)) return Gender.unknown;
|
|
var lower = s.ToLowerInvariant();
|
|
if (lower == "male" || lower == "m" || lower.Contains("männ")) return Gender.male;
|
|
if (lower == "female" || lower == "f" || lower == "w" || lower.Contains("weibl")) return Gender.female;
|
|
return Gender.unknown;
|
|
}
|
|
}
|
|
|
|
// ---- Source shapes (from extract_docx.py output) ----
|
|
|
|
public sealed class DocxLitter
|
|
{
|
|
public string LitterId { get; set; } = "";
|
|
public string Dob { get; set; } = "";
|
|
public string MotherName { get; set; } = "";
|
|
public string FatherName { get; set; } = "";
|
|
public string WsCode { get; set; } = "";
|
|
public string Note { get; set; } = "";
|
|
}
|
|
|
|
public sealed class DocxAnimal
|
|
{
|
|
public string WsCode { get; set; } = "";
|
|
public string LitterDob { get; set; } = "";
|
|
public string Name { get; set; } = "";
|
|
public string Farbschlag { get; set; } = "";
|
|
public string Gender { get; set; } = "";
|
|
public string Owner { get; set; } = "";
|
|
public string AbgabeDate { get; set; } = "";
|
|
public string AbgabeWeight { get; set; } = "";
|
|
public string DeathDate { get; set; } = "";
|
|
public string DeathCause { get; set; } = "";
|
|
public string PartnerName { get; set; } = "";
|
|
public string PartnerDob { get; set; } = "";
|
|
}
|
|
|
|
// ---- Report ----
|
|
|
|
public sealed record ImportDocxReport(
|
|
bool Executed,
|
|
int Created,
|
|
int LitterLinked,
|
|
int OwnerLinked,
|
|
int GoHomeFilled,
|
|
int DeathFilled,
|
|
int ContactsCreated,
|
|
int Skipped,
|
|
IReadOnlyList<string> Notes);
|
|
}
|