Gulum/GerbilManager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

FEAT-8d: docx-Importer fuer Wurfchronik-Detail (extract_docx.py + ImportDocxService)

Browse Source 
tools/import/extract_docx.py (stdlib-Python, kein pip):
  Parst 'Wurfchronik der Kleinen Chaoten im Detail.docx' (Word/XML via zipfile).
  93 Wuerfe + 227 benannte Tiere aus Tabellen extrahiert.
  Felder pro Tier: WS-Code, Wurfgeburtsdatum, Name, Farbschlag, Geschlecht (Stern-
  Suffix), Abnehmer, Abgabedatum, Tod-Datum + Ursache, Partnername + DOB.
  Sonderwerte (ZT/BLEIBT/FREI/VG:) werden herausgefiltert.
  Edge-Cases: Doppel-Datum (16./17.03.2021, 31.05/*01.06.2023), WS ohne Zaehler
  (/5), fehlende Leerzeichen vor WS:, mehrere Abnehmer (1.) ... 2.) ...).
  Output: output/docx_litters.json + output/docx_animals.json.

tools/import/test_extract_docx.py:
  Unit-Tests fuer Regex-Logik + Live-Tests gegen die echte docx (skip wenn fehlt).
  28/28 Tests gruen.

GerbilManagerWebAPI/Import/ImportDocxService.cs:
  Idempotenter NACHZUG-Loader (fill-NULL-only, nie ueberschreiben):
  - WS-Code + Wurfgeburtsdatum -> PairingCode -> Gerbil.LitterId
  - Abnehmer -> Contact lookup-or-create -> Gerbil.ReceiverContactId
  - Abgabedatum -> Gerbil.GoHomeDate
  - Tod-Datum + Ursache -> Gerbil.DateOfDeath + CauseOfDeath
  Dry-Run zaehlt geplante Aenderungen, Execute schreibt.

GerbilManagerWebAPI/Endpoints/ImportDocxEndpoints.cs:
  POST /import/docx/dry-run + /import/docx/execute (analog ImportEndpoints).

GATE: 157/157 C#, 28/28 Python-docx-Tests, ef has-pending=No.
NACHZUG: laueft NACH dem finalen WIPE+REIMPORT-3 (kein Impact auf aktuellen Pipeline).
This commit is contained in:
Gulum
2026-06-06 22:04:09 +02:00
parent 3706ba3ecd
commit f74af537e2
5 changed files with 677 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

259
GerbilManagerWebAPI/Import/ImportDocxService.cs Normal file
View File

@@ -0,0 +1,259 @@
using System.Text.Json;
using GerbilManagerWebAPI.Models;
using Microsoft.EntityFrameworkCore;
namespace GerbilManagerWebAPI.Import
{
/// <summary>
/// FEAT-8d docx loader. Consumes tools/import/output/docx_litters.json +
/// docx_animals.json (produced by extract_docx.py) and enriches the database:
///
/// Load policy (IDEMPOTENT NACHZUG after main WIPE+REIMPORT):
/// - Litter link: match docx WS-code to Litters.PairingCode → set Gerbil.LitterId
/// for animals matched by normalize(name) + litter birth date.
/// - ReceiverContact: lookup-or-create Contact by owner name → set ReceiverContactId.
/// - GoHomeDate, DateOfDeath, CauseOfDeath: fill if currently null (fill-NULL-only).
/// - NEVER overwrites a manually-set non-null value.
///
/// Idempotent: running multiple times is safe. Each run resolves whatever is still null.
/// Execute is gated by the endpoint; this service only acts when asked.
/// </summary>
public sealed class ImportDocxService
{
private static readonly JsonSerializerOptions Json = new() { PropertyNameCaseInsensitive = true };
private readonly ApplicationContext _db;
private readonly string _sourceDir;
public ImportDocxService(ApplicationContext db, IConfiguration config, IWebHostEnvironment env)
: this(db,
config["Import:SourcePath"]
?? Path.GetFullPath(Path.Combine(env.ContentRootPath, "..", "tools", "import", "output")))
{ }
public ImportDocxService(ApplicationContext db, string sourceDir)
{
_db = db;
_sourceDir = sourceDir;
}
public async Task<ImportDocxReport> RunAsync(bool execute)
{
var notes = new List<string>();
var docxLitters = Load<List<DocxLitter>>("docx_litters.json") ?? new();
var docxAnimals = Load<List<DocxAnimal>>("docx_animals.json") ?? new();
if (docxLitters.Count == 0 && docxAnimals.Count == 0)
{
notes.Add($"Keine Quelldaten in {_sourceDir} (docx_litters.json/docx_animals.json). " +
"extract_docx.py zuerst ausführen.");
return new ImportDocxReport(false, 0, 0, 0, 0, 0, 0, notes);
}
// Build lookup: PairingCode → Litter.Id (WS-code normalised: spaces removed)
var littersInDb = await _db.Litters
.Where(l => l.PairingCode != null)
.Select(l => new { l.Id, l.Date, l.PairingCode })
.ToListAsync();
var litterByWs = littersInDb
.GroupBy(l => l.PairingCode!.Replace(" ", ""))
.ToDictionary(g => g.Key, g => g.ToList());
// Build animal lookup: normalize(name) + litter_dob → Gerbil (for litter-link)
var gerbilsInDb = await _db.Gerbils
.Select(g => new { g.Id, g.Name, g.DateOfBirth, g.LitterId,
g.ReceiverContactId, g.GoHomeDate, g.DateOfDeath, g.CauseOfDeath })
.ToListAsync();
var gerbilByKey = gerbilsInDb
.Where(g => g.DateOfBirth is not null)
.GroupBy(g => NameDobKey(g.Name, g.DateOfBirth!.Value))
.ToDictionary(g => g.Key, g => g.ToList());
// Contact lookup: normalized name → existing Contact
var contactsInDb = await _db.Contacts
.Select(c => new { c.Id, c.Name })
.ToListAsync();
var contactByNorm = contactsInDb
.GroupBy(c => NormalizeName(c.Name))
.ToDictionary(g => g.Key, g => g.First().Id);
int litterLinked = 0, goHomeFilled = 0, deathFilled = 0;
int ownerLinked = 0, ownerCreated = 0, skipped = 0;
foreach (var da in docxAnimals)
{
if (string.IsNullOrWhiteSpace(da.Name)) { skipped++; continue; }
// Resolve the litter by WS-code + approximate birth date
Guid? litterId = null;
if (!string.IsNullOrWhiteSpace(da.WsCode) && !string.IsNullOrWhiteSpace(da.LitterDob))
{
var litterDob = ParseDate(da.LitterDob);
if (litterDob is not null && litterByWs.TryGetValue(da.WsCode.Replace(" ", ""), out var cands))
{
// Pick the litter whose date matches (within ±5 days for rounding)
var match = cands.FirstOrDefault(l =>
Math.Abs((l.Date.DayNumber - litterDob.Value.DayNumber)) <= 5);
litterId = match?.Id;
}
}
// Resolve the gerbil by name + litter birth date
var animalDob = litterId is not null
? (await _db.Litters.Where(l => l.Id == litterId).Select(l => (DateOnly?)l.Date).FirstOrDefaultAsync())
: ParseDate(da.LitterDob);
if (animalDob is null) { skipped++; continue; }
var key = NameDobKey(da.Name, animalDob.Value);
if (!gerbilByKey.TryGetValue(key, out var gerbilCands)) { skipped++; continue; }
// If multiple gerbils match (same name+dob), take the one without a litter link first
var gerbilSnap = gerbilCands.FirstOrDefault(g => g.LitterId == null)
?? gerbilCands.First();
// Resolve receiver contact (lookup-or-create)
Guid? receiverId = null;
if (!string.IsNullOrWhiteSpace(da.Owner))
{
var normOwner = NormalizeName(da.Owner);
if (contactByNorm.TryGetValue(normOwner, out var existingId))
{
receiverId = existingId;
ownerLinked++;
}
else
{
ownerCreated++;
if (execute)
{
var newContact = new Contact { Id = Guid.NewGuid(), Name = da.Owner.Trim() };
_db.Contacts.Add(newContact);
await _db.SaveChangesAsync();
receiverId = newContact.Id;
contactByNorm[normOwner] = receiverId.Value;
}
}
}
var goHomeDate = ParseDate(da.AbgabeDate);
var deathDate = ParseDate(da.DeathDate);
// Count what will change
bool willLinkLitter = litterId is not null && gerbilSnap.LitterId is null;
bool willFillGoHome = goHomeDate is not null && gerbilSnap.GoHomeDate is null;
bool willFillDeath = deathDate is not null && gerbilSnap.DateOfDeath is null;
if (willLinkLitter) litterLinked++;
if (willFillGoHome) goHomeFilled++;
if (willFillDeath) deathFilled++;
if (execute)
{
var row = await _db.Gerbils.FirstOrDefaultAsync(g => g.Id == gerbilSnap.Id);
if (row is null) continue;
if (willLinkLitter) row.LitterId = litterId;
if (receiverId is not null && row.ReceiverContactId is null)
row.ReceiverContactId = receiverId;
if (willFillGoHome) row.GoHomeDate = goHomeDate;
if (willFillDeath)
{
row.DateOfDeath = deathDate;
if (!string.IsNullOrWhiteSpace(da.DeathCause) && row.CauseOfDeath is null)
row.CauseOfDeath = da.DeathCause.Trim();
}
}
}
if (execute && (litterLinked + goHomeFilled + deathFilled + ownerLinked + ownerCreated) > 0)
await _db.SaveChangesAsync();
notes.Add($"Quelle: {docxLitters.Count} Würfe, {docxAnimals.Count} Tier-Zeilen aus der docx.");
notes.Add($"Litter-Links: {litterLinked} Tiere einem Wurf zugeordnet (WS-Code → PairingCode).");
notes.Add($"Abnehmer: {ownerLinked} bestehende Kontakte verknüpft, {ownerCreated} neue Kontakte angelegt.");
notes.Add($"GoHomeDate: {goHomeFilled} Abgabe-Daten nachgetragen.");
notes.Add($"Tod-Datum: {deathFilled} Todesdaten nachgetragen.");
notes.Add($"Übersprungen: {skipped} Zeilen (kein Name oder kein DB-Match).");
if (!execute) notes.Add("DRY-RUN: nichts gespeichert. /import/docx/execute schreibt die Änderungen.");
return new ImportDocxReport(execute, litterLinked, ownerLinked + ownerCreated,
goHomeFilled, deathFilled, ownerCreated, skipped, notes);
}
private T? Load<T>(string file)
{
var path = Path.Combine(_sourceDir, file);
if (!File.Exists(path)) return default;
using var fs = File.OpenRead(path);
return JsonSerializer.Deserialize<T>(fs, Json);
}
private static DateOnly? ParseDate(string? s)
{
if (string.IsNullOrWhiteSpace(s)) return null;
var m = System.Text.RegularExpressions.Regex.Match(s,
@"(\d{1,2})\.(\d{1,2})\.(\d{2,4})");
if (!m.Success) return null;
int d = int.Parse(m.Groups[1].Value), mo = int.Parse(m.Groups[2].Value);
int y = int.Parse(m.Groups[3].Value);
if (y < 100) y += 2000;
try { return new DateOnly(y, mo, d); } catch { return null; }
}
private static string NameDobKey(string name, DateOnly dob)
{
var n = System.Text.RegularExpressions.Regex.Replace(
(name ?? "").ToLowerInvariant(), @"[^a-z0-9äöüß]", "");
return $"{n}|{dob:yyyy-MM-dd}";
}
private static string NormalizeName(string name)
{
var n = (name ?? "").ToLowerInvariant();
n = System.Text.RegularExpressions.Regex.Replace(n, @"\s+", " ").Trim();
return n;
}
}
// ---- Source shapes (from extract_docx.py output) ----
public sealed class DocxLitter
{
public string LitterId { get; set; } = "";
public string Dob { get; set; } = "";
public string MotherName { get; set; } = "";
public string FatherName { get; set; } = "";
public string WsCode { get; set; } = "";
public string Note { get; set; } = "";
}
public sealed class DocxAnimal
{
public string WsCode { get; set; } = "";
public string LitterDob { get; set; } = "";
public string Name { get; set; } = "";
public string Farbschlag { get; set; } = "";
public string Gender { get; set; } = "";
public string Owner { get; set; } = "";
public string AbgabeDate { get; set; } = "";
public string AbgabeWeight { get; set; } = "";
public string DeathDate { get; set; } = "";
public string DeathCause { get; set; } = "";
public string PartnerName { get; set; } = "";
public string PartnerDob { get; set; } = "";
}
// ---- Report ----
public sealed record ImportDocxReport(
bool Executed,
int LitterLinked,
int OwnerLinked,
int GoHomeFilled,
int DeathFilled,
int ContactsCreated,
int Skipped,
IReadOnlyList<string> Notes);
}