Gulum/GerbilManager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(import): RennmausPro-III-Backup-Importer (analyze+execute, Dedup, Fotos)

Browse Source 
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Gulum
2026-06-22 22:43:19 +02:00
parent 5e14124322
commit 3ad276637c
19 changed files with 2620 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

197
GerbilManagerWebAPI/Import/Rpro3/Rpro3Dedup.cs Normal file
View File

@@ -0,0 +1,197 @@
using System.Globalization;
using System.Text;
namespace GerbilManagerWebAPI.Import.Rpro3
{
/// <summary>
/// Port von tools/import/compare_rpro3.py `dedup()`. RennmausPro erkennt beim Import vorhandene
/// Tiere NICHT → dasselbe Tier (v. a. die 7427 externen) kommt mehrfach vor.
///
/// Union-Find INNERHALB gleicher (normalisierter) Namen:
/// merge(a,b) wenn DOB/Farbe/Herkunft kompatibel (gleich ODER eine Seite unbekannt)
/// UND mindestens ein bekanntes Feld positiv übereinstimmt. Konflikt in einem bekannten
/// Feld → NICHT mergen (unsicher → mehrdeutig).
/// Platzhalter-Namen ("unbekannt","-","","?" …) werden nie dedupt.
/// </summary>
public static class Rpro3Dedup
{
private static readonly HashSet<string> PlaceholderNames = new(StringComparer.Ordinal)
{ "", "-", "n", "unbekannt", "unbekannt?", "?", "nn", "n.n.", "na", "namenlos", "." };
private static readonly HashSet<string> UnknownValues = new(StringComparer.Ordinal)
{ "", "unbekannt", "unbek.", "unbek", "unbekannte zucht", "?", "n.n.",
"unbekannter farbschlag", "keine angabe", "k.a.", "na", "unbekannte farbe" };
public sealed class DedupResult
{
/// <summary>Cluster (mehr als ein Datensatz) → die zusammengelegten Tiere. Key = Wurzel-Rid.</summary>
public Dictionary<string, List<Rpro3Animal>> MergeClusters { get; } = new();
/// <summary>Repräsentant-Rid je Cluster (auch Singletons) → Cluster-Wurzel.</summary>
public Dictionary<string, string> RidToRoot { get; } = new();
/// <summary>Mehrdeutige Namen: Name → Varianten (manuelle Entscheidung nötig).</summary>
public List<AmbiguousName> Ambiguous { get; } = new();
/// <summary>Platzhalter-Datensätze (nicht dedupbar) gruppiert nach Name.</summary>
public List<Rpro3Animal> Placeholders { get; } = new();
public int CollapsedRecords { get; set; } // Summe der Datensätze in Merge-Clustern
public int DuplicatesRemoved { get; set; } // CollapsedRecords - Anzahl Cluster
}
public sealed class AmbiguousName
{
public required string Name { get; init; }
public List<Variant> Variants { get; } = new();
public int BareCount { get; set; } // merkmalslose Varianten (nur Name)
}
public sealed class Variant
{
public int Count { get; set; }
public List<string> Dob { get; } = new();
public List<string> Farbe { get; } = new();
public List<string> Origin { get; } = new();
public bool IsOwn { get; set; }
}
public static string NormName(string? s)
{
if (s is null) return "";
var n = s.Normalize(NormalizationForm.FormKC).Trim().ToLowerInvariant();
return string.Join(' ', n.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries));
}
public static string NormValue(string? s)
{
var v = NormName(s);
return UnknownValues.Contains(v) ? "" : v;
}
public static DedupResult Run(IReadOnlyList<Rpro3Animal> animals)
{
var result = new DedupResult();
foreach (var a in animals)
{
a.NameKey = NormName(a.Name);
a.FarbeKey = NormValue(a.Farbe);
a.OriginKey = NormValue(a.Origin);
}
var byName = new Dictionary<string, List<Rpro3Animal>>();
foreach (var a in animals)
{
if (PlaceholderNames.Contains(a.NameKey)) { result.Placeholders.Add(a); continue; }
if (!byName.TryGetValue(a.NameKey, out var list)) byName[a.NameKey] = list = new();
list.Add(a);
}
var parent = new Dictionary<string, string>();
string Find(string x)
{
while (parent[x] != x) { parent[x] = parent[parent[x]]; x = parent[x]; }
return x;
}
void Union(string x, string y)
{
parent.TryAdd(x, x); parent.TryAdd(y, y);
parent[Find(x)] = Find(y);
}
static int Positive(Rpro3Animal a, Rpro3Animal b)
{
int agree = 0;
if (a.Dob is not null && b.Dob is not null && a.Dob == b.Dob) agree++;
if (a.FarbeKey.Length > 0 && a.FarbeKey == b.FarbeKey) agree++;
if (a.OriginKey.Length > 0 && a.OriginKey == b.OriginKey) agree++;
return agree;
}
static int Conflict(Rpro3Animal a, Rpro3Animal b)
{
int c = 0;
if (a.Dob is not null && b.Dob is not null && a.Dob != b.Dob) c++;
if (a.FarbeKey.Length > 0 && b.FarbeKey.Length > 0 && a.FarbeKey != b.FarbeKey) c++;
if (a.OriginKey.Length > 0 && b.OriginKey.Length > 0 && a.OriginKey != b.OriginKey) c++;
return c;
}
static bool Compat(string? x, string? y) =>
string.IsNullOrEmpty(x) || string.IsNullOrEmpty(y) || x == y;
static bool CompatDob(DateOnly? x, DateOnly? y) => x is null || y is null || x == y;
foreach (var (_, group) in byName)
{
foreach (var a in group) parent.TryAdd(a.Rid, a.Rid);
for (int i = 0; i < group.Count; i++)
for (int j = i + 1; j < group.Count; j++)
{
var a = group[i]; var b = group[j];
bool comp = CompatDob(a.Dob, b.Dob) && Compat(a.FarbeKey, b.FarbeKey) && Compat(a.OriginKey, b.OriginKey);
if (comp && Positive(a, b) >= 1 && Conflict(a, b) == 0)
Union(a.Rid, b.Rid);
}
}
// Cluster sammeln
var clusters = new Dictionary<string, List<Rpro3Animal>>();
foreach (var a in animals)
{
if (PlaceholderNames.Contains(a.NameKey)) continue;
var root = Find(a.Rid);
result.RidToRoot[a.Rid] = root;
if (!clusters.TryGetValue(root, out var list)) clusters[root] = list = new();
list.Add(a);
}
foreach (var (k, v) in clusters)
if (v.Count > 1)
{
result.MergeClusters[k] = v;
result.CollapsedRecords += v.Count;
}
result.DuplicatesRemoved = result.CollapsedRecords - result.MergeClusters.Count;
// Mehrdeutige Namen: Name löst sich nach sicherem Merge in >1 Cluster auf
var nameToRoots = new Dictionary<string, HashSet<string>>();
var disp = new Dictionary<string, string>();
foreach (var a in animals)
{
if (PlaceholderNames.Contains(a.NameKey)) continue;
if (!nameToRoots.TryGetValue(a.NameKey, out var set)) nameToRoots[a.NameKey] = set = new();
set.Add(Find(a.Rid));
disp.TryAdd(a.NameKey, a.Name ?? a.NameKey);
}
foreach (var (nm, roots) in nameToRoots)
{
if (roots.Count <= 1) continue;
var amb = new AmbiguousName { Name = disp.GetValueOrDefault(nm, nm) };
foreach (var root in roots)
{
var recs = clusters[root];
var dob = recs.Where(r => r.Dob is not null).Select(r => r.Dob!.Value.ToString("yyyy-MM-dd")).Distinct().OrderBy(x => x).ToList();
var farbe = recs.Where(r => !string.IsNullOrEmpty(r.Farbe)).Select(r => r.Farbe!).Distinct().OrderBy(x => x).ToList();
var origin = recs.Where(r => NormValue(r.Origin).Length > 0).Select(r => r.Origin!).Distinct().OrderBy(x => x).ToList();
bool bare = dob.Count == 0 && farbe.Count == 0 && origin.Count == 0;
if (bare) { amb.BareCount += recs.Count; continue; }
var v = new Variant { Count = recs.Count, IsOwn = recs.Any(r => r.Src == Rpro3Src.Stamm) };
v.Dob.AddRange(dob); v.Farbe.AddRange(farbe); v.Origin.AddRange(origin);
amb.Variants.Add(v);
}
amb.Variants.Sort((x, y) => y.Count.CompareTo(x.Count));
// Nur Namen mit mind. einer informativen Variante interessieren die Züchterin.
if (amb.Variants.Count >= 1) result.Ambiguous.Add(amb);
}
result.Ambiguous.Sort((x, y) => y.Variants.Count.CompareTo(x.Variants.Count));
return result;
}
/// <summary>Repräsentant eines Clusters: bevorzugt ein stamm-Tier, sonst das mit den meisten
/// bekannten Feldern (DOB/Farbe/Origin), Tiebreak per Rid (stabil).</summary>
public static Rpro3Animal Representative(List<Rpro3Animal> cluster)
{
return cluster
.OrderByDescending(a => a.Src == Rpro3Src.Stamm)
.ThenByDescending(a => (a.Dob is not null ? 1 : 0) + (a.FarbeKey.Length > 0 ? 1 : 0) + (a.OriginKey.Length > 0 ? 1 : 0))
.ThenBy(a => a.Rid, StringComparer.Ordinal)
.First();
}
}
}