198 lines
9.3 KiB
C#
198 lines
9.3 KiB
C#
using System.Globalization;
|
|
using System.Text;
|
|
|
|
namespace GerbilManagerWebAPI.Import.Rpro3
|
|
{
|
|
/// <summary>
|
|
/// Port von tools/import/compare_rpro3.py `dedup()`. RennmausPro erkennt beim Import vorhandene
|
|
/// Tiere NICHT → dasselbe Tier (v. a. die 7427 externen) kommt mehrfach vor.
|
|
///
|
|
/// Union-Find INNERHALB gleicher (normalisierter) Namen:
|
|
/// merge(a,b) wenn DOB/Farbe/Herkunft kompatibel (gleich ODER eine Seite unbekannt)
|
|
/// UND mindestens ein bekanntes Feld positiv übereinstimmt. Konflikt in einem bekannten
|
|
/// Feld → NICHT mergen (unsicher → mehrdeutig).
|
|
/// Platzhalter-Namen ("unbekannt","-","","?" …) werden nie dedupt.
|
|
/// </summary>
|
|
public static class Rpro3Dedup
|
|
{
|
|
private static readonly HashSet<string> PlaceholderNames = new(StringComparer.Ordinal)
|
|
{ "", "-", "n", "unbekannt", "unbekannt?", "?", "nn", "n.n.", "na", "namenlos", "." };
|
|
|
|
private static readonly HashSet<string> UnknownValues = new(StringComparer.Ordinal)
|
|
{ "", "unbekannt", "unbek.", "unbek", "unbekannte zucht", "?", "n.n.",
|
|
"unbekannter farbschlag", "keine angabe", "k.a.", "na", "unbekannte farbe" };
|
|
|
|
public sealed class DedupResult
|
|
{
|
|
/// <summary>Cluster (mehr als ein Datensatz) → die zusammengelegten Tiere. Key = Wurzel-Rid.</summary>
|
|
public Dictionary<string, List<Rpro3Animal>> MergeClusters { get; } = new();
|
|
/// <summary>Repräsentant-Rid je Cluster (auch Singletons) → Cluster-Wurzel.</summary>
|
|
public Dictionary<string, string> RidToRoot { get; } = new();
|
|
/// <summary>Mehrdeutige Namen: Name → Varianten (manuelle Entscheidung nötig).</summary>
|
|
public List<AmbiguousName> Ambiguous { get; } = new();
|
|
/// <summary>Platzhalter-Datensätze (nicht dedupbar) gruppiert nach Name.</summary>
|
|
public List<Rpro3Animal> Placeholders { get; } = new();
|
|
public int CollapsedRecords { get; set; } // Summe der Datensätze in Merge-Clustern
|
|
public int DuplicatesRemoved { get; set; } // CollapsedRecords - Anzahl Cluster
|
|
}
|
|
|
|
public sealed class AmbiguousName
|
|
{
|
|
public required string Name { get; init; }
|
|
public List<Variant> Variants { get; } = new();
|
|
public int BareCount { get; set; } // merkmalslose Varianten (nur Name)
|
|
}
|
|
|
|
public sealed class Variant
|
|
{
|
|
public int Count { get; set; }
|
|
public List<string> Dob { get; } = new();
|
|
public List<string> Farbe { get; } = new();
|
|
public List<string> Origin { get; } = new();
|
|
public bool IsOwn { get; set; }
|
|
}
|
|
|
|
public static string NormName(string? s)
|
|
{
|
|
if (s is null) return "";
|
|
var n = s.Normalize(NormalizationForm.FormKC).Trim().ToLowerInvariant();
|
|
return string.Join(' ', n.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries));
|
|
}
|
|
|
|
public static string NormValue(string? s)
|
|
{
|
|
var v = NormName(s);
|
|
return UnknownValues.Contains(v) ? "" : v;
|
|
}
|
|
|
|
public static DedupResult Run(IReadOnlyList<Rpro3Animal> animals)
|
|
{
|
|
var result = new DedupResult();
|
|
|
|
foreach (var a in animals)
|
|
{
|
|
a.NameKey = NormName(a.Name);
|
|
a.FarbeKey = NormValue(a.Farbe);
|
|
a.OriginKey = NormValue(a.Origin);
|
|
}
|
|
|
|
var byName = new Dictionary<string, List<Rpro3Animal>>();
|
|
foreach (var a in animals)
|
|
{
|
|
if (PlaceholderNames.Contains(a.NameKey)) { result.Placeholders.Add(a); continue; }
|
|
if (!byName.TryGetValue(a.NameKey, out var list)) byName[a.NameKey] = list = new();
|
|
list.Add(a);
|
|
}
|
|
|
|
var parent = new Dictionary<string, string>();
|
|
string Find(string x)
|
|
{
|
|
while (parent[x] != x) { parent[x] = parent[parent[x]]; x = parent[x]; }
|
|
return x;
|
|
}
|
|
void Union(string x, string y)
|
|
{
|
|
parent.TryAdd(x, x); parent.TryAdd(y, y);
|
|
parent[Find(x)] = Find(y);
|
|
}
|
|
|
|
static int Positive(Rpro3Animal a, Rpro3Animal b)
|
|
{
|
|
int agree = 0;
|
|
if (a.Dob is not null && b.Dob is not null && a.Dob == b.Dob) agree++;
|
|
if (a.FarbeKey.Length > 0 && a.FarbeKey == b.FarbeKey) agree++;
|
|
if (a.OriginKey.Length > 0 && a.OriginKey == b.OriginKey) agree++;
|
|
return agree;
|
|
}
|
|
static int Conflict(Rpro3Animal a, Rpro3Animal b)
|
|
{
|
|
int c = 0;
|
|
if (a.Dob is not null && b.Dob is not null && a.Dob != b.Dob) c++;
|
|
if (a.FarbeKey.Length > 0 && b.FarbeKey.Length > 0 && a.FarbeKey != b.FarbeKey) c++;
|
|
if (a.OriginKey.Length > 0 && b.OriginKey.Length > 0 && a.OriginKey != b.OriginKey) c++;
|
|
return c;
|
|
}
|
|
static bool Compat(string? x, string? y) =>
|
|
string.IsNullOrEmpty(x) || string.IsNullOrEmpty(y) || x == y;
|
|
static bool CompatDob(DateOnly? x, DateOnly? y) => x is null || y is null || x == y;
|
|
|
|
foreach (var (_, group) in byName)
|
|
{
|
|
foreach (var a in group) parent.TryAdd(a.Rid, a.Rid);
|
|
for (int i = 0; i < group.Count; i++)
|
|
for (int j = i + 1; j < group.Count; j++)
|
|
{
|
|
var a = group[i]; var b = group[j];
|
|
bool comp = CompatDob(a.Dob, b.Dob) && Compat(a.FarbeKey, b.FarbeKey) && Compat(a.OriginKey, b.OriginKey);
|
|
if (comp && Positive(a, b) >= 1 && Conflict(a, b) == 0)
|
|
Union(a.Rid, b.Rid);
|
|
}
|
|
}
|
|
|
|
// Cluster sammeln
|
|
var clusters = new Dictionary<string, List<Rpro3Animal>>();
|
|
foreach (var a in animals)
|
|
{
|
|
if (PlaceholderNames.Contains(a.NameKey)) continue;
|
|
var root = Find(a.Rid);
|
|
result.RidToRoot[a.Rid] = root;
|
|
if (!clusters.TryGetValue(root, out var list)) clusters[root] = list = new();
|
|
list.Add(a);
|
|
}
|
|
foreach (var (k, v) in clusters)
|
|
if (v.Count > 1)
|
|
{
|
|
result.MergeClusters[k] = v;
|
|
result.CollapsedRecords += v.Count;
|
|
}
|
|
result.DuplicatesRemoved = result.CollapsedRecords - result.MergeClusters.Count;
|
|
|
|
// Mehrdeutige Namen: Name löst sich nach sicherem Merge in >1 Cluster auf
|
|
var nameToRoots = new Dictionary<string, HashSet<string>>();
|
|
var disp = new Dictionary<string, string>();
|
|
foreach (var a in animals)
|
|
{
|
|
if (PlaceholderNames.Contains(a.NameKey)) continue;
|
|
if (!nameToRoots.TryGetValue(a.NameKey, out var set)) nameToRoots[a.NameKey] = set = new();
|
|
set.Add(Find(a.Rid));
|
|
disp.TryAdd(a.NameKey, a.Name ?? a.NameKey);
|
|
}
|
|
|
|
foreach (var (nm, roots) in nameToRoots)
|
|
{
|
|
if (roots.Count <= 1) continue;
|
|
var amb = new AmbiguousName { Name = disp.GetValueOrDefault(nm, nm) };
|
|
foreach (var root in roots)
|
|
{
|
|
var recs = clusters[root];
|
|
var dob = recs.Where(r => r.Dob is not null).Select(r => r.Dob!.Value.ToString("yyyy-MM-dd")).Distinct().OrderBy(x => x).ToList();
|
|
var farbe = recs.Where(r => !string.IsNullOrEmpty(r.Farbe)).Select(r => r.Farbe!).Distinct().OrderBy(x => x).ToList();
|
|
var origin = recs.Where(r => NormValue(r.Origin).Length > 0).Select(r => r.Origin!).Distinct().OrderBy(x => x).ToList();
|
|
bool bare = dob.Count == 0 && farbe.Count == 0 && origin.Count == 0;
|
|
if (bare) { amb.BareCount += recs.Count; continue; }
|
|
var v = new Variant { Count = recs.Count, IsOwn = recs.Any(r => r.Src == Rpro3Src.Stamm) };
|
|
v.Dob.AddRange(dob); v.Farbe.AddRange(farbe); v.Origin.AddRange(origin);
|
|
amb.Variants.Add(v);
|
|
}
|
|
amb.Variants.Sort((x, y) => y.Count.CompareTo(x.Count));
|
|
// Nur Namen mit mind. einer informativen Variante interessieren die Züchterin.
|
|
if (amb.Variants.Count >= 1) result.Ambiguous.Add(amb);
|
|
}
|
|
result.Ambiguous.Sort((x, y) => y.Variants.Count.CompareTo(x.Variants.Count));
|
|
|
|
return result;
|
|
}
|
|
|
|
/// <summary>Repräsentant eines Clusters: bevorzugt ein stamm-Tier, sonst das mit den meisten
|
|
/// bekannten Feldern (DOB/Farbe/Origin), Tiebreak per Rid (stabil).</summary>
|
|
public static Rpro3Animal Representative(List<Rpro3Animal> cluster)
|
|
{
|
|
return cluster
|
|
.OrderByDescending(a => a.Src == Rpro3Src.Stamm)
|
|
.ThenByDescending(a => (a.Dob is not null ? 1 : 0) + (a.FarbeKey.Length > 0 ? 1 : 0) + (a.OriginKey.Length > 0 ? 1 : 0))
|
|
.ThenBy(a => a.Rid, StringComparer.Ordinal)
|
|
.First();
|
|
}
|
|
}
|
|
}
|