feat(migration): implement offline-resolved database-ready ingestion, fix circular FKs, duplicate contacts and parser scanning bugs
This commit is contained in:
181
GerbilManager.Tests/IngestResolvedServiceTests.cs
Normal file
181
GerbilManager.Tests/IngestResolvedServiceTests.cs
Normal file
@@ -0,0 +1,181 @@
|
|||||||
|
using GerbilManagerWebAPI.Import;
|
||||||
|
using GerbilManagerWebAPI.Models;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using Microsoft.Extensions.Configuration;
|
||||||
|
using System.Text.Json;
|
||||||
|
|
||||||
|
namespace GerbilManager.Tests
|
||||||
|
{
|
||||||
|
public class IngestResolvedServiceTests : IDisposable
|
||||||
|
{
|
||||||
|
private readonly string _dir;
|
||||||
|
private readonly string _resolvedJsonPath;
|
||||||
|
|
||||||
|
public IngestResolvedServiceTests()
|
||||||
|
{
|
||||||
|
_dir = Path.Combine(Path.GetTempPath(), "ingest-resolved-" + Guid.NewGuid().ToString("N"));
|
||||||
|
Directory.CreateDirectory(_dir);
|
||||||
|
_resolvedJsonPath = Path.Combine(_dir, "resolved_import.json");
|
||||||
|
|
||||||
|
var contactId = Guid.NewGuid();
|
||||||
|
var fatherId = Guid.NewGuid();
|
||||||
|
var motherId = Guid.NewGuid();
|
||||||
|
var litterId = Guid.NewGuid();
|
||||||
|
var photoId = Guid.NewGuid();
|
||||||
|
|
||||||
|
var data = new
|
||||||
|
{
|
||||||
|
Contacts = new[]
|
||||||
|
{
|
||||||
|
new
|
||||||
|
{
|
||||||
|
Id = contactId,
|
||||||
|
Name = "Test Breeder",
|
||||||
|
ZuchtName = "Test Zucht",
|
||||||
|
City = "Test City",
|
||||||
|
Email = "test@example.com",
|
||||||
|
Homepage = "",
|
||||||
|
Phone = "",
|
||||||
|
Address = ""
|
||||||
|
}
|
||||||
|
},
|
||||||
|
Litters = new[]
|
||||||
|
{
|
||||||
|
new
|
||||||
|
{
|
||||||
|
Id = litterId,
|
||||||
|
Name = "Wurf A",
|
||||||
|
Date = "2026-01-01",
|
||||||
|
TotalBorn = 5,
|
||||||
|
DeathsWithin8Weeks = 0,
|
||||||
|
FatherId = fatherId,
|
||||||
|
MotherId = motherId,
|
||||||
|
ExpectedGoHomeDate = (string?)null,
|
||||||
|
Notes = "Test litter notes",
|
||||||
|
PairingCode = "PC01",
|
||||||
|
ExternalRef = "ext-litter-1",
|
||||||
|
LitterLetter = "A"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
Gerbils = new[]
|
||||||
|
{
|
||||||
|
new
|
||||||
|
{
|
||||||
|
Id = fatherId,
|
||||||
|
Name = "Papa",
|
||||||
|
Gender = "male",
|
||||||
|
Status = "Breeding",
|
||||||
|
LitterId = (Guid?)null,
|
||||||
|
OriginContactId = contactId,
|
||||||
|
ReceiverContactId = (Guid?)null,
|
||||||
|
EnclosureId = (Guid?)null,
|
||||||
|
ColorVarietyId = new Guid("00000000-0000-0000-0000-000000000006"), // Schwarz
|
||||||
|
DateOfBirth = "2025-01-01",
|
||||||
|
DateOfDeath = (string?)null,
|
||||||
|
CauseOfDeath = (string?)null,
|
||||||
|
GoHomeDate = (string?)null,
|
||||||
|
Genotype = "aa CC DD EE GG PP spsp rere",
|
||||||
|
Notes = "",
|
||||||
|
ImportSource = "docx-export",
|
||||||
|
ExternalRef = "ext-papa",
|
||||||
|
RawImportData = "{}",
|
||||||
|
OriginBreeder = "Test Zucht",
|
||||||
|
NameSearch = "papa",
|
||||||
|
CharacterTraits = new string[] {},
|
||||||
|
CharacterNote = (string?)null,
|
||||||
|
IsDeaf = false,
|
||||||
|
IsResident = true
|
||||||
|
},
|
||||||
|
new
|
||||||
|
{
|
||||||
|
Id = motherId,
|
||||||
|
Name = "Mama",
|
||||||
|
Gender = "female",
|
||||||
|
Status = "Breeding",
|
||||||
|
LitterId = (Guid?)null,
|
||||||
|
OriginContactId = contactId,
|
||||||
|
ReceiverContactId = (Guid?)null,
|
||||||
|
EnclosureId = (Guid?)null,
|
||||||
|
ColorVarietyId = new Guid("00000000-0000-0000-0000-000000000006"), // Schwarz
|
||||||
|
DateOfBirth = "2025-01-01",
|
||||||
|
DateOfDeath = (string?)null,
|
||||||
|
CauseOfDeath = (string?)null,
|
||||||
|
GoHomeDate = (string?)null,
|
||||||
|
Genotype = "aa CC DD EE GG PP spsp rere",
|
||||||
|
Notes = "",
|
||||||
|
ImportSource = "docx-export",
|
||||||
|
ExternalRef = "ext-mama",
|
||||||
|
RawImportData = "{}",
|
||||||
|
OriginBreeder = "Test Zucht",
|
||||||
|
NameSearch = "mama",
|
||||||
|
CharacterTraits = new string[] {},
|
||||||
|
CharacterNote = (string?)null,
|
||||||
|
IsDeaf = false,
|
||||||
|
IsResident = true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
GerbilPhotos = new[]
|
||||||
|
{
|
||||||
|
new
|
||||||
|
{
|
||||||
|
Id = photoId,
|
||||||
|
GerbilId = fatherId,
|
||||||
|
FileName = "photo1.jpg",
|
||||||
|
SortOrder = 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
File.WriteAllText(_resolvedJsonPath, JsonSerializer.Serialize(data));
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Dispose()
|
||||||
|
{
|
||||||
|
try { Directory.Delete(_dir, recursive: true); } catch { }
|
||||||
|
}
|
||||||
|
|
||||||
|
private ApplicationContext NewDb()
|
||||||
|
{
|
||||||
|
var opts = new DbContextOptionsBuilder<ApplicationContext>()
|
||||||
|
.UseInMemoryDatabase("ingest-resolved-" + Guid.NewGuid().ToString("N"))
|
||||||
|
.Options;
|
||||||
|
var db = new ApplicationContext(opts);
|
||||||
|
db.Database.EnsureCreated();
|
||||||
|
return db;
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task IngestResolved_loads_data_and_resolves_all_relations()
|
||||||
|
{
|
||||||
|
using var db = NewDb();
|
||||||
|
var config = new ConfigurationBuilder()
|
||||||
|
.AddInMemoryCollection(new Dictionary<string, string?>
|
||||||
|
{
|
||||||
|
{ "Import:SourcePath", _dir }
|
||||||
|
})
|
||||||
|
.Build();
|
||||||
|
|
||||||
|
var service = new IngestResolvedService(db, config, null!);
|
||||||
|
var result = await service.RunAsync();
|
||||||
|
|
||||||
|
Assert.Contains("Ingestion successful!", result);
|
||||||
|
|
||||||
|
// Verify counts
|
||||||
|
Assert.Equal(1, await db.Contacts.CountAsync());
|
||||||
|
Assert.Equal(1, await db.Litters.CountAsync());
|
||||||
|
Assert.Equal(2, await db.Gerbils.CountAsync());
|
||||||
|
Assert.Equal(1, await db.GerbilPhotos.CountAsync());
|
||||||
|
|
||||||
|
// Verify relations
|
||||||
|
var litter = await db.Litters.SingleAsync();
|
||||||
|
var father = await db.Gerbils.SingleAsync(g => g.Name == "Papa");
|
||||||
|
var mother = await db.Gerbils.SingleAsync(g => g.Name == "Mama");
|
||||||
|
var photo = await db.GerbilPhotos.SingleAsync();
|
||||||
|
|
||||||
|
Assert.Equal(father.Id, litter.FatherId);
|
||||||
|
Assert.Equal(mother.Id, litter.MotherId);
|
||||||
|
Assert.Equal(father.Id, photo.GerbilId);
|
||||||
|
Assert.Equal(father.OriginContactId, mother.OriginContactId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -29,6 +29,13 @@ namespace GerbilManagerWebAPI.Endpoints
|
|||||||
return TypedResults.Ok(report);
|
return TypedResults.Ok(report);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
group.MapPost("/ingest-resolved", async Task<Ok<string>> (
|
||||||
|
ApplicationContext db, IConfiguration config, IWebHostEnvironment env) =>
|
||||||
|
{
|
||||||
|
var result = await new IngestResolvedService(db, config, env).RunAsync();
|
||||||
|
return TypedResults.Ok(result);
|
||||||
|
});
|
||||||
|
|
||||||
return app;
|
return app;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
192
GerbilManagerWebAPI/Import/IngestResolvedService.cs
Normal file
192
GerbilManagerWebAPI/Import/IngestResolvedService.cs
Normal file
@@ -0,0 +1,192 @@
|
|||||||
|
using System.Text.Json;
|
||||||
|
using GerbilManagerWebAPI.Models;
|
||||||
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
|
||||||
|
namespace GerbilManagerWebAPI.Import
|
||||||
|
{
|
||||||
|
public sealed class IngestResolvedService
|
||||||
|
{
|
||||||
|
private readonly ApplicationContext _db;
|
||||||
|
private readonly string _resolvedJsonPath;
|
||||||
|
|
||||||
|
public IngestResolvedService(ApplicationContext db, IConfiguration config, IWebHostEnvironment env)
|
||||||
|
{
|
||||||
|
_db = db;
|
||||||
|
var sourceDir = config["Import:SourcePath"]
|
||||||
|
?? Path.GetFullPath(Path.Combine(env.ContentRootPath, "..", "tools", "import", "output"));
|
||||||
|
_resolvedJsonPath = Path.Combine(sourceDir, "resolved_import.json");
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<string> RunAsync()
|
||||||
|
{
|
||||||
|
if (!File.Exists(_resolvedJsonPath))
|
||||||
|
{
|
||||||
|
return $"Error: Resolved import file not found at {_resolvedJsonPath}. Run resolve_import.py first.";
|
||||||
|
}
|
||||||
|
|
||||||
|
string jsonContent = await File.ReadAllTextAsync(_resolvedJsonPath);
|
||||||
|
var jsonOptions = new JsonSerializerOptions
|
||||||
|
{
|
||||||
|
PropertyNameCaseInsensitive = true
|
||||||
|
};
|
||||||
|
jsonOptions.Converters.Add(new System.Text.Json.Serialization.JsonStringEnumConverter());
|
||||||
|
var data = JsonSerializer.Deserialize<ResolvedImportData>(jsonContent, jsonOptions);
|
||||||
|
|
||||||
|
if (data == null)
|
||||||
|
{
|
||||||
|
return "Error: Failed to deserialize resolved_import.json.";
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1. Provider-agnostic clean wipe of dependent tables to prevent duplicate keys
|
||||||
|
if (_db.Database.ProviderName == "Microsoft.EntityFrameworkCore.InMemory")
|
||||||
|
{
|
||||||
|
// Break circular references first
|
||||||
|
foreach (var l in _db.Litters)
|
||||||
|
{
|
||||||
|
l.FatherId = null;
|
||||||
|
l.MotherId = null;
|
||||||
|
}
|
||||||
|
foreach (var g in _db.Gerbils)
|
||||||
|
{
|
||||||
|
g.LitterId = null;
|
||||||
|
}
|
||||||
|
await _db.SaveChangesAsync();
|
||||||
|
|
||||||
|
_db.GerbilPhotos.RemoveRange(_db.GerbilPhotos);
|
||||||
|
_db.WeightRecords.RemoveRange(_db.WeightRecords);
|
||||||
|
_db.HealthRecords.RemoveRange(_db.HealthRecords);
|
||||||
|
_db.Gerbils.RemoveRange(_db.Gerbils);
|
||||||
|
_db.Litters.RemoveRange(_db.Litters);
|
||||||
|
await _db.SaveChangesAsync();
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// Break circular references first using ExecuteUpdateAsync
|
||||||
|
await _db.Litters.ExecuteUpdateAsync(s => s
|
||||||
|
.SetProperty(l => l.FatherId, (Guid?)null)
|
||||||
|
.SetProperty(l => l.MotherId, (Guid?)null));
|
||||||
|
|
||||||
|
await _db.Gerbils.ExecuteUpdateAsync(s => s
|
||||||
|
.SetProperty(g => g.LitterId, (Guid?)null));
|
||||||
|
|
||||||
|
await _db.GerbilPhotos.ExecuteDeleteAsync();
|
||||||
|
await _db.WeightRecords.ExecuteDeleteAsync();
|
||||||
|
await _db.HealthRecords.ExecuteDeleteAsync();
|
||||||
|
await _db.Gerbils.ExecuteDeleteAsync();
|
||||||
|
await _db.Litters.ExecuteDeleteAsync();
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Import Contacts (Only add new ones to preserve manually entered contacts)
|
||||||
|
var existingContactIds = await _db.Contacts.Select(c => c.Id).ToHashSetAsync();
|
||||||
|
var addedContactIds = new HashSet<Guid>();
|
||||||
|
int contactsAdded = 0;
|
||||||
|
foreach (var c in data.Contacts)
|
||||||
|
{
|
||||||
|
if (!existingContactIds.Contains(c.Id) && addedContactIds.Add(c.Id))
|
||||||
|
{
|
||||||
|
_db.Contacts.Add(c);
|
||||||
|
contactsAdded++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await _db.SaveChangesAsync();
|
||||||
|
|
||||||
|
// 3. Pass 1: Insert all Litters with parent references set to null to avoid FK errors
|
||||||
|
var littersToInsert = new List<Litter>();
|
||||||
|
foreach (var l in data.Litters)
|
||||||
|
{
|
||||||
|
littersToInsert.Add(new Litter
|
||||||
|
{
|
||||||
|
Id = l.Id,
|
||||||
|
Name = l.Name,
|
||||||
|
Date = l.Date,
|
||||||
|
TotalBorn = l.TotalBorn,
|
||||||
|
DeathsWithin8Weeks = l.DeathsWithin8Weeks,
|
||||||
|
FatherId = null, // Set in Pass 2
|
||||||
|
MotherId = null, // Set in Pass 2
|
||||||
|
ExpectedGoHomeDate = l.ExpectedGoHomeDate,
|
||||||
|
Notes = l.Notes,
|
||||||
|
PairingCode = l.PairingCode,
|
||||||
|
ExternalRef = l.ExternalRef,
|
||||||
|
LitterLetter = l.LitterLetter
|
||||||
|
});
|
||||||
|
}
|
||||||
|
_db.Litters.AddRange(littersToInsert);
|
||||||
|
await _db.SaveChangesAsync();
|
||||||
|
|
||||||
|
// 4. Pass 1: Insert all Gerbils with LitterId set to null to avoid FK errors
|
||||||
|
var gerbilsToInsert = new List<Gerbil>();
|
||||||
|
foreach (var g in data.Gerbils)
|
||||||
|
{
|
||||||
|
gerbilsToInsert.Add(new Gerbil
|
||||||
|
{
|
||||||
|
Id = g.Id,
|
||||||
|
Name = g.Name,
|
||||||
|
Gender = g.Gender,
|
||||||
|
Status = g.Status,
|
||||||
|
LitterId = null, // Set in Pass 2
|
||||||
|
OriginContactId = g.OriginContactId,
|
||||||
|
ReceiverContactId = g.ReceiverContactId,
|
||||||
|
EnclosureId = g.EnclosureId,
|
||||||
|
ColorVarietyId = g.ColorVarietyId,
|
||||||
|
DateOfBirth = g.DateOfBirth,
|
||||||
|
DateOfDeath = g.DateOfDeath,
|
||||||
|
CauseOfDeath = g.CauseOfDeath,
|
||||||
|
GoHomeDate = g.GoHomeDate,
|
||||||
|
Genotype = g.Genotype,
|
||||||
|
Notes = g.Notes,
|
||||||
|
ImportSource = g.ImportSource,
|
||||||
|
ExternalRef = g.ExternalRef,
|
||||||
|
RawImportData = g.RawImportData,
|
||||||
|
OriginBreeder = g.OriginBreeder,
|
||||||
|
NameSearch = g.NameSearch,
|
||||||
|
CharacterTraits = g.CharacterTraits,
|
||||||
|
CharacterNote = g.CharacterNote,
|
||||||
|
IsDeaf = g.IsDeaf,
|
||||||
|
IsResident = g.IsResident
|
||||||
|
});
|
||||||
|
}
|
||||||
|
_db.Gerbils.AddRange(gerbilsToInsert);
|
||||||
|
await _db.SaveChangesAsync();
|
||||||
|
|
||||||
|
// 5. Pass 2: Set the actual foreign key relationships
|
||||||
|
// Update Litters with their parent animal IDs
|
||||||
|
var littersInDb = await _db.Litters.ToDictionaryAsync(l => l.Id);
|
||||||
|
foreach (var l in data.Litters)
|
||||||
|
{
|
||||||
|
if (littersInDb.TryGetValue(l.Id, out var dbLitter))
|
||||||
|
{
|
||||||
|
dbLitter.FatherId = l.FatherId;
|
||||||
|
dbLitter.MotherId = l.MotherId;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update Gerbils with their actual LitterId
|
||||||
|
var gerbilsInDb = await _db.Gerbils.ToDictionaryAsync(g => g.Id);
|
||||||
|
foreach (var g in data.Gerbils)
|
||||||
|
{
|
||||||
|
if (gerbilsInDb.TryGetValue(g.Id, out var dbGerbil))
|
||||||
|
{
|
||||||
|
dbGerbil.LitterId = g.LitterId;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await _db.SaveChangesAsync();
|
||||||
|
|
||||||
|
// 6. Insert all Photos
|
||||||
|
foreach (var p in data.GerbilPhotos)
|
||||||
|
{
|
||||||
|
_db.GerbilPhotos.Add(p);
|
||||||
|
}
|
||||||
|
await _db.SaveChangesAsync();
|
||||||
|
|
||||||
|
return $"Ingestion successful! Imported {contactsAdded} contacts, {data.Litters.Count} litters, {data.Gerbils.Count} gerbils, and {data.GerbilPhotos.Count} photos.";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public class ResolvedImportData
|
||||||
|
{
|
||||||
|
public List<Contact> Contacts { get; set; } = new();
|
||||||
|
public List<Litter> Litters { get; set; } = new();
|
||||||
|
public List<Gerbil> Gerbils { get; set; } = new();
|
||||||
|
public List<GerbilPhoto> GerbilPhotos { get; set; } = new();
|
||||||
|
}
|
||||||
|
}
|
||||||
88
import_todo.md
Normal file
88
import_todo.md
Normal file
@@ -0,0 +1,88 @@
|
|||||||
|
# TODO: Migration to Unified Database-Ready Format
|
||||||
|
|
||||||
|
This is the checklist for migrating the GerbilManager Excel datasets (41 Stammbaum pedigree charts + 1 Wurfchronik litter chronicle) one-by-one into a clean, resolved, relational JSON format (`gerbilmanager_resolved.json`) that can be loaded directly into the database without on-the-fly reconciliation.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🟩 Phase 1: Preparation & Schema Definition
|
||||||
|
- [x] Define the target unified JSON schema that maps 1-to-1 with the Entity Framework database entities:
|
||||||
|
- `Contacts` (breeders/buyers)
|
||||||
|
- `ColorVarieties` (genetics catalog)
|
||||||
|
- `Litters` (with resolved `MotherId` and `FatherId` Guid relations)
|
||||||
|
- `Gerbils` (with resolved `LitterId`, `ColorVarietyId`, `OriginContactId`, `ReceiverContactId` Guid relations)
|
||||||
|
- `GerbilPhotos` (mapped by Guid relation)
|
||||||
|
- [x] Extract and verify existing `ColorVarieties` and `Contacts` to establish basic lookup dictionaries.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🟦 Phase 2: Spreadsheet Extraction & Conversion (One-by-One)
|
||||||
|
Convert and extract each spreadsheet's records. For each chart, extract the animals, birth dates, death dates, genders (from cell background color), and genotypes.
|
||||||
|
|
||||||
|
### Litter Chronicle
|
||||||
|
- [x] `Wurfchronik der Kleine Chaoten Teil1.xlsx` (Litters data base)
|
||||||
|
|
||||||
|
### Pedigree Charts (Stammbäume)
|
||||||
|
- [x] `Stammbaum von Akio Kids.xlsx`
|
||||||
|
- [x] `Stammbaum von Alberto Kids.xlsx`
|
||||||
|
- [x] `Stammbaum von CP-Fuchs, CP-Sa Sp von Unity.xlsx`
|
||||||
|
- [x] `Stammbaum von Chesnut.xlsx`
|
||||||
|
- [x] `Stammbaum von Danako.xlsx`
|
||||||
|
- [x] `Stammbaum von Ella.xlsx`
|
||||||
|
- [x] `Stammbaum von Emi.xlsx`
|
||||||
|
- [x] `Stammbaum von Fire Kids.xlsx`
|
||||||
|
- [x] `Stammbaum von Goldfuchs Sp (Pikachu) Kids.xlsx`
|
||||||
|
- [x] `Stammbaum von Hana.xlsx`
|
||||||
|
- [x] `Stammbaum von Jeremy.xlsx`
|
||||||
|
- [x] `Stammbaum von Jiminy of Black Forest.xlsx`
|
||||||
|
- [x] `Stammbaum von Jin.xlsx`
|
||||||
|
- [x] `Stammbaum von Kalea.xlsx`
|
||||||
|
- [x] `Stammbaum von Kazuya.xlsx`
|
||||||
|
- [x] `Stammbaum von Kentucky.xlsx`
|
||||||
|
- [x] `Stammbaum von Kohlief, Goldfuchsef Sp von Chrissi.xlsx`
|
||||||
|
- [x] `Stammbaum von Marlin of Black Forest.xlsx`
|
||||||
|
- [x] `Stammbaum von Martin.xlsx`
|
||||||
|
- [x] `Stammbaum von Oscar of Black Forest.xlsx`
|
||||||
|
- [x] `Stammbaum von Pinto of Fiomi.xlsx`
|
||||||
|
- [x] `Stammbaum von Quied Soldier of Black Forest.xlsx`
|
||||||
|
- [x] `Stammbaum von Rainny.xlsx`
|
||||||
|
- [x] `Stammbaum von Ren.xlsx`
|
||||||
|
- [x] `Stammbaum von South Dakota.xlsx`
|
||||||
|
- [x] `Stammbaum von Stella Kids.xlsx`
|
||||||
|
- [x] `Stammbaum von Tennessee.xlsx`
|
||||||
|
- [x] `Stammbaum von Unique of Wild Dreams.xlsx`
|
||||||
|
- [x] `Stammbaum von Unit.xlsx`
|
||||||
|
- [x] `Stammbaum von Uriana.xlsx`
|
||||||
|
- [x] `Stammbaum von Valentino Firehearts Kids.xlsx`
|
||||||
|
- [x] `Stammbaum von Vance.xlsx`
|
||||||
|
- [x] `Stammbaum von Vestra von den Schlossmäusen.xlsx`
|
||||||
|
- [x] `Stammbaum von Watarus Kids.xlsx`
|
||||||
|
- [x] `Stammbaum von Wildfire und Vestras Kids.xlsx`
|
||||||
|
- [x] `Stammbaum von Xelina.xlsx`
|
||||||
|
- [x] `Stammbaum von Yuki.xlsx`
|
||||||
|
- [x] `Stammbaum von Yurikas und Pintos Sohn.xlsx`
|
||||||
|
- [x] `Stammbaum von Zac (Vance.Dorie).xlsx`
|
||||||
|
- [x] `Stammbaum von Zenon von Elea.xlsx`
|
||||||
|
- [x] `Stammbaum von little Heros Sohn of little runners.xlsx`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🟨 Phase 3: Resolution & Relational Mapping
|
||||||
|
- [x] Resolve duplicates using the canonical key: `normalize(name) + normalize(dob)` and `Zucht` (breeding line).
|
||||||
|
- [x] Inject resolutions from `conflict-decisions.json` to automatically resolve the remaining conflict animals (un-quarantining them and applying correct genotypes/dates).
|
||||||
|
- [x] Parse and map all parent-child relationships positionally from the pedigree charts.
|
||||||
|
- [x] Map and link all animals to their respective Wurfchronik litters using parentage and DOB.
|
||||||
|
- [x] Resolve photo attachments to actual extracted media file paths.
|
||||||
|
- [x] Assign permanent UUIDs (Guids) to all records and replace string names in relationships with the actual UUID foreign keys.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🟧 Phase 4: Data Export & Validation
|
||||||
|
- [x] Generate the final `resolved_import.json` containing the relational array collections.
|
||||||
|
- [x] Run automated validation tests on the JSON file (check for circular dependencies, broken foreign keys, invalid genotypes, and null dates).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🟥 Phase 5: Loader Implementation & Database Ingestion
|
||||||
|
- [x] Build a simplified loader utility (C# endpoint or local python script) that takes the resolved JSON and directly runs `INSERT`/`UPDATE` transactions.
|
||||||
|
- [x] Execute the import onto the target PostgreSQL database.
|
||||||
|
- [x] Run verification tests (verify total counts: ~956 animals, and sample check pedigree trees).
|
||||||
@@ -217,7 +217,7 @@ def extract_stammbaum(path):
|
|||||||
# full block: name above, farbschlag/genotype/breeder below
|
# full block: name above, farbschlag/genotype/breeder below
|
||||||
dob, death, geno0 = parse_detail(t)
|
dob, death, geno0 = parse_detail(t)
|
||||||
name = ""
|
name = ""
|
||||||
for rr in range(r - 1, r - 4, -1):
|
for rr in range(r - 3, r):
|
||||||
if (c, rr) in cells and not re.match(r"^\*?\s?\d", cells[(c, rr)]) \
|
if (c, rr) in cells and not re.match(r"^\*?\s?\d", cells[(c, rr)]) \
|
||||||
and not gt.looks_like_genotype(cells[(c, rr)]):
|
and not gt.looks_like_genotype(cells[(c, rr)]):
|
||||||
name = clean_name(cells[(c, rr)])
|
name = clean_name(cells[(c, rr)])
|
||||||
|
|||||||
@@ -4,36 +4,47 @@ _Automatisch erzeugt von `tools/import/extract.py` — **noch nichts in die Date
|
|||||||
|
|
||||||
## Überblick
|
## Überblick
|
||||||
|
|
||||||
- Rohe Tier-Einträge aus den Stammbäumen: **950**
|
- Rohe Tier-Einträge aus den Stammbäumen: **2451**
|
||||||
- Nach Zusammenführung (eindeutige Tiere): **621**
|
- Nach Zusammenführung (eindeutige Tiere): **1007**
|
||||||
- davon mit Geburtsdatum: 326
|
- davon mit Geburtsdatum: 682
|
||||||
- in mehreren Dateien gefunden (Dubletten zusammengeführt): 158
|
- in mehreren Dateien gefunden (Dubletten zusammengeführt): 461
|
||||||
- Konflikte zur Klärung: **5**
|
- Konflikte zur Klärung: **7**
|
||||||
- Mehrdeutige / unvollständige Einträge (ohne Name+Datum): **310**
|
- Mehrdeutige / unvollständige Einträge (ohne Name+Datum): **342**
|
||||||
- Fotos zugeordnet: **137**
|
- Fotos zugeordnet: **417**
|
||||||
- Würfe aus der Wurfchronik: **752**
|
- Würfe aus der Wurfchronik: **752**
|
||||||
- Tiere mit Wurf verknüpft: **159** (davon über Geburtsdatum **und** Eltern: 110, nur über Geburtsdatum: 49; mehrdeutig: 10)
|
- Tiere mit Wurf verknüpft: **274** (davon über Geburtsdatum **und** Eltern: 175, nur über Geburtsdatum: 99; mehrdeutig: 14)
|
||||||
- Würfe mit Datenqualitäts-Hinweisen: 113 (+ 138 Zeilen mit abweichendem Spaltenschema)
|
- Würfe mit Datenqualitäts-Hinweisen: 113 (+ 138 Zeilen mit abweichendem Spaltenschema)
|
||||||
|
|
||||||
## Zusammenführungs-Schlüssel
|
## Zusammenführungs-Schlüssel
|
||||||
|
|
||||||
Tiere wurden zusammengeführt über **normalisierter Rufname + Geburtsdatum**, mit der **Zucht als Unterscheidungsmerkmal** (Julians Regel: die `[Klammern]` in der Wurfchronik und das `of/von <Linie>`-Suffix der Stammbäume bezeichnen beide die Zucht und werden zusammengeführt — z. B. `[ZdkC]` ≙ `von den Kleinen Chaoten`). Namensvarianten (z. B. `v.d.` ↔ `von den`, `gen.`-Spitznamen) werden als `nameVariants` erhalten.
|
Tiere wurden zusammengeführt über **normalisierter Rufname + Geburtsdatum**, mit der **Zucht als Unterscheidungsmerkmal** (Julians Regel: die `[Klammern]` in der Wurfchronik und das `of/von <Linie>`-Suffix der Stammbäume bezeichnen beide die Zucht und werden zusammengeführt — z. B. `[ZdkC]` ≙ `von den Kleinen Chaoten`). Namensvarianten (z. B. `v.d.` ↔ `von den`, `gen.`-Spitznamen) werden als `nameVariants` erhalten.
|
||||||
|
|
||||||
|
### Gleicher Name + Geburtsdatum, aber unterschiedliche Zucht (NICHT zusammengeführt — bitte prüfen)
|
||||||
|
|
||||||
|
| Tier | Geburtsdatum | Zuchten | Dateien |
|
||||||
|
|---|---|---|---|
|
||||||
|
| Blacky | 23.07.2009 | PZ Seligenstadt // Privatzucht Seligenstadt | Stammbaum von Danako, Stammbaum von Hana, Stammbaum von Jin, Stammbaum von Unit, Stammbaum von Uriana, Stammbaum von Wildfire und Vestras Kids |
|
||||||
|
| Danny | 09.10.2009 | Maintaler PZ // PZ Maintal | Stammbaum von Hana, Stammbaum von Vance |
|
||||||
|
| Lola | 10.02.2014 | Chalfont Stud, Freddy Braun // Lennylengo | Stammbaum von Jeremy, Stammbaum von Jiminy of Black Forest, Stammbaum von Marlin of Black Forest, Stammbaum von Zac (Vance.Dorie), Stammbaum von Zenon von Elea |
|
||||||
|
| Debby gen. Eva | 08.08.2012 | KK Chaos // KK Chaos of KK Chaos | Stammbaum von Kalea, Stammbaum von Unit, Stammbaum von Uriana |
|
||||||
|
|
||||||
## ⚠️ Konflikte (bitte prüfen)
|
## ⚠️ Konflikte (bitte prüfen)
|
||||||
|
|
||||||
Gleiches Tier (Name+Datum), aber widersprüchliche Angaben in verschiedenen Dateien:
|
Gleiches Tier (Name+Datum), aber widersprüchliche Angaben in verschiedenen Dateien:
|
||||||
|
|
||||||
| Tier | Geburtsdatum | abweichende Genotypen | abweichende Farbschläge | Sterbedaten | Dateien |
|
| Tier | Geburtsdatum | abweichende Genotypen | abweichende Farbschläge | Sterbedaten | Dateien |
|
||||||
|---|---|---|---|---|---|
|
|---|---|---|---|---|---|
|
||||||
| Kazu von den Kleinen Chaoten | 23.04.2013 | Aa Cc[chm] DD e[f]e[f] Gg P Spsp // Aa Cc[chm] DD ee[f] UwUw PP Spsp | — | 03.09.2017 | Stammbaum von Akio Kids, Stammbaum von Vance |
|
| Joghurt von Privat | 06.09.2013 | aa CC D- ee GG PP spsp // aa Cc[-] D- ee UwUw PP spsp | Privat | 06.03.2017 | Stammbaum von Akio Kids, Stammbaum von Jin, Stammbaum von Kentucky |
|
||||||
| Little Runner's Big Ben | 03.02.2020 | Aa Cc[chm] DD Ee Gg PP Spsp // Aa Cc[chm] DD Ee Gg Pp Spsp | — | 14.10.2023 | Stammbaum von CP-Fuchs, CP-Sa Sp von Unity, Stammbaum von Fire Kids, Stammbaum von Goldfuchs Sp (Pikachu) Kids, Stammbaum von Kohlief, Goldfuchsef Sp von Chrissi, Stammbaum von Valentino Firehearts Kids, Stammbaum von Watarus Kids |
|
| Ken'ichi | 01.03.2015 | AA CC DD Ee GG PP spsp | Agouti // DD-Tumor | 02.11.2018 | Stammbaum von Chesnut, Stammbaum von Emi, Stammbaum von Hana, Stammbaum von Kentucky, Stammbaum von Kohlief, Goldfuchsef Sp von Chrissi |
|
||||||
| Vance Jr. von den Kleinen Chaoten | 10.04.2022 | aa Cc[hm] Dd Ee gg P- Spsp // aa Cc[hm] Dd Ee gg P- spsp | Kohlfuchs, hell | — | Stammbaum von Fire Kids, Stammbaum von Stella Kids |
|
| DD-Tumor bei Geschwister | 10.09.2015 | aa Cc[chm] D- Ee gg P- spsp // aa Cc[chm] D- Ee uw[d]uw[d] P- spsp | — | 02.01.2020 // 12.12.2019 | Stammbaum von Chesnut, Stammbaum von Emi |
|
||||||
| Hanami von den Kleinen Chaoten | 10.09.2015 | aa Cc[chm] D- Ee gg P- spsp | — | 12.12.2019 // 14.01.2020 | Stammbaum von Kentucky, Stammbaum von Stella Kids |
|
| Kleiner Warnowrenner Elieus gen. Eragon | 18.05.2016 | aa CC D- Ee Gg pp Spsp [DP] // aa c[chm]c[chm] D- Ee Gg pp Spsp | — | 09.10.2019 | Stammbaum von Chesnut, Stammbaum von Jeremy, Stammbaum von Kentucky, Stammbaum von Martin, Stammbaum von Quied Soldier of Black Forest |
|
||||||
| Skarlett v.d. Kleinen Chaoten | 14.07.2013 | / +2018, Aa Cc[chm] DD ee uw[d]uw[d] PP spsp // Aa Cc[chm] DD ee uw[d]uw[d] PP spsp | — | 17.04.2016 // 2018 | Stammbaum von Vance |
|
| Osamu | 10.12.2015 | AA CC DD ee gg P- spsp // AA CC DD ee gg PP spsp // AA CC DD ee uw[d]uw[d] PP spsp | Polarfuchs | 01.10.2020 // 18.12.2020 | Stammbaum von Danako, Stammbaum von Ella, Stammbaum von Jin, Stammbaum von Kazuya, Stammbaum von Kentucky, Stammbaum von Martin, Stammbaum von Rainny, Stammbaum von Ren, Stammbaum von South Dakota, Stammbaum von Stella Kids, Stammbaum von Tennessee, Stammbaum von Zenon von Elea |
|
||||||
|
| Harumi | 21.02.2015 | aa Cc[chm] DD EE GG PP Spsp | — | 03.06.2017 // 11.01.2018 | Stammbaum von Danako, Stammbaum von Kalea, Stammbaum von Yurikas und Pintos Sohn |
|
||||||
|
| Hanami | 10.09.2015 | aa Cc[chm] D- Ee gg P- spsp // aa Cc[chm] D- Ee uw[d]uw[d] P- spsp | — | 02.01.2020 // 12.12.2019 // 14.01.2020 | Stammbaum von Hana, Stammbaum von Kentucky, Stammbaum von Rainny, Stammbaum von Ren, Stammbaum von Stella Kids, Stammbaum von Vance, Stammbaum von Zac (Vance.Dorie) |
|
||||||
|
|
||||||
## Mehrdeutige / unvollständige Einträge
|
## Mehrdeutige / unvollständige Einträge
|
||||||
|
|
||||||
310 Einträge ohne sichere Name+Datum-Kombination (z. B. `Name1 & Name2`-Paarzellen der tiefsten Generation, oder Zellen ohne Datum). Diese werden NICHT automatisch zusammengeführt.
|
342 Einträge ohne sichere Name+Datum-Kombination (z. B. `Name1 & Name2`-Paarzellen der tiefsten Generation, oder Zellen ohne Datum). Diese werden NICHT automatisch zusammengeführt.
|
||||||
|
|
||||||
- Oskar v.d. bunten Fellnase · Stammbaum von Akio Kids.xlsx
|
- Oskar v.d. bunten Fellnase · Stammbaum von Akio Kids.xlsx
|
||||||
- Raya v.d. bunten Fellnasen · Stammbaum von Akio Kids.xlsx
|
- Raya v.d. bunten Fellnasen · Stammbaum von Akio Kids.xlsx
|
||||||
@@ -78,23 +89,55 @@ Gleiches Tier (Name+Datum), aber widersprüchliche Angaben in verschiedenen Date
|
|||||||
|
|
||||||
## Wahrscheinliche Zuordnungen unvollständiger Einträge
|
## Wahrscheinliche Zuordnungen unvollständiger Einträge
|
||||||
|
|
||||||
39 namenlose/datenlose Einträge tragen denselben Namen wie ein vollständiges Tier — vermutlich dasselbe Tier (zur Bestätigung):
|
91 namenlose/datenlose Einträge tragen denselben Namen wie ein vollständiges Tier — vermutlich dasselbe Tier (zur Bestätigung):
|
||||||
|
|
||||||
|
- „Tai of Lennylengo“ → Tai of Lennylengo (*01.10.2011)
|
||||||
|
- „Arrow PZ Niederlande“ → Arrow PZ Niederlande (*20.05.2014)
|
||||||
|
- „Isa of Golden Lights“ → Isa of Golden Lights (*24.12.2014)
|
||||||
|
- „Leila v. Jessica Walldorf“ → Leila v. Jessica Walldorf (*08.04.2014)
|
||||||
|
- „Osamu“ → Osamu (*10.12.2015)
|
||||||
|
- „Nala“ → Nala (*06.03.2015)
|
||||||
- „Oscar of Black Forest“ → Oscar of Black Forest (*12.06.2019)
|
- „Oscar of Black Forest“ → Oscar of Black Forest (*12.06.2019)
|
||||||
- „Hagrid Rubeus of Black Forest“ → Hagrid Rubeus of Black Forest (*18.07.2019)
|
- „Hagrid Rubeus of Black Forest“ → Hagrid Rubeus of Black Forest (*18.07.2019)
|
||||||
- „Lilo of LennyLengo“ → Lilo of LennyLengo (*04.11.2018)
|
- „Lilo of LennyLengo“ → Lilo of LennyLengo (*04.11.2018)
|
||||||
|
- „Kazuya“ → Kazuya (*14.07.2019)
|
||||||
|
- „Harumi“ → Harumi (*21.02.2015)
|
||||||
|
- „Pan“ → Pan (*09.12.2014)
|
||||||
|
- „Gin“ → Gin (*05.02.2015)
|
||||||
|
- „Isa of Golden Lights“ → Isa of Golden Lights (*24.12.2014)
|
||||||
- „Mystique of Black Forest“ → Mystique of Black Forest (*12.03.2022)
|
- „Mystique of Black Forest“ → Mystique of Black Forest (*12.03.2022)
|
||||||
|
- „Elay“ → Elay (*16.03.2016)
|
||||||
|
- „Osamu“ → Osamu (*10.12.2015)
|
||||||
|
- „Baiko“ → Baiko (*02.06.2019)
|
||||||
- „Hagrid Rubeus of Black Forest“ → Hagrid Rubeus of Black Forest (*18.07.2019)
|
- „Hagrid Rubeus of Black Forest“ → Hagrid Rubeus of Black Forest (*18.07.2019)
|
||||||
|
- „Osamu“ → Osamu (*10.12.2015)
|
||||||
|
- „Umi“ → Umi (*13.08.2016)
|
||||||
- „Charly of Golden Lights“ → Charly of Golden Lights (*05.04.2016)
|
- „Charly of Golden Lights“ → Charly of Golden Lights (*05.04.2016)
|
||||||
- „Ziwa of Golden Lights“ → Ziwa of Golden Lights (*29.04.2016)
|
- „Ziwa of Golden Lights“ → Ziwa of Golden Lights (*29.04.2016)
|
||||||
|
- „Harumi“ → Harumi (*21.02.2015)
|
||||||
|
- „Pan“ → Pan (*09.12.2014)
|
||||||
|
- „Gin“ → Gin (*05.02.2015)
|
||||||
|
- „Isa of golden lights“ → Isa of Golden Lights (*24.12.2014)
|
||||||
|
- „Porter“ → Porter (*23.05.2015)
|
||||||
- „Chelsea von den Kleinen Chaoten“ → Chelsea von den Kleinen Chaoten (*02.04.2021)
|
- „Chelsea von den Kleinen Chaoten“ → Chelsea von den Kleinen Chaoten (*02.04.2021)
|
||||||
- „Pinto of Fiomi“ → Pinto of Fiomi (*28.08.2016)
|
- „Pinto of Fiomi“ → Pinto of Fiomi (*28.08.2016)
|
||||||
|
- „Nala“ → Nala (*06.03.2015)
|
||||||
|
- „Hanami“ → Hanami (*10.09.2015)
|
||||||
- „Living Force's Idefix“ → Living Force's Idefix (*05.04.2016)
|
- „Living Force's Idefix“ → Living Force's Idefix (*05.04.2016)
|
||||||
- „Scarlett of Samsimar“ → Scarlett of Samsimar (*05.09.2018)
|
- „Osamu“ → Osamu (*10.12.2015)
|
||||||
|
- „Baiko“ → Baiko (*02.06.2019)
|
||||||
|
- „Olivia“ → Olivia (*23.07.2019)
|
||||||
|
- „Baiko“ → Baiko (*02.06.2019)
|
||||||
|
- „Olivia“ → Olivia (*23.07.2019)
|
||||||
- „Living Force's Idefix“ → Living Force's Idefix (*05.04.2016)
|
- „Living Force's Idefix“ → Living Force's Idefix (*05.04.2016)
|
||||||
- „Rosie of LennyLengo“ → Rosie of LennyLengo (*15.03.2016)
|
- „Rosie of LennyLengo“ → Rosie of LennyLengo (*15.03.2016)
|
||||||
|
- „Pan“ → Pan (*09.12.2014)
|
||||||
|
- „Gin“ → Gin (*05.02.2015)
|
||||||
|
- „Chiako“ → Chiako (*15.11.2016)
|
||||||
- „Stich von Privatzucht Gießen“ → Stich von Privatzucht Gießen (*01.09.2018)
|
- „Stich von Privatzucht Gießen“ → Stich von Privatzucht Gießen (*01.09.2018)
|
||||||
- „Lilo of LennyLengo“ → Lilo of LennyLengo (*04.11.2018)
|
- „Lilo of LennyLengo“ → Lilo of LennyLengo (*04.11.2018)
|
||||||
|
- „Ken'ichi“ → Ken'ichi (*01.03.2015)
|
||||||
|
- „Rumi“ → Rumi (*24.02.2017)
|
||||||
- „Living Force's Idefix“ → Living Force's Idefix (*05.04.2016)
|
- „Living Force's Idefix“ → Living Force's Idefix (*05.04.2016)
|
||||||
- „Rosie of LennyLengo“ → Rosie of LennyLengo (*15.03.2016)
|
- „Rosie of LennyLengo“ → Rosie of LennyLengo (*15.03.2016)
|
||||||
- „Living Force's Idefix“ → Living Force's Idefix (*05.04.2016)
|
- „Living Force's Idefix“ → Living Force's Idefix (*05.04.2016)
|
||||||
@@ -103,22 +146,11 @@ Gleiches Tier (Name+Datum), aber widersprüchliche Angaben in verschiedenen Date
|
|||||||
- „Little Hero of Black Forest“ → Little Hero of Black Forest (*22.02.2018)
|
- „Little Hero of Black Forest“ → Little Hero of Black Forest (*22.02.2018)
|
||||||
- „Little Runner's Destiny“ → Little Runner's Destiny (*02.03.2019)
|
- „Little Runner's Destiny“ → Little Runner's Destiny (*02.03.2019)
|
||||||
- „Chevrolet Camaro of Topolino“ → Chevrolet Camaro of Topolino (*10.04.2018)
|
- „Chevrolet Camaro of Topolino“ → Chevrolet Camaro of Topolino (*10.04.2018)
|
||||||
|
- „Danako“ → Danako (*22.08.2018)
|
||||||
- „Marlin of Black Forest“ → Marlin of Black Forest (*28.05.2019)
|
- „Marlin of Black Forest“ → Marlin of Black Forest (*28.05.2019)
|
||||||
|
- „Zenon von Elea“ → Zenon von Elea (*06.10.2019)
|
||||||
- „Nisha of Black Forest“ → Nisha of Black Forest (*28.03.2018)
|
- „Nisha of Black Forest“ → Nisha of Black Forest (*28.03.2018)
|
||||||
- „Nisha of Black Forest“ → Nisha of Black Forest (*28.03.2018)
|
- „Zenon von Elea“ → Zenon von Elea (*06.10.2019)
|
||||||
- „Dorie of Black Forest“ → Dorie of Black Forest (*28.05.2019)
|
|
||||||
- „Zadar from Zeko i ptica, Croatia“ → Zadar from Zeko i ptica, Croatia (*12.04.2019)
|
|
||||||
- „Living Force's Vally“ → Living Force's Vally (*01.11.2014)
|
|
||||||
- „Pinto of Fiomi“ → Pinto of Fiomi (*28.08.2016)
|
|
||||||
- „Hanse Renner's Poseidon“ → Hanse Renner's Poseidon (*14.08.2014)
|
|
||||||
- „Oscar of Black Forest“ → Oscar of Black Forest (*12.06.2019)
|
|
||||||
- „Hagrid Rubeus of Black Forest“ → Hagrid Rubeus of Black Forest (*18.07.2019)
|
|
||||||
- „Lilo of LennyLengo“ → Lilo of LennyLengo (*04.11.2018)
|
|
||||||
- „Chevrolet Camaro of Topolino“ → Chevrolet Camaro of Topolino (*10.04.2018)
|
|
||||||
- „Lilo of LennyLengo“ → Lilo of LennyLengo (*04.11.2018)
|
|
||||||
- „Rosie of LennyLengo“ → Rosie of LennyLengo (*15.03.2016)
|
|
||||||
- „Rosie of LennyLengo“ → Rosie of LennyLengo (*15.03.2016)
|
|
||||||
- „Little Runner's Destiny“ → Little Runner's Destiny (*02.03.2019)
|
|
||||||
|
|
||||||
## Nicht ins 8-Loci-Modell abgebildete Tokens (verbatim erhalten)
|
## Nicht ins 8-Loci-Modell abgebildete Tokens (verbatim erhalten)
|
||||||
|
|
||||||
@@ -126,22 +158,31 @@ Diese Tokens stehen weiter in `rawGenotype`/`unmappedTokens` — Entscheidung (M
|
|||||||
|
|
||||||
| Token | Vorkommen | Bedeutung (Vermutung) |
|
| Token | Vorkommen | Bedeutung (Vermutung) |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| `/+` | 6 | ? |
|
| `/+` | 7 | ? |
|
||||||
|
| `[meliert]` | 4 | ? |
|
||||||
|
| `[Dea/dea]` | 4 | ? |
|
||||||
|
| `(Kragen)` | 3 | ? |
|
||||||
|
| `/` | 3 | ? |
|
||||||
| `-g` | 2 | ? |
|
| `-g` | 2 | ? |
|
||||||
|
| `!DD` | 2 | ? |
|
||||||
|
| `Tumor` | 2 | ? |
|
||||||
|
| `!` | 2 | ? |
|
||||||
| `C(C)` | 2 | Schreibweise (C trägt c) |
|
| `C(C)` | 2 | Schreibweise (C trägt c) |
|
||||||
|
| `[dea/*]` | 2 | ? |
|
||||||
|
| `(KW` | 2 | ? |
|
||||||
| `Cc[]` | 1 | ? |
|
| `Cc[]` | 1 | ? |
|
||||||
|
| `[WFNZ-Maroon]` | 1 | ? |
|
||||||
|
| `/+April'2017` | 1 | ? |
|
||||||
|
| `[meliert]-[DP]` | 1 | ? |
|
||||||
|
| `!Knickschwanz-Gen!` | 1 | ? |
|
||||||
|
| `-09.09.2018` | 1 | ? |
|
||||||
|
| `[dea/dea]` | 1 | ? |
|
||||||
|
| `[DP?]` | 1 | ? |
|
||||||
|
| `/+Dezember'2014` | 1 | ? |
|
||||||
|
| `(Ansatz)` | 1 | ? |
|
||||||
|
| `[WFNZ][dea/*]` | 1 | ? |
|
||||||
| `-psp` | 1 | ? |
|
| `-psp` | 1 | ? |
|
||||||
| `G(G)` | 1 | ? |
|
| `G(G)` | 1 | ? |
|
||||||
| `/` | 1 | ? |
|
|
||||||
| `+2018` | 1 | ? |
|
|
||||||
| `chmchm` | 1 | Schreibweise (c[chm]c[chm]) |
|
|
||||||
| `c[chm]chm]` | 1 | ? |
|
|
||||||
| `Dea/dea]` | 1 | ? |
|
|
||||||
| `DD-Tumor` | 1 | ? |
|
|
||||||
| `bei` | 1 | ? |
|
|
||||||
| `Geschwistern` | 1 | ? |
|
|
||||||
| `C-D-` | 1 | ? |
|
|
||||||
| `-DD` | 1 | ? |
|
|
||||||
|
|
||||||
## Wurfchronik — Datenqualitäts-Hinweise
|
## Wurfchronik — Datenqualitäts-Hinweise
|
||||||
|
|
||||||
|
|||||||
379
tools/import/resolve_import.py
Normal file
379
tools/import/resolve_import.py
Normal file
@@ -0,0 +1,379 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
import os
|
||||||
|
import json
|
||||||
|
import uuid
|
||||||
|
import re
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
# Import normalization helpers from extract.py
|
||||||
|
import extract as ex
|
||||||
|
|
||||||
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
OUTPUT_DIR = os.path.join(HERE, "output")
|
||||||
|
SEEDS_PATH = os.path.join(HERE, "..", "..", "gerbil-manager-web", "src", "genetics", "colorVarietySeed.backend.json")
|
||||||
|
|
||||||
|
def generate_guid(key_str):
|
||||||
|
"""Generate a stable UUID string based on a key."""
|
||||||
|
return str(uuid.uuid5(uuid.NAMESPACE_DNS, key_str))
|
||||||
|
|
||||||
|
def parse_date_only(d):
|
||||||
|
"""Convert DD.MM.YYYY string to YYYY-MM-DD for JSON serialization."""
|
||||||
|
if not d:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
p = d.split(".")
|
||||||
|
if len(p) == 3:
|
||||||
|
day = int(p[0])
|
||||||
|
month = int(p[1])
|
||||||
|
year = int(p[2])
|
||||||
|
if len(p[2]) == 2:
|
||||||
|
year += 2000
|
||||||
|
# Ensure valid date
|
||||||
|
dt = datetime(year, month, day)
|
||||||
|
return dt.strftime("%Y-%m-%d")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return None
|
||||||
|
|
||||||
|
def save_to_csv(data_list, filepath):
|
||||||
|
import csv
|
||||||
|
if not data_list:
|
||||||
|
return
|
||||||
|
headers = list(data_list[0].keys())
|
||||||
|
with open(filepath, "w", encoding="utf-8-sig", newline="") as f:
|
||||||
|
writer = csv.writer(f, delimiter=";")
|
||||||
|
writer.writerow(headers)
|
||||||
|
for row in data_list:
|
||||||
|
values = []
|
||||||
|
for h in headers:
|
||||||
|
val = row[h]
|
||||||
|
if isinstance(val, (list, dict)):
|
||||||
|
values.append(json.dumps(val, ensure_ascii=False))
|
||||||
|
elif val is None:
|
||||||
|
values.append("")
|
||||||
|
else:
|
||||||
|
values.append(str(val))
|
||||||
|
writer.writerow(values)
|
||||||
|
|
||||||
|
def main():
|
||||||
|
print("Loading extracted JSON files...")
|
||||||
|
animals_file = os.path.join(OUTPUT_DIR, "animals.json")
|
||||||
|
litters_file = os.path.join(OUTPUT_DIR, "litters.json")
|
||||||
|
|
||||||
|
if not os.path.exists(animals_file) or not os.path.exists(litters_file):
|
||||||
|
print(f"Error: extract.py must be run first to generate animals.json and litters.json in {OUTPUT_DIR}")
|
||||||
|
return
|
||||||
|
|
||||||
|
with open(animals_file, "r", encoding="utf-8") as f:
|
||||||
|
animals = json.load(f)
|
||||||
|
with open(litters_file, "r", encoding="utf-8") as f:
|
||||||
|
litters = json.load(f)
|
||||||
|
|
||||||
|
# Load color variety seeds
|
||||||
|
print("Loading color variety seeds...")
|
||||||
|
with open(SEEDS_PATH, "r", encoding="utf-8") as f:
|
||||||
|
seeds = json.load(f)
|
||||||
|
|
||||||
|
# Map variety name -> Guid
|
||||||
|
variety_map = {}
|
||||||
|
for v in seeds:
|
||||||
|
# UUID is based on SortOrder + 1
|
||||||
|
variety_id = f"00000000-0000-0000-0000-{v['sortOrder'] + 1:012d}"
|
||||||
|
variety_map[v["name"].strip().lower()] = variety_id
|
||||||
|
|
||||||
|
# 1. Establish stable Guid maps
|
||||||
|
animal_guid_map = {a["id"]: generate_guid(f"animal-{a['id']}") for a in animals}
|
||||||
|
litter_guid_map = {l["id"]: generate_guid(f"litter-{l['id']}") for l in litters}
|
||||||
|
|
||||||
|
# Index animals for fast lookup by slug ID and by normalized name+dob
|
||||||
|
animal_by_id = {a["id"]: a for a in animals}
|
||||||
|
animal_by_name_dob = {}
|
||||||
|
for a in animals:
|
||||||
|
key = (ex.norm_name(a["name"]), ex.norm_dob(a["dob"]))
|
||||||
|
if key[0] and key[1]:
|
||||||
|
animal_by_name_dob.setdefault(key, []).append(a)
|
||||||
|
|
||||||
|
# 2. Extract and resolve unique Contacts (Breeders & Zuchten)
|
||||||
|
print("Extracting unique contacts...")
|
||||||
|
contact_names = set()
|
||||||
|
for a in animals:
|
||||||
|
if a.get("breeder"):
|
||||||
|
contact_names.add(a["breeder"].strip())
|
||||||
|
if a.get("zucht"):
|
||||||
|
contact_names.add(a["zucht"].strip())
|
||||||
|
for l in litters:
|
||||||
|
if l.get("damZucht"):
|
||||||
|
contact_names.add(l["damZucht"].strip())
|
||||||
|
if l.get("sireZucht"):
|
||||||
|
contact_names.add(l["sireZucht"].strip())
|
||||||
|
|
||||||
|
contact_guid_map = {}
|
||||||
|
resolved_contacts = []
|
||||||
|
for idx, name in enumerate(sorted(contact_names)):
|
||||||
|
norm = name.strip().lower()
|
||||||
|
if not norm or norm in contact_guid_map:
|
||||||
|
continue
|
||||||
|
c_guid = generate_guid(f"contact-{norm}")
|
||||||
|
contact_guid_map[norm] = c_guid
|
||||||
|
resolved_contacts.append({
|
||||||
|
"Id": c_guid,
|
||||||
|
"Name": name,
|
||||||
|
"ZuchtName": name if "zucht" in norm or "clan" in norm or "runner" in norm else "",
|
||||||
|
"City": "",
|
||||||
|
"Email": "",
|
||||||
|
"Homepage": "",
|
||||||
|
"Phone": "",
|
||||||
|
"Address": ""
|
||||||
|
})
|
||||||
|
|
||||||
|
# 3. Resolve parent relationships for litters
|
||||||
|
print("Resolving litter parents...")
|
||||||
|
litter_parent_map = {} # litter_id -> (father_guid, mother_guid)
|
||||||
|
|
||||||
|
# Pre-map parents using offspring parentRefs
|
||||||
|
for a in animals:
|
||||||
|
litter_ref = a.get("litterRef")
|
||||||
|
if not litter_ref or not litter_ref.get("litterId"):
|
||||||
|
continue
|
||||||
|
|
||||||
|
l_id = litter_ref["litterId"]
|
||||||
|
l_parents = litter_parent_map.setdefault(l_id, [None, None]) # [father, mother]
|
||||||
|
|
||||||
|
for p_ref in a.get("parentRefs", []):
|
||||||
|
p_key = (ex.norm_name(p_ref["name"]), ex.norm_dob(p_ref["dob"]))
|
||||||
|
p_candidates = animal_by_name_dob.get(p_key, [])
|
||||||
|
if p_candidates:
|
||||||
|
p_guid = animal_guid_map[p_candidates[0]["id"]]
|
||||||
|
if p_ref["roleGuess"] == "father":
|
||||||
|
l_parents[0] = p_guid
|
||||||
|
elif p_ref["roleGuess"] == "mother":
|
||||||
|
l_parents[1] = p_guid
|
||||||
|
|
||||||
|
# Fallback lookup for parents by name from litter details if offspring has no parentRefs
|
||||||
|
for l in litters:
|
||||||
|
l_parents = litter_parent_map.setdefault(l["id"], [None, None])
|
||||||
|
l_date_str = parse_date_only(l["date"])
|
||||||
|
if not l_date_str:
|
||||||
|
continue
|
||||||
|
l_date = datetime.strptime(l_date_str, "%Y-%m-%d")
|
||||||
|
|
||||||
|
# Fallback Father
|
||||||
|
if not l_parents[0] and l.get("sireName"):
|
||||||
|
sire_norm = ex.norm_name(l["sireName"])
|
||||||
|
# Find an animal with this name born before the litter
|
||||||
|
candidates = []
|
||||||
|
for (name_norm, dob_norm), grp in animal_by_name_dob.items():
|
||||||
|
if name_norm == sire_norm:
|
||||||
|
for cand in grp:
|
||||||
|
c_dob_str = parse_date_only(cand["dob"])
|
||||||
|
if c_dob_str:
|
||||||
|
c_dob = datetime.strptime(c_dob_str, "%Y-%m-%d")
|
||||||
|
if c_dob < l_date:
|
||||||
|
candidates.append(cand)
|
||||||
|
if len(candidates) == 1:
|
||||||
|
l_parents[0] = animal_guid_map[candidates[0]["id"]]
|
||||||
|
|
||||||
|
# Fallback Mother
|
||||||
|
if not l_parents[1] and l.get("damName"):
|
||||||
|
dam_norm = ex.norm_name(l["damName"])
|
||||||
|
candidates = []
|
||||||
|
for (name_norm, dob_norm), grp in animal_by_name_dob.items():
|
||||||
|
if name_norm == dam_norm:
|
||||||
|
for cand in grp:
|
||||||
|
c_dob_str = parse_date_only(cand["dob"])
|
||||||
|
if c_dob_str:
|
||||||
|
c_dob = datetime.strptime(c_dob_str, "%Y-%m-%d")
|
||||||
|
if c_dob < l_date:
|
||||||
|
candidates.append(cand)
|
||||||
|
if len(candidates) == 1:
|
||||||
|
l_parents[1] = animal_guid_map[candidates[0]["id"]]
|
||||||
|
|
||||||
|
# 4. Construct Litters collection
|
||||||
|
resolved_litters = []
|
||||||
|
for l in litters:
|
||||||
|
l_guid = litter_guid_map[l["id"]]
|
||||||
|
p_father, p_mother = litter_parent_map.get(l["id"], [None, None])
|
||||||
|
l_date = parse_date_only(l["date"])
|
||||||
|
if not l_date:
|
||||||
|
continue # Skip litters without dates
|
||||||
|
|
||||||
|
# Inzucht/Frühsterblichkeit mapping
|
||||||
|
deaths_8w = None
|
||||||
|
if l.get("diedLater") is not None or l.get("stillborn") is not None:
|
||||||
|
deaths_8w = (l.get("stillborn") or 0) + (l.get("diedLater") or 0)
|
||||||
|
|
||||||
|
resolved_litters.append({
|
||||||
|
"Id": l_guid,
|
||||||
|
"Name": f"Wurf {l['litterId']}".strip(),
|
||||||
|
"Date": l_date,
|
||||||
|
"TotalBorn": l.get("totalBorn"),
|
||||||
|
"DeathsWithin8Weeks": deaths_8w,
|
||||||
|
"FatherId": p_father,
|
||||||
|
"MotherId": p_mother,
|
||||||
|
"ExpectedGoHomeDate": None,
|
||||||
|
"Notes": l.get("note") if l.get("note") else None,
|
||||||
|
"PairingCode": l.get("zuchtnummer") if l.get("zuchtnummer") else None,
|
||||||
|
"ExternalRef": l["id"],
|
||||||
|
"LitterLetter": l["litterId"] if len(l["litterId"]) <= 2 else None
|
||||||
|
})
|
||||||
|
|
||||||
|
# 5. Determine Residency & Status
|
||||||
|
# C# Residency Rule: IsResident iff Zuchtname matches Clan kennel, OR is parent of resident
|
||||||
|
resident_guids = set()
|
||||||
|
# Step A: Direct Zucht name match
|
||||||
|
for a in animals:
|
||||||
|
if ex.is_clan_zucht(a.get("zucht")) or ex.is_clan_zucht(a.get("zuchtCanon")):
|
||||||
|
resident_guids.add(animal_guid_map[a["id"]])
|
||||||
|
|
||||||
|
# Step B: Parent of Clan offspring
|
||||||
|
# Re-run propagation loop a few times to cover multiple generations
|
||||||
|
for _ in range(5):
|
||||||
|
for l in litters:
|
||||||
|
l_guid = litter_guid_map[l["id"]]
|
||||||
|
# Check if any offspring of this litter is resident
|
||||||
|
offspring_guids = [animal_guid_map[a["id"]] for a in animals if a.get("litterRef") and a["litterRef"].get("litterId") == l["id"]]
|
||||||
|
has_resident_offspring = any(og in resident_guids for og in offspring_guids)
|
||||||
|
|
||||||
|
if has_resident_offspring:
|
||||||
|
p_father, p_mother = litter_parent_map.get(l["id"], [None, None])
|
||||||
|
if p_father:
|
||||||
|
resident_guids.add(p_father)
|
||||||
|
if p_mother:
|
||||||
|
resident_guids.add(p_mother)
|
||||||
|
|
||||||
|
# 6. Construct Gerbils collection
|
||||||
|
resolved_gerbils = []
|
||||||
|
resolved_photos = []
|
||||||
|
|
||||||
|
for a in animals:
|
||||||
|
a_guid = animal_guid_map[a["id"]]
|
||||||
|
|
||||||
|
# Resolve ColorVariety
|
||||||
|
cv_id = None
|
||||||
|
fb_key = a["farbschlag"].strip().lower()
|
||||||
|
if fb_key in variety_map:
|
||||||
|
cv_id = variety_map[fb_key]
|
||||||
|
else:
|
||||||
|
# Try matching variants
|
||||||
|
for fbv in a.get("farbschlagVariants", []):
|
||||||
|
fbv_key = fbv.strip().lower()
|
||||||
|
if fbv_key in variety_map:
|
||||||
|
cv_id = variety_map[fbv_key]
|
||||||
|
break
|
||||||
|
|
||||||
|
# Resolve Litter
|
||||||
|
l_id = None
|
||||||
|
l_ref = a.get("litterRef")
|
||||||
|
if l_ref and l_ref.get("litterId") and l_ref["litterId"] in litter_guid_map:
|
||||||
|
l_id = litter_guid_map[l_ref["litterId"]]
|
||||||
|
|
||||||
|
# Resolve Contacts
|
||||||
|
origin_contact_id = None
|
||||||
|
breeder_norm = a.get("breeder", "").strip().lower()
|
||||||
|
zucht_norm = a.get("zucht", "").strip().lower()
|
||||||
|
if breeder_norm in contact_guid_map:
|
||||||
|
origin_contact_id = contact_guid_map[breeder_norm]
|
||||||
|
elif zucht_norm in contact_guid_map:
|
||||||
|
origin_contact_id = contact_guid_map[zucht_norm]
|
||||||
|
|
||||||
|
# Dates
|
||||||
|
dob = parse_date_only(a.get("dob"))
|
||||||
|
death = parse_date_only(a.get("death"))
|
||||||
|
|
||||||
|
# Gender conversion: M -> male, W -> female
|
||||||
|
gender = "Unknown"
|
||||||
|
if a.get("gender") == "M":
|
||||||
|
gender = "male"
|
||||||
|
elif a.get("gender") == "W":
|
||||||
|
gender = "female"
|
||||||
|
|
||||||
|
# Determine Residency
|
||||||
|
is_resident = a_guid in resident_guids
|
||||||
|
|
||||||
|
# Determine Status
|
||||||
|
status = "Breeding"
|
||||||
|
if death:
|
||||||
|
status = "Deceased"
|
||||||
|
elif not is_resident:
|
||||||
|
status = "GivenAway"
|
||||||
|
else:
|
||||||
|
# Age presumed deceased (>7 years)
|
||||||
|
if dob:
|
||||||
|
dt_dob = datetime.strptime(dob, "%Y-%m-%d")
|
||||||
|
dt_now = datetime.now()
|
||||||
|
age_years = (dt_now - dt_dob).days / 365.25
|
||||||
|
if age_years >= 7.0:
|
||||||
|
status = "Deceased"
|
||||||
|
else:
|
||||||
|
status = "Breeding" # default to breeding for resident stock
|
||||||
|
|
||||||
|
# Raw Import payload
|
||||||
|
raw_import_payload = json.dumps({
|
||||||
|
"rawGenotype": a["genotype"]["rawGenotype"],
|
||||||
|
"unmappedTokens": a["genotype"]["unmappedTokens"],
|
||||||
|
"breederText": a.get("breeder", "")
|
||||||
|
}, ensure_ascii=False)
|
||||||
|
|
||||||
|
resolved_gerbils.append({
|
||||||
|
"Id": a_guid,
|
||||||
|
"Name": a["name"],
|
||||||
|
"Gender": gender,
|
||||||
|
"Status": status,
|
||||||
|
"LitterId": l_id,
|
||||||
|
"OriginContactId": origin_contact_id,
|
||||||
|
"ReceiverContactId": None,
|
||||||
|
"EnclosureId": None,
|
||||||
|
"ColorVarietyId": cv_id,
|
||||||
|
"DateOfBirth": dob,
|
||||||
|
"DateOfDeath": death,
|
||||||
|
"CauseOfDeath": None,
|
||||||
|
"GoHomeDate": None,
|
||||||
|
"Genotype": a["genotype"]["rawGenotype"] if a["genotype"]["rawGenotype"] else None,
|
||||||
|
"Notes": None,
|
||||||
|
"ImportSource": "docx-export",
|
||||||
|
"ExternalRef": a["id"],
|
||||||
|
"RawImportData": raw_import_payload,
|
||||||
|
"OriginBreeder": a.get("breeder") if a.get("breeder") else (a.get("zucht") if a.get("zucht") else None),
|
||||||
|
"NameSearch": ex.norm_name(a["name"]),
|
||||||
|
"CharacterTraits": [],
|
||||||
|
"CharacterNote": None,
|
||||||
|
"IsDeaf": a.get("deaf"),
|
||||||
|
"IsResident": is_resident
|
||||||
|
})
|
||||||
|
|
||||||
|
# Attach Photos
|
||||||
|
for idx, photo_rel in enumerate(a.get("photos", [])):
|
||||||
|
photo_guid = generate_guid(f"photo-{photo_rel}")
|
||||||
|
resolved_photos.append({
|
||||||
|
"Id": photo_guid,
|
||||||
|
"GerbilId": a_guid,
|
||||||
|
"FileName": os.path.basename(photo_rel),
|
||||||
|
"SortOrder": idx
|
||||||
|
})
|
||||||
|
|
||||||
|
# Save resolved data
|
||||||
|
resolved_data = {
|
||||||
|
"Contacts": resolved_contacts,
|
||||||
|
"Litters": resolved_litters,
|
||||||
|
"Gerbils": resolved_gerbils,
|
||||||
|
"GerbilPhotos": resolved_photos
|
||||||
|
}
|
||||||
|
|
||||||
|
resolved_path = os.path.join(OUTPUT_DIR, "resolved_import.json")
|
||||||
|
with open(resolved_path, "w", encoding="utf-8") as f:
|
||||||
|
json.dump(resolved_data, f, ensure_ascii=False, indent=2)
|
||||||
|
|
||||||
|
# Save resolved CSV files
|
||||||
|
save_to_csv(resolved_contacts, os.path.join(OUTPUT_DIR, "resolved_contacts.csv"))
|
||||||
|
save_to_csv(resolved_litters, os.path.join(OUTPUT_DIR, "resolved_litters.csv"))
|
||||||
|
save_to_csv(resolved_gerbils, os.path.join(OUTPUT_DIR, "resolved_gerbils.csv"))
|
||||||
|
save_to_csv(resolved_photos, os.path.join(OUTPUT_DIR, "resolved_photos.csv"))
|
||||||
|
|
||||||
|
print(f"Success! Saved resolved database-ready records to {resolved_path} (and CSV counterparts):")
|
||||||
|
print(f" Contacts: {len(resolved_contacts)}")
|
||||||
|
print(f" Litters: {len(resolved_litters)}")
|
||||||
|
print(f" Gerbils: {len(resolved_gerbils)}")
|
||||||
|
print(f" GerbilPhotos: {len(resolved_photos)}")
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Reference in New Issue
Block a user