feat(migration): implement offline-resolved database-ready ingestion, fix circular FKs, duplicate contacts and parser scanning bugs
This commit is contained in:
192
GerbilManagerWebAPI/Import/IngestResolvedService.cs
Normal file
192
GerbilManagerWebAPI/Import/IngestResolvedService.cs
Normal file
@@ -0,0 +1,192 @@
|
||||
using System.Text.Json;
|
||||
using GerbilManagerWebAPI.Models;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace GerbilManagerWebAPI.Import
|
||||
{
|
||||
public sealed class IngestResolvedService
|
||||
{
|
||||
private readonly ApplicationContext _db;
|
||||
private readonly string _resolvedJsonPath;
|
||||
|
||||
public IngestResolvedService(ApplicationContext db, IConfiguration config, IWebHostEnvironment env)
|
||||
{
|
||||
_db = db;
|
||||
var sourceDir = config["Import:SourcePath"]
|
||||
?? Path.GetFullPath(Path.Combine(env.ContentRootPath, "..", "tools", "import", "output"));
|
||||
_resolvedJsonPath = Path.Combine(sourceDir, "resolved_import.json");
|
||||
}
|
||||
|
||||
public async Task<string> RunAsync()
|
||||
{
|
||||
if (!File.Exists(_resolvedJsonPath))
|
||||
{
|
||||
return $"Error: Resolved import file not found at {_resolvedJsonPath}. Run resolve_import.py first.";
|
||||
}
|
||||
|
||||
string jsonContent = await File.ReadAllTextAsync(_resolvedJsonPath);
|
||||
var jsonOptions = new JsonSerializerOptions
|
||||
{
|
||||
PropertyNameCaseInsensitive = true
|
||||
};
|
||||
jsonOptions.Converters.Add(new System.Text.Json.Serialization.JsonStringEnumConverter());
|
||||
var data = JsonSerializer.Deserialize<ResolvedImportData>(jsonContent, jsonOptions);
|
||||
|
||||
if (data == null)
|
||||
{
|
||||
return "Error: Failed to deserialize resolved_import.json.";
|
||||
}
|
||||
|
||||
// 1. Provider-agnostic clean wipe of dependent tables to prevent duplicate keys
|
||||
if (_db.Database.ProviderName == "Microsoft.EntityFrameworkCore.InMemory")
|
||||
{
|
||||
// Break circular references first
|
||||
foreach (var l in _db.Litters)
|
||||
{
|
||||
l.FatherId = null;
|
||||
l.MotherId = null;
|
||||
}
|
||||
foreach (var g in _db.Gerbils)
|
||||
{
|
||||
g.LitterId = null;
|
||||
}
|
||||
await _db.SaveChangesAsync();
|
||||
|
||||
_db.GerbilPhotos.RemoveRange(_db.GerbilPhotos);
|
||||
_db.WeightRecords.RemoveRange(_db.WeightRecords);
|
||||
_db.HealthRecords.RemoveRange(_db.HealthRecords);
|
||||
_db.Gerbils.RemoveRange(_db.Gerbils);
|
||||
_db.Litters.RemoveRange(_db.Litters);
|
||||
await _db.SaveChangesAsync();
|
||||
}
|
||||
else
|
||||
{
|
||||
// Break circular references first using ExecuteUpdateAsync
|
||||
await _db.Litters.ExecuteUpdateAsync(s => s
|
||||
.SetProperty(l => l.FatherId, (Guid?)null)
|
||||
.SetProperty(l => l.MotherId, (Guid?)null));
|
||||
|
||||
await _db.Gerbils.ExecuteUpdateAsync(s => s
|
||||
.SetProperty(g => g.LitterId, (Guid?)null));
|
||||
|
||||
await _db.GerbilPhotos.ExecuteDeleteAsync();
|
||||
await _db.WeightRecords.ExecuteDeleteAsync();
|
||||
await _db.HealthRecords.ExecuteDeleteAsync();
|
||||
await _db.Gerbils.ExecuteDeleteAsync();
|
||||
await _db.Litters.ExecuteDeleteAsync();
|
||||
}
|
||||
|
||||
// 2. Import Contacts (Only add new ones to preserve manually entered contacts)
|
||||
var existingContactIds = await _db.Contacts.Select(c => c.Id).ToHashSetAsync();
|
||||
var addedContactIds = new HashSet<Guid>();
|
||||
int contactsAdded = 0;
|
||||
foreach (var c in data.Contacts)
|
||||
{
|
||||
if (!existingContactIds.Contains(c.Id) && addedContactIds.Add(c.Id))
|
||||
{
|
||||
_db.Contacts.Add(c);
|
||||
contactsAdded++;
|
||||
}
|
||||
}
|
||||
await _db.SaveChangesAsync();
|
||||
|
||||
// 3. Pass 1: Insert all Litters with parent references set to null to avoid FK errors
|
||||
var littersToInsert = new List<Litter>();
|
||||
foreach (var l in data.Litters)
|
||||
{
|
||||
littersToInsert.Add(new Litter
|
||||
{
|
||||
Id = l.Id,
|
||||
Name = l.Name,
|
||||
Date = l.Date,
|
||||
TotalBorn = l.TotalBorn,
|
||||
DeathsWithin8Weeks = l.DeathsWithin8Weeks,
|
||||
FatherId = null, // Set in Pass 2
|
||||
MotherId = null, // Set in Pass 2
|
||||
ExpectedGoHomeDate = l.ExpectedGoHomeDate,
|
||||
Notes = l.Notes,
|
||||
PairingCode = l.PairingCode,
|
||||
ExternalRef = l.ExternalRef,
|
||||
LitterLetter = l.LitterLetter
|
||||
});
|
||||
}
|
||||
_db.Litters.AddRange(littersToInsert);
|
||||
await _db.SaveChangesAsync();
|
||||
|
||||
// 4. Pass 1: Insert all Gerbils with LitterId set to null to avoid FK errors
|
||||
var gerbilsToInsert = new List<Gerbil>();
|
||||
foreach (var g in data.Gerbils)
|
||||
{
|
||||
gerbilsToInsert.Add(new Gerbil
|
||||
{
|
||||
Id = g.Id,
|
||||
Name = g.Name,
|
||||
Gender = g.Gender,
|
||||
Status = g.Status,
|
||||
LitterId = null, // Set in Pass 2
|
||||
OriginContactId = g.OriginContactId,
|
||||
ReceiverContactId = g.ReceiverContactId,
|
||||
EnclosureId = g.EnclosureId,
|
||||
ColorVarietyId = g.ColorVarietyId,
|
||||
DateOfBirth = g.DateOfBirth,
|
||||
DateOfDeath = g.DateOfDeath,
|
||||
CauseOfDeath = g.CauseOfDeath,
|
||||
GoHomeDate = g.GoHomeDate,
|
||||
Genotype = g.Genotype,
|
||||
Notes = g.Notes,
|
||||
ImportSource = g.ImportSource,
|
||||
ExternalRef = g.ExternalRef,
|
||||
RawImportData = g.RawImportData,
|
||||
OriginBreeder = g.OriginBreeder,
|
||||
NameSearch = g.NameSearch,
|
||||
CharacterTraits = g.CharacterTraits,
|
||||
CharacterNote = g.CharacterNote,
|
||||
IsDeaf = g.IsDeaf,
|
||||
IsResident = g.IsResident
|
||||
});
|
||||
}
|
||||
_db.Gerbils.AddRange(gerbilsToInsert);
|
||||
await _db.SaveChangesAsync();
|
||||
|
||||
// 5. Pass 2: Set the actual foreign key relationships
|
||||
// Update Litters with their parent animal IDs
|
||||
var littersInDb = await _db.Litters.ToDictionaryAsync(l => l.Id);
|
||||
foreach (var l in data.Litters)
|
||||
{
|
||||
if (littersInDb.TryGetValue(l.Id, out var dbLitter))
|
||||
{
|
||||
dbLitter.FatherId = l.FatherId;
|
||||
dbLitter.MotherId = l.MotherId;
|
||||
}
|
||||
}
|
||||
|
||||
// Update Gerbils with their actual LitterId
|
||||
var gerbilsInDb = await _db.Gerbils.ToDictionaryAsync(g => g.Id);
|
||||
foreach (var g in data.Gerbils)
|
||||
{
|
||||
if (gerbilsInDb.TryGetValue(g.Id, out var dbGerbil))
|
||||
{
|
||||
dbGerbil.LitterId = g.LitterId;
|
||||
}
|
||||
}
|
||||
await _db.SaveChangesAsync();
|
||||
|
||||
// 6. Insert all Photos
|
||||
foreach (var p in data.GerbilPhotos)
|
||||
{
|
||||
_db.GerbilPhotos.Add(p);
|
||||
}
|
||||
await _db.SaveChangesAsync();
|
||||
|
||||
return $"Ingestion successful! Imported {contactsAdded} contacts, {data.Litters.Count} litters, {data.Gerbils.Count} gerbils, and {data.GerbilPhotos.Count} photos.";
|
||||
}
|
||||
}
|
||||
|
||||
public class ResolvedImportData
|
||||
{
|
||||
public List<Contact> Contacts { get; set; } = new();
|
||||
public List<Litter> Litters { get; set; } = new();
|
||||
public List<Gerbil> Gerbils { get; set; } = new();
|
||||
public List<GerbilPhoto> GerbilPhotos { get; set; } = new();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user