Gulum/GerbilManager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(import): resolve color variety ID mappings mismatch, support unnamed parents in virtual litters, and implement SaleContract updates

Browse Source
This commit is contained in:
Gulum
2026-06-21 21:57:40 +02:00
parent b83e96f552
commit e460cd5905
29 changed files with 3783 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

167
tools/import/merge_and_resolve.py
View File

@@ -231,6 +231,34 @@ def get_normalized_contact_name(name):
return name, True
def get_normalized_gerbil_name(name):
if not name:
return ""
n = name.strip()
norm_key = "".join(c for c in n.lower() if c.isalnum())
gerbil_norm_map = {
"samgenshellyvdbuntenfellnasen": "Sammy gen. Shelly von den bunten Fellnasen",
"sammygenshellyvdbuntenfellnasen": "Sammy gen. Shelly von den bunten Fellnasen",
"schmidt": "Schmidti",
"sheila": "Sheila of Ulmer Strolche",
"shinichi": "Shinichi von PZ Mücke",
"silenosgenadonis": "Silenos gen. Adonis von den Kleinen Chaoten",
"silenosgenadonisvdkleinenchaoten": "Silenos gen. Adonis von den Kleinen Chaoten",
"silver": "Silver von den kleinen Chaoten",
"snoops": "Snoopsi",
"sokrates": "Sokrates von den Kleinen Chaoten",
"splash": "Slash",
"teiko": "Teiko von den kleinen Chaoten",
"trixy": "Trixxy von den Kleinen Chaoten",
"unique": "Unique of Wild Dreams",
}
if norm_key in gerbil_norm_map:
return gerbil_norm_map[norm_key]
return n
def get_call_name(name):
if not name:
return ""
@@ -485,15 +513,42 @@ def main():
print("Loading color variety seeds...")
variety_map = {}
variety_genotypes = {}
# Load from C# ApplicationContext.cs catalog for stable database GUIDs (index + 1)
here = os.path.dirname(os.path.abspath(__file__))
app_context_path = os.path.abspath(os.path.join(here, "../../GerbilManagerWebAPI/ApplicationContext.cs"))
cs_name_to_id = {}
if os.path.exists(app_context_path):
with open(app_context_path, 'r', encoding='utf-8') as f:
content = f.read()
catalog_match = re.search(r'catalog\s*=\s*\{(.*?)\};', content, re.DOTALL)
if catalog_match:
block = catalog_match.group(1)
entries = re.findall(r'\(\s*"([^"]+)"\s*,\s*"([^"]+)"\s*,\s*(\d+)\s*\)', block)
for idx, (name, genotype, sort_order) in enumerate(entries):
variety_id = f"00000000-0000-0000-0000-{idx + 1:012d}"
cs_name_to_id[name.strip().lower()] = variety_id
variety_map[name.strip().lower()] = variety_id
variety_genotypes[variety_id] = genotype.strip()
else:
print(f"Warning: ApplicationContext.cs not found at {app_context_path}")
if os.path.exists(SEEDS_PATH):
with open(SEEDS_PATH, "r", encoding="utf-8") as f:
seeds = json.load(f)
for v in seeds:
variety_id = f"00000000-0000-0000-0000-{v['sortOrder'] + 1:012d}"
variety_map[v["name"].strip().lower()] = variety_id
name_lower = v["name"].strip().lower()
variety_id = cs_name_to_id.get(name_lower)
if not variety_id:
variety_id = f"00000000-0000-0000-0000-{v['sortOrder'] + 1:012d}"
variety_map[name_lower] = variety_id
# Map English name if present
if "english" in v and v["english"]:
variety_map[v["english"].strip().lower()] = variety_id
variety_genotypes[variety_id] = v.get("canonicalGenotype")
if variety_id not in variety_genotypes:
variety_genotypes[variety_id] = v.get("canonicalGenotype")
else:
print(f"Warning: Seeds path not found at {SEEDS_PATH}")
@@ -694,8 +749,8 @@ def main():
for dl in docx_litters:
l_name = dl["litterId"]
dob_val = parse_date(dl["dob"])
f_name = dl["fatherName"]
m_name = dl["motherName"]
f_name = get_normalized_gerbil_name(dl["fatherName"])
m_name = get_normalized_gerbil_name(dl["motherName"])
ws_code = dl["wsCode"]
note_val = dl.get("note")
@@ -719,7 +774,7 @@ def main():
raw_litters.append({
"Id": l_scoped_id,
"Name": l_name,
"Date": dob_val or "0001-01-01",
"Date": dob_val,
"TotalBorn": total_born,
"DeathsWithin8Weeks": deaths_8w,
"FatherId": generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"), # placeholder
@@ -784,8 +839,8 @@ def main():
# Pre-index Wurfchronik litters from markdown
md_litters_idx = {}
for rl in raw_litters:
f_name = rl.get("FatherName") or rl.get("fatherName") or rl.get("ParentMaleName") or rl.get("parentMaleName")
m_name = rl.get("MotherName") or rl.get("motherName") or rl.get("ParentFemaleName") or rl.get("parentFemaleName")
f_name = get_normalized_gerbil_name(rl.get("FatherName") or rl.get("fatherName") or rl.get("ParentMaleName") or rl.get("parentMaleName"))
m_name = get_normalized_gerbil_name(rl.get("MotherName") or rl.get("motherName") or rl.get("ParentFemaleName") or rl.get("parentFemaleName"))
ldate = parse_date(rl.get("Date") or rl.get("date") or rl.get("DateOfBirth") or rl.get("dateOfBirth"))
if f_name and m_name and ldate:
key = (normalize_name(f_name), normalize_name(m_name), ldate)
@@ -800,8 +855,8 @@ def main():
a["_mapped_litter_scoped_id"] = None
if father_ref and mother_ref:
f_name = father_ref.get("name")
m_name = mother_ref.get("name")
f_name = get_normalized_gerbil_name(father_ref.get("name"))
m_name = get_normalized_gerbil_name(mother_ref.get("name"))
dob_val = parse_date(a.get("dob"))
mapped_litter = None
@@ -827,14 +882,16 @@ def main():
m_dob = parse_date(mother_ref.get("dob"))
for p_cand in stammbaum_only_animals:
cand_call_norm = normalize_name(get_call_name(p_cand["name"]))
if cand_call_norm == normalize_name(f_name) or normalize_name(p_cand["name"]) == normalize_name(f_name):
cand_call_norm = normalize_name(get_call_name(p_cand["name"])) or "unbekannt"
cand_name_norm = normalize_name(p_cand["name"]) or "unbekannt"
if cand_call_norm == normalize_name(f_name) or cand_name_norm == normalize_name(f_name):
if not f_dob or parse_date(p_cand.get("dob")) == f_dob:
f_scoped_id = generate_guid(f"stammbaum-animal-{p_cand['id']}")
break
for p_cand in stammbaum_only_animals:
cand_call_norm = normalize_name(get_call_name(p_cand["name"]))
if cand_call_norm == normalize_name(m_name) or normalize_name(p_cand["name"]) == normalize_name(m_name):
cand_call_norm = normalize_name(get_call_name(p_cand["name"])) or "unbekannt"
cand_name_norm = normalize_name(p_cand["name"]) or "unbekannt"
if cand_call_norm == normalize_name(m_name) or cand_name_norm == normalize_name(m_name):
if not m_dob or parse_date(p_cand.get("dob")) == m_dob:
m_scoped_id = generate_guid(f"stammbaum-animal-{p_cand['id']}")
break
@@ -842,7 +899,7 @@ def main():
raw_litters.append({
"Id": l_scoped_id,
"Name": f"Wurf von {f_name} + {m_name}",
"Date": dob_val or "0001-01-01",
"Date": dob_val,
"TotalBorn": None,
"DeathsWithin8Weeks": None,
"FatherId": f_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"),
@@ -924,8 +981,6 @@ def main():
name_val = "Wurf"
dob_val = parse_date(rl.get("Date") or rl.get("date") or rl.get("DateOfBirth") or rl.get("dateOfBirth"))
if not dob_val:
dob_val = "0001-01-01"
new_guid = rl["_scoped_id"]
if not new_guid:
@@ -986,7 +1041,7 @@ def main():
def get_litter_date(l_id):
if l_id in litter_by_scoped_id:
d = litter_by_scoped_id[l_id]["Date"]
if d and d != "0001-01-01":
if d:
return d
return None
@@ -994,7 +1049,7 @@ def main():
all_processed_gerbils = []
for rg in raw_gerbils:
filename = rg.get("_filename")
name_val = rg.get("Name") or rg.get("name") or rg.get("callName")
name_val = get_normalized_gerbil_name(rg.get("Name") or rg.get("name") or rg.get("callName"))
if not name_val:
name_val = "Unbekannt"
@@ -1142,7 +1197,7 @@ def main():
# Map and append stammbaum animals to all_processed_gerbils
for a in stammbaum_only_animals:
a_id = a["id"]
name_val = a["name"]
name_val = get_normalized_gerbil_name(a["name"])
gender_val = str(a.get("gender") or "").lower().strip()
if gender_val in ["m", "male"]:
@@ -1166,7 +1221,7 @@ def main():
dt_dob = datetime.strptime(dob_val, "%Y-%m-%d")
dt_now = datetime.now()
age_years = (dt_now - dt_dob).days / 365.25
if age_years >= 7.0:
if age_years >= 6.0:
status = "Deceased"
except Exception:
pass
@@ -1249,7 +1304,7 @@ def main():
# Map and append docx animals to all_processed_gerbils
for idx, da in enumerate(docx_animals):
name_val = da["name"]
name_val = get_normalized_gerbil_name(da["name"])
gender = da["gender"]
dob_val = parse_date(da.get("litterDob"))
@@ -1335,7 +1390,7 @@ def main():
for l in resolved_litters:
ld = l["Date"]
if ld and ld != "0001-01-01":
if ld:
for pid in [l["FatherId"], l["MotherId"]]:
if pid:
parent_litter_dates.setdefault(pid, []).append(ld)
@@ -1459,6 +1514,8 @@ def main():
if ph not in merged_photos:
merged_photos.append(ph)
if not best_g.get("_old_scoped_litter_id") and g.get("_old_scoped_litter_id"):
best_g["_old_scoped_litter_id"] = g["_old_scoped_litter_id"]
if not best_g["LitterId"] and g["LitterId"]:
best_g["LitterId"] = g["LitterId"]
if not best_g["DateOfBirth"] and g["DateOfBirth"]:
@@ -1499,6 +1556,22 @@ def main():
if not any(kw in g["Notes"].lower() for kw in ["parent listed", "mutter von", "vater von", "dam of", "sire of"]):
merged_notes.append(g["Notes"])
# Reconcile fields based on number of source files supporting them
for field in ["DateOfBirth", "DateOfDeath", "Gender", "Genotype", "ColorVarietyId"]:
votes = {}
for g in sub:
val = g.get(field)
if val and val != "unknown":
# count source files
sources_count = len(str(g.get("ImportSource") or "").split(","))
votes[val] = votes.get(val, 0) + sources_count
if votes:
best_val = max(votes, key=votes.get)
best_g[field] = best_val
# Keep helper fields in sync if we changed DateOfBirth
if field == "DateOfBirth":
best_g["_birth_date"] = best_val
best_g["_eff_dob"] = best_val or "2010-01-01"
if merged_notes:
best_g["Notes"] = " | ".join(merged_notes)
@@ -1513,13 +1586,49 @@ def main():
print(f"Deduplicated to {len(resolved_gerbils)} unique gerbil records.")
# Apply age-based death threshold (6.0 years) to all resolved gerbils
dt_now = datetime.now()
for g in resolved_gerbils:
if g.get("Status") != "Deceased" and g.get("Status") != "GivenAway":
if not g.get("DateOfDeath") and not g.get("ReceiverContactId"):
dob_str = g.get("DateOfBirth")
if dob_str:
try:
dt_dob = datetime.strptime(dob_str, "%Y-%m-%d")
age_years = (dt_now - dt_dob).days / 365.25
if age_years >= 6.0:
g["Status"] = "Deceased"
except Exception:
pass
# 5. Map Gerbils to Litters
for g in resolved_gerbils:
old_lid = g["_old_scoped_litter_id"]
if old_lid in litter_id_map:
g["LitterId"] = litter_id_map[old_lid]
else:
g["LitterId"] = None
l_guid = litter_id_map.get(old_lid)
g["LitterId"] = l_guid
if l_guid and l_guid in litter_by_scoped_id:
l = litter_by_scoped_id[l_guid]
l_date = l.get("Date")
if l_date:
if not g.get("DateOfBirth"):
g["DateOfBirth"] = l_date
g["_birth_date"] = l_date
g["_eff_dob"] = l_date
# Apply age-based death threshold (6.0 years) to all resolved gerbils (including newly backfilled ones)
if g.get("Status") != "Deceased" and g.get("Status") != "GivenAway":
if not g.get("DateOfDeath") and not g.get("ReceiverContactId"):
dob_str = g.get("DateOfBirth")
if dob_str:
try:
dt_dob = datetime.strptime(dob_str, "%Y-%m-%d")
age_years = (dt_now - dt_dob).days / 365.25
if age_years >= 6.0:
g["Status"] = "Deceased"
except Exception:
pass
# Clean helper fields
del g["_old_scoped_litter_id"]
del g["_eff_dob"]
@@ -1575,7 +1684,7 @@ def main():
final_c = next((rg for rg in resolved_gerbils if rg["Id"] == final_id), None)
if final_c and final_c["Gender"] in ["male", "unknown"]:
# Ensure parent is born before litter if birth date is known
if l["Date"] != "0001-01-01" and final_c["DateOfBirth"]:
if l["Date"] and final_c["DateOfBirth"]:
if final_c["DateOfBirth"] < l["Date"]:
valid_candidates.append(final_c)
else:
@@ -1600,7 +1709,7 @@ def main():
continue
final_c = next((rg for rg in resolved_gerbils if rg["Id"] == final_id), None)
if final_c and final_c["Gender"] in ["female", "unknown"]:
if l["Date"] != "0001-01-01" and final_c["DateOfBirth"]:
if l["Date"] and final_c["DateOfBirth"]:
if final_c["DateOfBirth"] < l["Date"]:
valid_candidates.append(final_c)
else: