feat(import): resolve color variety ID mappings mismatch, support unnamed parents in virtual litters, and implement SaleContract updates

tools/import/extract.py

@@ -323,9 +323,10 @@ def _reconstruct_parents(animals):
             father = above[-1] if above else None
             mother = below[0] if below else None
             for parent, role in ((father, "father"), (mother, "mother")):
-                if parent and parent["name"]:
+                if parent:
+                    p_name = parent["name"] or "unbekannt"
                     a["parentRefs"].append({
-                        "name": parent["name"],
+                        "name": p_name,
                         "dob": parent["dob"],
                         "roleGuess": role,
                         "method": "chart-position",
@@ -348,7 +349,7 @@ def _attach_photos(z, sheets, animals, fname):
         if not cands:
             # fall back to nearest animal by row across all gens
             cands = animals
-        target = min(cands, key=lambda a: abs(a["_row"] - row)) if cands else None
+        target = min(cands, key=lambda a: abs((a["_row"] - row) - 10)) if cands else None
         if not target:
             continue
         ext = os.path.splitext(media)[1] or ".img"

tools/import/merge_and_resolve.py

@@ -231,6 +231,34 @@ def get_normalized_contact_name(name):
         
     return name, True
 
+def get_normalized_gerbil_name(name):
+    if not name:
+        return ""
+    n = name.strip()
+    norm_key = "".join(c for c in n.lower() if c.isalnum())
+    
+    gerbil_norm_map = {
+        "samgenshellyvdbuntenfellnasen": "Sammy gen. Shelly von den bunten Fellnasen",
+        "sammygenshellyvdbuntenfellnasen": "Sammy gen. Shelly von den bunten Fellnasen",
+        "schmidt": "Schmidti",
+        "sheila": "Sheila of Ulmer Strolche",
+        "shinichi": "Shinichi von PZ Mücke",
+        "silenosgenadonis": "Silenos gen. Adonis von den Kleinen Chaoten",
+        "silenosgenadonisvdkleinenchaoten": "Silenos gen. Adonis von den Kleinen Chaoten",
+        "silver": "Silver von den kleinen Chaoten",
+        "snoops": "Snoopsi",
+        "sokrates": "Sokrates von den Kleinen Chaoten",
+        "splash": "Slash",
+        "teiko": "Teiko von den kleinen Chaoten",
+        "trixy": "Trixxy von den Kleinen Chaoten",
+        "unique": "Unique of Wild Dreams",
+    }
+    
+    if norm_key in gerbil_norm_map:
+        return gerbil_norm_map[norm_key]
+        
+    return n
+
 def get_call_name(name):
     if not name:
         return ""
@@ -485,15 +513,42 @@ def main():
     print("Loading color variety seeds...")
     variety_map = {}
     variety_genotypes = {}
+    
+    # Load from C# ApplicationContext.cs catalog for stable database GUIDs (index + 1)
+    here = os.path.dirname(os.path.abspath(__file__))
+    app_context_path = os.path.abspath(os.path.join(here, "../../GerbilManagerWebAPI/ApplicationContext.cs"))
+    cs_name_to_id = {}
+    if os.path.exists(app_context_path):
+        with open(app_context_path, 'r', encoding='utf-8') as f:
+            content = f.read()
+        catalog_match = re.search(r'catalog\s*=\s*\{(.*?)\};', content, re.DOTALL)
+        if catalog_match:
+            block = catalog_match.group(1)
+            entries = re.findall(r'\(\s*"([^"]+)"\s*,\s*"([^"]+)"\s*,\s*(\d+)\s*\)', block)
+            for idx, (name, genotype, sort_order) in enumerate(entries):
+                variety_id = f"00000000-0000-0000-0000-{idx + 1:012d}"
+                cs_name_to_id[name.strip().lower()] = variety_id
+                variety_map[name.strip().lower()] = variety_id
+                variety_genotypes[variety_id] = genotype.strip()
+    else:
+        print(f"Warning: ApplicationContext.cs not found at {app_context_path}")
+
     if os.path.exists(SEEDS_PATH):
         with open(SEEDS_PATH, "r", encoding="utf-8") as f:
             seeds = json.load(f)
         for v in seeds:
-            variety_id = f"00000000-0000-0000-0000-{v['sortOrder'] + 1:012d}"
-            variety_map[v["name"].strip().lower()] = variety_id
+            name_lower = v["name"].strip().lower()
+            variety_id = cs_name_to_id.get(name_lower)
+            if not variety_id:
+                variety_id = f"00000000-0000-0000-0000-{v['sortOrder'] + 1:012d}"
+                variety_map[name_lower] = variety_id
+            
+            # Map English name if present
             if "english" in v and v["english"]:
                 variety_map[v["english"].strip().lower()] = variety_id
-            variety_genotypes[variety_id] = v.get("canonicalGenotype")
+                
+            if variety_id not in variety_genotypes:
+                variety_genotypes[variety_id] = v.get("canonicalGenotype")
     else:
         print(f"Warning: Seeds path not found at {SEEDS_PATH}")
 
@@ -694,8 +749,8 @@ def main():
     for dl in docx_litters:
         l_name = dl["litterId"]
         dob_val = parse_date(dl["dob"])
-        f_name = dl["fatherName"]
-        m_name = dl["motherName"]
+        f_name = get_normalized_gerbil_name(dl["fatherName"])
+        m_name = get_normalized_gerbil_name(dl["motherName"])
         ws_code = dl["wsCode"]
         note_val = dl.get("note")
         
@@ -719,7 +774,7 @@ def main():
         raw_litters.append({
             "Id": l_scoped_id,
             "Name": l_name,
-            "Date": dob_val or "0001-01-01",
+            "Date": dob_val,
             "TotalBorn": total_born,
             "DeathsWithin8Weeks": deaths_8w,
             "FatherId": generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"), # placeholder
@@ -784,8 +839,8 @@ def main():
     # Pre-index Wurfchronik litters from markdown
     md_litters_idx = {}
     for rl in raw_litters:
-        f_name = rl.get("FatherName") or rl.get("fatherName") or rl.get("ParentMaleName") or rl.get("parentMaleName")
-        m_name = rl.get("MotherName") or rl.get("motherName") or rl.get("ParentFemaleName") or rl.get("parentFemaleName")
+        f_name = get_normalized_gerbil_name(rl.get("FatherName") or rl.get("fatherName") or rl.get("ParentMaleName") or rl.get("parentMaleName"))
+        m_name = get_normalized_gerbil_name(rl.get("MotherName") or rl.get("motherName") or rl.get("ParentFemaleName") or rl.get("parentFemaleName"))
         ldate = parse_date(rl.get("Date") or rl.get("date") or rl.get("DateOfBirth") or rl.get("dateOfBirth"))
         if f_name and m_name and ldate:
             key = (normalize_name(f_name), normalize_name(m_name), ldate)
@@ -800,8 +855,8 @@ def main():
         
         a["_mapped_litter_scoped_id"] = None
         if father_ref and mother_ref:
-            f_name = father_ref.get("name")
-            m_name = mother_ref.get("name")
+            f_name = get_normalized_gerbil_name(father_ref.get("name"))
+            m_name = get_normalized_gerbil_name(mother_ref.get("name"))
             dob_val = parse_date(a.get("dob"))
             
             mapped_litter = None
@@ -827,14 +882,16 @@ def main():
                     m_dob = parse_date(mother_ref.get("dob"))
                     
                     for p_cand in stammbaum_only_animals:
-                        cand_call_norm = normalize_name(get_call_name(p_cand["name"]))
-                        if cand_call_norm == normalize_name(f_name) or normalize_name(p_cand["name"]) == normalize_name(f_name):
+                        cand_call_norm = normalize_name(get_call_name(p_cand["name"])) or "unbekannt"
+                        cand_name_norm = normalize_name(p_cand["name"]) or "unbekannt"
+                        if cand_call_norm == normalize_name(f_name) or cand_name_norm == normalize_name(f_name):
                             if not f_dob or parse_date(p_cand.get("dob")) == f_dob:
                                 f_scoped_id = generate_guid(f"stammbaum-animal-{p_cand['id']}")
                                 break
                     for p_cand in stammbaum_only_animals:
-                        cand_call_norm = normalize_name(get_call_name(p_cand["name"]))
-                        if cand_call_norm == normalize_name(m_name) or normalize_name(p_cand["name"]) == normalize_name(m_name):
+                        cand_call_norm = normalize_name(get_call_name(p_cand["name"])) or "unbekannt"
+                        cand_name_norm = normalize_name(p_cand["name"]) or "unbekannt"
+                        if cand_call_norm == normalize_name(m_name) or cand_name_norm == normalize_name(m_name):
                             if not m_dob or parse_date(p_cand.get("dob")) == m_dob:
                                 m_scoped_id = generate_guid(f"stammbaum-animal-{p_cand['id']}")
                                 break
@@ -842,7 +899,7 @@ def main():
                     raw_litters.append({
                         "Id": l_scoped_id,
                         "Name": f"Wurf von {f_name} + {m_name}",
-                        "Date": dob_val or "0001-01-01",
+                        "Date": dob_val,
                         "TotalBorn": None,
                         "DeathsWithin8Weeks": None,
                         "FatherId": f_scoped_id or generate_guid(f"stammbaum-animal-{normalize_name(f_name)}"),
@@ -924,8 +981,6 @@ def main():
             name_val = "Wurf"
 
         dob_val = parse_date(rl.get("Date") or rl.get("date") or rl.get("DateOfBirth") or rl.get("dateOfBirth"))
-        if not dob_val:
-            dob_val = "0001-01-01"
 
         new_guid = rl["_scoped_id"]
         if not new_guid:
@@ -986,7 +1041,7 @@ def main():
     def get_litter_date(l_id):
         if l_id in litter_by_scoped_id:
             d = litter_by_scoped_id[l_id]["Date"]
-            if d and d != "0001-01-01":
+            if d:
                 return d
         return None
 
@@ -994,7 +1049,7 @@ def main():
     all_processed_gerbils = []
     for rg in raw_gerbils:
         filename = rg.get("_filename")
-        name_val = rg.get("Name") or rg.get("name") or rg.get("callName")
+        name_val = get_normalized_gerbil_name(rg.get("Name") or rg.get("name") or rg.get("callName"))
         if not name_val:
             name_val = "Unbekannt"
 
@@ -1142,7 +1197,7 @@ def main():
     # Map and append stammbaum animals to all_processed_gerbils
     for a in stammbaum_only_animals:
         a_id = a["id"]
-        name_val = a["name"]
+        name_val = get_normalized_gerbil_name(a["name"])
         
         gender_val = str(a.get("gender") or "").lower().strip()
         if gender_val in ["m", "male"]:
@@ -1166,7 +1221,7 @@ def main():
                     dt_dob = datetime.strptime(dob_val, "%Y-%m-%d")
                     dt_now = datetime.now()
                     age_years = (dt_now - dt_dob).days / 365.25
-                    if age_years >= 7.0:
+                    if age_years >= 6.0:
                         status = "Deceased"
                 except Exception:
                     pass
@@ -1249,7 +1304,7 @@ def main():
 
     # Map and append docx animals to all_processed_gerbils
     for idx, da in enumerate(docx_animals):
-        name_val = da["name"]
+        name_val = get_normalized_gerbil_name(da["name"])
         gender = da["gender"]
         
         dob_val = parse_date(da.get("litterDob"))
@@ -1335,7 +1390,7 @@ def main():
 
     for l in resolved_litters:
         ld = l["Date"]
-        if ld and ld != "0001-01-01":
+        if ld:
             for pid in [l["FatherId"], l["MotherId"]]:
                 if pid:
                     parent_litter_dates.setdefault(pid, []).append(ld)
@@ -1459,6 +1514,8 @@ def main():
                     if ph not in merged_photos:
                         merged_photos.append(ph)
 
+                if not best_g.get("_old_scoped_litter_id") and g.get("_old_scoped_litter_id"):
+                    best_g["_old_scoped_litter_id"] = g["_old_scoped_litter_id"]
                 if not best_g["LitterId"] and g["LitterId"]:
                     best_g["LitterId"] = g["LitterId"]
                 if not best_g["DateOfBirth"] and g["DateOfBirth"]:
@@ -1499,6 +1556,22 @@ def main():
                      if not any(kw in g["Notes"].lower() for kw in ["parent listed", "mutter von", "vater von", "dam of", "sire of"]):
                          merged_notes.append(g["Notes"])
 
+            # Reconcile fields based on number of source files supporting them
+            for field in ["DateOfBirth", "DateOfDeath", "Gender", "Genotype", "ColorVarietyId"]:
+                votes = {}
+                for g in sub:
+                    val = g.get(field)
+                    if val and val != "unknown":
+                        # count source files
+                        sources_count = len(str(g.get("ImportSource") or "").split(","))
+                        votes[val] = votes.get(val, 0) + sources_count
+                if votes:
+                    best_val = max(votes, key=votes.get)
+                    best_g[field] = best_val
+                    # Keep helper fields in sync if we changed DateOfBirth
+                    if field == "DateOfBirth":
+                        best_g["_birth_date"] = best_val
+                        best_g["_eff_dob"] = best_val or "2010-01-01"
 
             if merged_notes:
                 best_g["Notes"] = " | ".join(merged_notes)
@@ -1513,13 +1586,49 @@ def main():
 
     print(f"Deduplicated to {len(resolved_gerbils)} unique gerbil records.")
 
+    # Apply age-based death threshold (6.0 years) to all resolved gerbils
+    dt_now = datetime.now()
+    for g in resolved_gerbils:
+        if g.get("Status") != "Deceased" and g.get("Status") != "GivenAway":
+            if not g.get("DateOfDeath") and not g.get("ReceiverContactId"):
+                dob_str = g.get("DateOfBirth")
+                if dob_str:
+                    try:
+                        dt_dob = datetime.strptime(dob_str, "%Y-%m-%d")
+                        age_years = (dt_now - dt_dob).days / 365.25
+                        if age_years >= 6.0:
+                            g["Status"] = "Deceased"
+                    except Exception:
+                        pass
+
     # 5. Map Gerbils to Litters
     for g in resolved_gerbils:
         old_lid = g["_old_scoped_litter_id"]
-        if old_lid in litter_id_map:
-            g["LitterId"] = litter_id_map[old_lid]
-        else:
-            g["LitterId"] = None
+        l_guid = litter_id_map.get(old_lid)
+        g["LitterId"] = l_guid
+        
+        if l_guid and l_guid in litter_by_scoped_id:
+            l = litter_by_scoped_id[l_guid]
+            l_date = l.get("Date")
+            if l_date:
+                if not g.get("DateOfBirth"):
+                    g["DateOfBirth"] = l_date
+                    g["_birth_date"] = l_date
+                    g["_eff_dob"] = l_date
+
+        # Apply age-based death threshold (6.0 years) to all resolved gerbils (including newly backfilled ones)
+        if g.get("Status") != "Deceased" and g.get("Status") != "GivenAway":
+            if not g.get("DateOfDeath") and not g.get("ReceiverContactId"):
+                dob_str = g.get("DateOfBirth")
+                if dob_str:
+                    try:
+                        dt_dob = datetime.strptime(dob_str, "%Y-%m-%d")
+                        age_years = (dt_now - dt_dob).days / 365.25
+                        if age_years >= 6.0:
+                            g["Status"] = "Deceased"
+                    except Exception:
+                        pass
+
         # Clean helper fields
         del g["_old_scoped_litter_id"]
         del g["_eff_dob"]
@@ -1575,7 +1684,7 @@ def main():
                 final_c = next((rg for rg in resolved_gerbils if rg["Id"] == final_id), None)
                 if final_c and final_c["Gender"] in ["male", "unknown"]:
                     # Ensure parent is born before litter if birth date is known
-                    if l["Date"] != "0001-01-01" and final_c["DateOfBirth"]:
+                    if l["Date"] and final_c["DateOfBirth"]:
                         if final_c["DateOfBirth"] < l["Date"]:
                             valid_candidates.append(final_c)
                     else:
@@ -1600,7 +1709,7 @@ def main():
                     continue
                 final_c = next((rg for rg in resolved_gerbils if rg["Id"] == final_id), None)
                 if final_c and final_c["Gender"] in ["female", "unknown"]:
-                    if l["Date"] != "0001-01-01" and final_c["DateOfBirth"]:
+                    if l["Date"] and final_c["DateOfBirth"]:
                         if final_c["DateOfBirth"] < l["Date"]:
                             valid_candidates.append(final_c)
                     else:

tools/import/output/review-report.md

@@ -10,7 +10,7 @@ _Automatisch erzeugt von `tools/import/extract.py` — **noch nichts in die Date
   - in mehreren Dateien gefunden (Dubletten zusammengeführt): 460
 - Konflikte zur Klärung: **2**
 - Mehrdeutige / unvollständige Einträge (ohne Name+Datum): **342**
-- Fotos zugeordnet: **417**
+- Fotos zugeordnet: **418**
 - Würfe aus der Wurfchronik: **752**
   - Tiere mit Wurf verknüpft: **270** (davon über Geburtsdatum **und** Eltern: 167, nur über Geburtsdatum: 103; mehrdeutig: 17)
   - Würfe mit Datenqualitäts-Hinweisen: 113 (+ 138 Zeilen mit abweichendem Spaltenschema)

tools/import/resolve_import.py

@@ -71,15 +71,38 @@ def main():
 
     # Load color variety seeds
     print("Loading color variety seeds...")
+    variety_map = {}
+    
+    # Load from C# ApplicationContext.cs catalog for stable database GUIDs (index + 1)
+    app_context_path = os.path.abspath(os.path.join(HERE, "..", "..", "GerbilManagerWebAPI", "ApplicationContext.cs"))
+    cs_name_to_id = {}
+    if os.path.exists(app_context_path):
+        with open(app_context_path, 'r', encoding='utf-8') as f:
+            content = f.read()
+        catalog_match = re.search(r'catalog\s*=\s*\{(.*?)\};', content, re.DOTALL)
+        if catalog_match:
+            block = catalog_match.group(1)
+            entries = re.findall(r'\(\s*"([^"]+)"\s*,\s*"([^"]+)"\s*,\s*(\d+)\s*\)', block)
+            for idx, (name, genotype, sort_order) in enumerate(entries):
+                variety_id = f"00000000-0000-0000-0000-{idx + 1:012d}"
+                cs_name_to_id[name.strip().lower()] = variety_id
+                variety_map[name.strip().lower()] = variety_id
+    else:
+        print(f"Warning: ApplicationContext.cs not found at {app_context_path}")
+
     with open(SEEDS_PATH, "r", encoding="utf-8") as f:
         seeds = json.load(f)
 
-    # Map variety name -> Guid
-    variety_map = {}
     for v in seeds:
-        # UUID is based on SortOrder + 1
-        variety_id = f"00000000-0000-0000-0000-{v['sortOrder'] + 1:012d}"
-        variety_map[v["name"].strip().lower()] = variety_id
+        name_lower = v["name"].strip().lower()
+        variety_id = cs_name_to_id.get(name_lower)
+        if not variety_id:
+            variety_id = f"00000000-0000-0000-0000-{v['sortOrder'] + 1:012d}"
+            variety_map[name_lower] = variety_id
+        
+        # Map English name if present
+        if "english" in v and v["english"]:
+            variety_map[v["english"].strip().lower()] = variety_id
 
     # 1. Establish stable Guid maps
     animal_guid_map = {a["id"]: generate_guid(f"animal-{a['id']}") for a in animals}
@@ -297,12 +320,12 @@ def main():
         elif not is_resident:
             status = "GivenAway"
         else:
-            # Age presumed deceased (>7 years)
+            # Age presumed deceased (>6 years)
             if dob:
                 dt_dob = datetime.strptime(dob, "%Y-%m-%d")
                 dt_now = datetime.now()
                 age_years = (dt_now - dt_dob).days / 365.25
-                if age_years >= 7.0:
+                if age_years >= 6.0:
                     status = "Deceased"
                 else:
                     status = "Breeding" # default to breeding for resident stock

tools/import/test_extract.py

@@ -337,6 +337,65 @@ check("KC-matcher: norm_zucht regression — 'von den Kleinen Chaoten'",
 check("KC-matcher: norm_zucht('v.d. Kleinen Chaoten') == 'kleinechaote' (was broken before fix)",
       e.norm_zucht("v.d. Kleinen Chaoten") == "kleinechaote")
 
+# --- Stammbaum von Danako validation ---
+danako_path = r"C:\Users\gulum\dev\Wurfchronik_Bilder\Stammbaum von Danako.xlsx"
+if not os.path.exists(danako_path):
+    danako_path = r"C:\Users\gulum\dev\Sttammbäume\Stammbaum von Danako.xlsx"
+
+if os.path.exists(danako_path):
+    print(f"\nFound Danako stammbaum at {danako_path}, running integration validation...")
+    danako_animals = e.extract_stammbaum(danako_path)
+    danako_by_name = {a["name"]: a for a in danako_animals}
+    
+    check("Danako present in Danako sheet", "Danako" in danako_by_name)
+    if "Danako" in danako_by_name:
+        d = danako_by_name["Danako"]
+        check("Danako DOB is 22.08.2018", d["dob"] == "22.08.2018")
+        check("Danako photo matches image7.png", d["photos"] == ["photos/danako-22082018/image7.png"])
+        
+    check("Osamu present in Danako sheet", "Osamu" in danako_by_name)
+    if "Osamu" in danako_by_name:
+        o = danako_by_name["Osamu"]
+        check("Osamu DOB is 10.12.2015", o["dob"] == "10.12.2015")
+        check("Osamu photo matches image4.jpeg", o["photos"] == ["photos/osamu-10122015/image4.jpeg"])
+        
+        # Check parentRefs of Osamu in Danako sheet
+        o_parents = o.get("parentRefs", [])
+        o_father = next((p for p in o_parents if p.get("roleGuess") == "father"), None)
+        o_mother = next((p for p in o_parents if p.get("roleGuess") == "mother"), None)
+        check("Osamu father is Porter", o_father is not None and o_father["name"] == "Porter")
+        check("Osamu mother is Yuka", o_mother is not None and o_mother["name"] == "Yuka")
+        if o_father:
+            check("Osamu father DOB is 23.05.2015", o_father["dob"] == "23.05.2015")
+        if o_mother:
+            check("Osamu mother DOB is 12.07.2015", o_mother["dob"] == "12.07.2015")
+
+    check("Porter present in Danako sheet", "Porter" in danako_by_name)
+    if "Porter" in danako_by_name:
+        p = danako_by_name["Porter"]
+        check("Porter DOB is 23.05.2015", p["dob"] == "23.05.2015")
+        check("Porter photo matches image6.jpeg", p["photos"] == ["photos/porter-23052015/image6.jpeg"])
+
+    check("Yuka present in Danako sheet", "Yuka" in danako_by_name)
+    if "Yuka" in danako_by_name:
+        y = danako_by_name["Yuka"]
+        check("Yuka DOB is 12.07.2015", y["dob"] == "12.07.2015")
+        check("Yuka photo matches image5.jpeg", y["photos"] == ["photos/yuka-12072015/image5.jpeg"])
+
+    check("Eddward present in Danako sheet", "Eddward" in danako_by_name)
+    if "Eddward" in danako_by_name:
+        ed = danako_by_name["Eddward"]
+        check("Eddward DOB is 18.11.2015", ed["dob"] == "18.11.2015")
+        check("Eddward photo matches image1.jpeg", ed["photos"] == ["photos/eddward-18112015/image1.jpeg"])
+
+    check("Harumi present in Danako sheet", "Harumi" in danako_by_name)
+    if "Harumi" in danako_by_name:
+        h = danako_by_name["Harumi"]
+        check("Harumi DOB is 21.02.2015", h["dob"] == "21.02.2015")
+        check("Harumi photo matches image2.jpeg", h["photos"] == ["photos/harumi-21022015/image2.jpeg"])
+else:
+    print("\nWarning: Danako stammbaum file not found, skipping integration checks.")
+
 if failed:
     print(f"\n{failed} test(s) FAILED")
     sys.exit(1)