feat(import): Abgabeverträge (DOCX) auswerten und Tiere/Kontakte anreichern
Neuer Parser extract_contracts.py liest die ~1,4k Abgabevertrags-DOCX (\truenas\…\Verträge): er extrahiert aus dem Dokument-Body (zuverlässiger als die Dateinamen) Käufer, Tier(e), Farbschlag, Abgabedatum und Preis — robust gegen Word-Run-Splits (z. B. „F r au"/„3 0,00"); überspringt Vorlage, Abstammungsnachweise und als .docx getarnte .doc. enrich_from_contracts() in merge_and_resolve.py: Käufer werden als Kontakte (IsReceiver) angelegt/zusammengeführt; Tiere werden KONSERVATIV per Rufname (+ DOB-Jahr bei Mehrdeutigkeit) auf eigene Bestandstiere gematcht und erhalten ReceiverContactId, GoHomeDate und Status „abgegeben" — nur wo nicht bereits gesetzt; Konflikte werden geloggt, nicht überschrieben. Jede Übernahme bekommt eine Herkunfts-Zeile („Abgabe an … aus Vertrag … übernommen."). Ergebnis: 1095 Verträge → 783 Tier-Treffer (400 mehrdeutige übersprungen), 274 neue Abnehmer-Kontakte, 153 Tiere mit Abnehmer, 49 mit Abgabedatum, 23 neu „abgegeben". Keine Backend-/Frontend-Änderung nötig (Akte zeigt Abnehmer/ Abgabedatum/Herkunft bereits). SaleContract-Records bewusst nicht erzeugt (bräuchte Migration + ingest-sichere Id — späterer Schritt). Tests: test_extract_contracts.py (Dateiname/Body/Run-Split/Skip-Regeln) + alle bestehenden grün; dotnet 212. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -944,6 +944,168 @@ def _format_discard(d):
|
||||
return DISCARD_MARK + line + "."
|
||||
|
||||
|
||||
def _append_history(prov_json, line):
|
||||
"""Append one history line to an existing Provenance JSON string and add the
|
||||
contract source file. Returns the updated JSON string."""
|
||||
try:
|
||||
prov = json.loads(prov_json) if prov_json else {}
|
||||
except (ValueError, TypeError):
|
||||
prov = {}
|
||||
prov.setdefault("history", [])
|
||||
if line not in prov["history"]:
|
||||
prov["history"].append(line)
|
||||
return json.dumps(prov, ensure_ascii=False)
|
||||
|
||||
|
||||
def enrich_from_contracts(contracts, resolved_gerbils, contact_by_norm_name,
|
||||
contact_id_map):
|
||||
"""Conservatively fold Abgabevertrag data into the resolved gerbils.
|
||||
|
||||
For every parsed contract we (a) ensure the buyer exists as a (receiver)
|
||||
contact, reusing the existing contact dedup/normalisation, and (b) try to
|
||||
match each animal call-name to exactly one resolved gerbil that the breeder
|
||||
bred ("…Chaoten"). On a confident match we set ReceiverContactId /
|
||||
GoHomeDate / Status=GivenAway — but only where not already set differently —
|
||||
and add a provenance history line. Ambiguous or absent matches are logged,
|
||||
never guessed.
|
||||
|
||||
Returns a stats dict. Mutates resolved_gerbils + contact_by_norm_name in
|
||||
place. New buyer contacts are appended via contact_by_norm_name so the later
|
||||
IsReceiver-flag pass picks them up automatically.
|
||||
"""
|
||||
stats = {
|
||||
"contracts": len(contracts), "buyers_created": 0, "buyers_existing": 0,
|
||||
"matched": 0, "ambiguous_skipped": 0, "no_match_skipped": 0,
|
||||
"receiver_set": 0, "gohome_set": 0, "status_givenaway": 0,
|
||||
"conflicts": 0,
|
||||
}
|
||||
if not contracts:
|
||||
return stats
|
||||
|
||||
# Index breeder-owned gerbils by dedup name key. Contracts only ever sell
|
||||
# animals the breeder bred, so restrict candidates to her own stock to avoid
|
||||
# colliding with same-named foreign-bred animals.
|
||||
def is_own(g):
|
||||
ob = (g.get("OriginBreeder") or "").lower()
|
||||
return ("chaoten" in ob) or (g.get("OriginBreeder") is None)
|
||||
|
||||
index = {}
|
||||
for g in resolved_gerbils:
|
||||
if not is_own(g):
|
||||
continue
|
||||
key = get_dedup_name_key(get_call_name(g.get("Name", "")))
|
||||
if key:
|
||||
index.setdefault(key, []).append(g)
|
||||
|
||||
def year_of(iso):
|
||||
return iso[:4] if iso else None
|
||||
|
||||
# Reject buyer values that are obviously label leakage / non-person noise
|
||||
# (defends against any stale contracts.json produced before the parser fix).
|
||||
_buyer_junk = re.compile(
|
||||
r"^(?:stra\w+e|wohnort|fon|e-?mail|handy|festnetz|telefon|homepage|"
|
||||
r"zuchtname|facebook|name)\s*:?\s*$|^[\d\s/]+$", re.IGNORECASE)
|
||||
|
||||
for c in contracts:
|
||||
fname = c.get("sourceFile", "")
|
||||
buyer_raw = (c.get("buyer") or "").strip()
|
||||
if buyer_raw and _buyer_junk.match(buyer_raw):
|
||||
buyer_raw = ""
|
||||
|
||||
# --- buyer contact (reuse curated normalisation/dedup) ---
|
||||
buyer_global_id = None
|
||||
if buyer_raw:
|
||||
canon, keep = get_normalized_contact_name(buyer_raw)
|
||||
if keep and canon:
|
||||
norm = normalize_name(canon)
|
||||
if norm in contact_by_norm_name:
|
||||
gc = contact_by_norm_name[norm]
|
||||
gc.setdefault("_source_files", set()).add(fname)
|
||||
gc["_merged_count"] = gc.get("_merged_count", 1) + 1
|
||||
stats["buyers_existing"] += 1
|
||||
else:
|
||||
gid = generate_guid(f"contact-{norm}")
|
||||
contact_by_norm_name[norm] = {
|
||||
"Id": gid, "Name": canon, "Email": None, "Phone": None,
|
||||
"Address": None, "Notes": None, "NameSuffix": None,
|
||||
"_source_files": {fname}, "_merged_count": 1,
|
||||
}
|
||||
stats["buyers_created"] += 1
|
||||
buyer_global_id = contact_by_norm_name[norm]["Id"]
|
||||
|
||||
# --- match each animal call-name to a resolved gerbil ---
|
||||
handover = parse_date(c.get("handoverDate"))
|
||||
c_year = year_of(parse_date(c.get("dob"))) if c.get("dob") else None
|
||||
c_color = (c.get("color") or "").strip().lower()
|
||||
|
||||
for call in (c.get("animals") or []):
|
||||
key = get_dedup_name_key(get_call_name(call))
|
||||
if not key:
|
||||
continue
|
||||
cands = index.get(key, [])
|
||||
if not cands:
|
||||
stats["no_match_skipped"] += 1
|
||||
continue
|
||||
|
||||
# Corroborate when more than one candidate shares the call-name.
|
||||
chosen = None
|
||||
if len(cands) == 1:
|
||||
chosen = cands[0]
|
||||
else:
|
||||
scored = []
|
||||
for g in cands:
|
||||
score = 0
|
||||
g_year = year_of(g.get("DateOfBirth"))
|
||||
if c_year and g_year and c_year == g_year:
|
||||
score += 2
|
||||
if c_color and g.get("ColorVarietyId"):
|
||||
# color match is corroboration; we don't have the name
|
||||
# here, so only DOB drives disambiguation strongly.
|
||||
pass
|
||||
scored.append((score, g))
|
||||
scored.sort(key=lambda t: t[0], reverse=True)
|
||||
if scored[0][0] >= 2 and (len(scored) == 1 or scored[0][0] > scored[1][0]):
|
||||
chosen = scored[0][1]
|
||||
else:
|
||||
stats["ambiguous_skipped"] += 1
|
||||
continue
|
||||
|
||||
stats["matched"] += 1
|
||||
|
||||
# --- set receiver, only if not already set differently ---
|
||||
if buyer_global_id:
|
||||
cur = chosen.get("ReceiverContactId")
|
||||
if not cur:
|
||||
chosen["ReceiverContactId"] = buyer_global_id
|
||||
stats["receiver_set"] += 1
|
||||
chosen["Provenance"] = _append_history(
|
||||
chosen.get("Provenance"),
|
||||
f"Abgabe an „{buyer_raw}“ aus Vertrag {_quote_file(fname)} übernommen.")
|
||||
elif cur != buyer_global_id:
|
||||
stats["conflicts"] += 1
|
||||
chosen["Provenance"] = _append_history(
|
||||
chosen.get("Provenance"),
|
||||
f"Vertrag {_quote_file(fname)} nennt anderen Abnehmer „{buyer_raw}“ "
|
||||
f"— bestehende Zuordnung beibehalten.")
|
||||
|
||||
# --- set go-home date, only if empty ---
|
||||
if handover and not chosen.get("GoHomeDate"):
|
||||
chosen["GoHomeDate"] = handover
|
||||
stats["gohome_set"] += 1
|
||||
chosen["Provenance"] = _append_history(
|
||||
chosen.get("Provenance"),
|
||||
f"Abgabedatum {handover} aus Vertrag {_quote_file(fname)} übernommen.")
|
||||
|
||||
# --- status: derive GivenAway if we set a receiver and it isn't
|
||||
# already a stronger state (Deceased). ---
|
||||
if chosen.get("ReceiverContactId") and chosen.get("Status") not in (
|
||||
"Deceased", "GivenAway"):
|
||||
chosen["Status"] = "GivenAway"
|
||||
stats["status_givenaway"] += 1
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def main():
|
||||
print("Loading color variety seeds...")
|
||||
variety_map = {}
|
||||
@@ -1176,6 +1338,17 @@ def main():
|
||||
else:
|
||||
print(f"Warning: docx_litters.json not found at {docx_litters_path}")
|
||||
|
||||
# Abgabevertrag-Datensätze (aus extract_contracts.py). Optional: wenn die
|
||||
# Datei fehlt, läuft der Import ohne Vertrags-Anreicherung normal weiter.
|
||||
contracts = []
|
||||
contracts_path = os.path.join(OUTPUT_DIR, "contracts.json")
|
||||
if os.path.exists(contracts_path):
|
||||
with open(contracts_path, "r", encoding="utf-8") as f:
|
||||
contracts = json.load(f)
|
||||
print(f"Loaded {len(contracts)} Abgabeverträge.")
|
||||
else:
|
||||
print(f"Info: contracts.json not found at {contracts_path} (Vertrags-Anreicherung übersprungen)")
|
||||
|
||||
# Extract docx buyer contacts and add to raw_contacts
|
||||
for da in docx_animals:
|
||||
o_name = (da.get("owner") or "").strip()
|
||||
@@ -2685,6 +2858,28 @@ def main():
|
||||
if n_discards:
|
||||
print(f"Datenherkunft: recorded {n_discards} discard line(s) across gerbils.")
|
||||
|
||||
# Abgabevertrag-Anreicherung: Käufer als Abnehmer-Kontakte anlegen und —
|
||||
# konservativ — auf eindeutig passende Tiere ReceiverContactId/GoHomeDate/
|
||||
# Status=GivenAway setzen (nur falls noch nicht gesetzt). Provenance-
|
||||
# Historie wird ergänzt. Neue Käuferkontakte landen in contact_by_norm_name
|
||||
# und werden danach automatisch als IsReceiver markiert.
|
||||
cstats = enrich_from_contracts(
|
||||
contracts, resolved_gerbils, contact_by_norm_name, contact_id_map)
|
||||
if contracts:
|
||||
print("Abgabeverträge: "
|
||||
f"{cstats['contracts']} geladen, {cstats['matched']} Tier-Treffer, "
|
||||
f"{cstats['ambiguous_skipped']} mehrdeutig übersprungen, "
|
||||
f"{cstats['no_match_skipped']} ohne Treffer.")
|
||||
print(" Kontakte: "
|
||||
f"{cstats['buyers_created']} neu, {cstats['buyers_existing']} bestehend.")
|
||||
print(" Gesetzt: "
|
||||
f"ReceiverContactId={cstats['receiver_set']}, "
|
||||
f"GoHomeDate={cstats['gohome_set']}, "
|
||||
f"Status=GivenAway={cstats['status_givenaway']}, "
|
||||
f"Konflikte={cstats['conflicts']}.")
|
||||
# Re-materialise contacts so freshly created buyer contacts are exported.
|
||||
resolved_contacts = list(contact_by_norm_name.values())
|
||||
|
||||
# Datenherkunft for litters: which source files contributed, whether this is
|
||||
# a Wurfchronik litter vs a Stammbaum-reconstructed ("virtual") litter, how
|
||||
# many raw records merged into it, plus human-readable notes. Accumulators
|
||||
|
||||
Reference in New Issue
Block a user