diff --git a/tools/import/README.md b/tools/import/README.md index 537d319..77576dd 100644 --- a/tools/import/README.md +++ b/tools/import/README.md @@ -51,12 +51,25 @@ preserving everything: `genotype.mapped8locus`, `genotype.rawGenotype` (verbatim ```sh cd tools/import -python extract.py # uses the default source paths +python extract.py # xlsx → animals.json / litters.json python extract.py --stammbaeume "" --wurfchronik "" +python extract_docx.py # Wurfchronik-Detail.docx → docx_*.json +python extract_contracts.py # Abgabeverträge (.docx) → contracts.json +python merge_and_resolve.py # → resolved_import.json (DB-ready) ``` -Requires Python 3. **Re-runnable / idempotent** — re-run when more files arrive -(Wurfchronik `Teil2+`, or new charts). +Requires Python 3 (zero third-party deps). **Re-runnable / idempotent** — re-run +when more files arrive (Wurfchronik `Teil2+`, new charts, or new contracts). + +`extract_contracts.py` scans the breeder's sale-contract share +(`\\truenas\…\Verträge`, ~1.4k `.docx`) and emits one record per contract +(buyer, animal call-names, Farbschlag, dates, price, source filename). It skips +the blank template, `Abstammungsnachweis`/`Geburtsurkunde` documents, and any +file that is not a readable `.docx`. `merge_and_resolve.py` then conservatively +folds contracts into the resolved data: buyers become receiver `Contacts`, and +unambiguously matched gerbils get `ReceiverContactId` / `GoHomeDate` / +`Status=GivenAway` (only where not already set), with a provenance history line. +Ambiguous / unmatched animals are counted and skipped, never guessed. ## Output (`tools/import/output/`, git-ignored except the report) @@ -64,6 +77,9 @@ Requires Python 3. **Re-runnable / idempotent** — re-run when more files arriv |---|---| | `animals.json` | deduped animals with genotype, parentRefs, photos, sourceFiles | | `litters.json` | litters from the Wurfchronik | +| `docx_animals.json` / `docx_litters.json` | Wurfchronik-Detail.docx rows | +| `contracts.json` | one record per Abgabevertrag (buyer, animals, dates, price) | +| `resolved_import.json` | merged DB-ready payload consumed by `IngestResolvedService` | | `photos//…` | extracted, anchor-mapped images | | `review-report.md` | **human review deliverable** (committed) | diff --git a/tools/import/extract_contracts.py b/tools/import/extract_contracts.py new file mode 100644 index 0000000..cf204b8 --- /dev/null +++ b/tools/import/extract_contracts.py @@ -0,0 +1,446 @@ +#!/usr/bin/env python3 +"""FEAT-Contracts Stage 1 — Abgabeverträge (.docx) extrahieren. + +Liest die Sammlung der Vermittlungs-/Abgabeverträge ("Zucht der kleinen +Chaoten _ … (Tier.Namen) - _.docx") vom Netzlaufwerk und erzeugt: + output/contracts.json — strukturierte Vertrags-Datensätze + +Reines stdlib-Python (kein pip), nach Vorbild von extract_docx.py: .docx ist +ein ZIP mit word/document.xml; daraus werden die "Label: Wert"-Zeilen des +Vertragskörpers gelesen. Der Dateiname dient als Zusatz-Signal für die +Tier-Rufnamen und den Farbschlag, wenn der Körper sie nicht hergibt. + +Zuverlässig extrahierbar (beobachtete Abdeckung über 250er-Stichprobe): + Name (Tier): 100 % — Body-Label "Name:" + Geschlecht: 100 % — Body-Label "Geschlecht:" + Geburtsdatum: 98 % — Body-Label "Geburtsdatum:" + Abgabedatum: 78 % — Body-Label "Abgabedatum:" + Preis: 78 % — "Gesamtpreis/Schutzgebühr/Kaufpreis" + Käufer (Body): 79 % — Block "Abnehmer/Empfänger" → "Name:" + Farbschlag(Body): 13 % — Body-Label "Farbschlag:" (oft im Dateinamen) + +Übersprungen werden: + * die Vorlage "Vertrags mustter new.docx" + * Abstammungsnachweise / Geburtsurkunden (kein Verkauf, eigener Doc-Typ) + * Dateien, die kein lesbares ZIP sind (alte .doc als .docx getarnt) + +Ausführung: python extract_contracts.py [--dir PFAD] [--limit N] +Idempotent: mehrfaches Ausführen überschreibt output/contracts.json. +""" +import os +import re +import sys +import json +import zipfile +import argparse + +HERE = os.path.dirname(os.path.abspath(__file__)) +DEFAULT_DIR = r"\\truenas\Datengrab\Rennmäuse\Verträge" +OUT = os.path.join(HERE, "output") + +TEMPLATE_MARKERS = ("mustter", "muster new") +# Doc types that are NOT sale contracts (skip): +NON_CONTRACT_MARKERS = ("Abstammungsnachweis", "Geburtsurkunde") + +# --- Body label regexes (work on whitespace-collapsed plain text) ---------- +# Generic "Label: value up to next known label or end". +_KNOWN_LABELS = ( + r"Name|Geburtsdatum|Geschlecht|Farbschlag|Farbe|Abgabedatum|Abgabe\s*am|" + r"Gesamtpreis|Schutzgeb\w+|Kaufpreis|Preis|Zuchtname|Zuchtbuchnummer|" + r"Stra\w+e|Wohnort|Fon|Festnetz|Handy|Handynummer|Telefon|E-?Mail|Homepage|" + r"Facebook|Mutter|Vater|Linie|Wurf|Bemerkung|Z\w+chter\w*|Abnehmer|Empf\w+nger" +) + + +def _norm_date(d: str) -> str: + """German DD.MM.YY[YY] → ISO YYYY-MM-DD (or '' if unparseable).""" + if not d: + return "" + m = re.search(r"(\d{1,2})\.(\d{1,2})\.(\d{2,4})", d) + if not m: + return "" + day, mon, yr = m.group(1), m.group(2), m.group(3) + if len(yr) == 2: + yr = "20" + yr + try: + di, mi, yi = int(day), int(mon), int(yr) + if not (1 <= di <= 31 and 1 <= mi <= 12 and 1900 <= yi <= 2100): + return "" + except ValueError: + return "" + return f"{yi:04d}-{mi:02d}-{di:02d}" + + +def _unescape(t: str) -> str: + for a, b in (("&", "&"), ("<", "<"), (">", ">"), + ("'", "'"), (""", '"'), (" ", " ")): + t = t.replace(a, b) + return t + + +def _para_text(p_xml: str) -> str: + """Plain text of one . + + IMPORTANT: Word frequently splits a single word across multiple / + runs (formatting/spell-check artefacts). Run boundaries are NOT word + boundaries, so we concatenate contents directly (no separator) and + only turn explicit tabs/breaks into spaces. This avoids mangling + "Frau"→"F r au" or "30,00"→"3 0,00". + """ + p_xml = re.sub(r"]*/?>", " ", p_xml) + p_xml = re.sub(r"]*/?>", " ", p_xml) + texts = re.findall(r"]*>(.*?)", p_xml, re.DOTALL) + t = _unescape("".join(texts)) + return re.sub(r"\s+", " ", t).strip() + + +def _full_text(docx_path: str) -> str: + """Return whitespace-collapsed plain text of the document body. + + Paragraphs (and table cells, also wrapped in ) are joined by a single + space so adjacent labels stay separable. + """ + with zipfile.ZipFile(docx_path) as z: + xml = z.read("word/document.xml").decode("utf-8", errors="replace") + paras = [_para_text(p) for p in re.findall(r"].*?", xml, re.DOTALL)] + return re.sub(r"\s+", " ", " ".join(p for p in paras if p)).strip() + + +def _paragraphs(docx_path: str) -> list[str]: + """Return per-paragraph plain text (preserves the Label/value line breaks).""" + with zipfile.ZipFile(docx_path) as z: + xml = z.read("word/document.xml").decode("utf-8", errors="replace") + out = [] + for p in re.findall(r"].*?", xml, re.DOTALL): + t = _para_text(p) + if t: + out.append(t) + return out + + +def _label_value(text: str, label_rx: str) -> str: + """Find 'Label: value' in collapsed text; stop at the next known label.""" + rx = re.compile( + rf"(?:{label_rx})\s*:\s*(.+?)(?=\s*(?:{_KNOWN_LABELS})\s*:|$)", + re.IGNORECASE, + ) + m = rx.search(text) + return m.group(1).strip() if m else "" + + +def _clean_money(raw: str) -> str: + """'27,50 Euro (Überweisung)' → '27,50'. Returns '' if no number.""" + m = re.search(r"(\d+(?:[.,]\d{1,2})?)", raw) + if not m: + return "" + return m.group(1).replace(".", ",") + + +def parse_filename(fname: str) -> dict: + """Best-effort structured pieces from the contract filename. + + Returns dict with keys: color, animals (list of call-names), buyer + (may be empty). Robust to the many messy variants observed. + """ + base = re.sub(r"\.docx$", "", fname, flags=re.IGNORECASE) + # Strip the leading cattery prefix and the leading/trailing underscores. + base = re.sub(r"^\s*(?:Zucht der kleinen Chaoten|Clan[^_]*Chaoten)\s*", + "", base, flags=re.IGNORECASE) + base = base.strip().strip("_").strip() + + color = "" + animals: list[str] = [] + buyer = "" + + # Animals are inside the (…) group, dot-separated call-names. + pm = re.search(r"\(([^)]*)\)", base) + if pm: + inner = pm.group(1).strip() + # split on dot (call-name separator) but keep multi-word names + animals = [a.strip() for a in inner.split(".") if a.strip()] + # color = text before "(" + color = base[:pm.start()].strip(" -–_") + # buyer = text after ")" + after = base[pm.end():].strip() + bm = re.match(r"\s*[-–]\s*(.+)", after) + if bm: + buyer = bm.group(1).strip().strip("_").strip() + else: + # No parens. Two shapes: + # " - " (token is animal-name or color) + # "" (just a name) + dm = re.split(r"\s*[-–]\s*", base, maxsplit=1) + if len(dm) == 2 and dm[1].strip(): + color = "" # ambiguous; treat the left token as an animal name + animals = [dm[0].strip().strip("_")] if dm[0].strip() else [] + buyer = dm[1].strip().strip("_").strip() + else: + tok = base.strip().strip("_").strip() + # comma list like "Speedy,Agouti" → first is the name + if tok: + animals = [tok.split(",")[0].strip()] + + # Clean buyer: drop trailing numbering "2", file-version noise + buyer = re.sub(r"\s*\(?\d+\)?$", "", buyer).strip() if buyer else "" + return {"color": color, "animals": animals, "buyer": buyer} + + +# Seller is always the breeder Drazena Rimac (the cattery owner); never a buyer. +_SELLER_RX = re.compile(r"^(?:(?:Herr|Frau|Familie)\s+)?Drazena\b", re.IGNORECASE) +_ADDR_STOP = r"Stra\w+e|Wohnort|Fon|E-?Mail|Handy|Festnetz|Telefon|Homepage|Zuchtname|Facebook" + + +_BUYER_REJECT_RX = re.compile( + r"^(?:Stra\w+e|Wohnort|Fon|E-?Mail|Handy|Festnetz|Telefon|Homepage|" + r"Zuchtname|Facebook|Name)\s*:?\s*$|^[\d\s/]+$", + re.IGNORECASE, +) + + +def _clean_person(name: str) -> str: + name = re.sub(r"\s+", " ", name).strip().strip(",").strip() + # Reject label leakage / non-person artefacts (e.g. "Straße:", phone runs). + if _BUYER_REJECT_RX.match(name): + return "" + return name + + +def _extract_buyer(text: str, paras: list[str]) -> str: + """Buyer (Abnehmer/Empfänger) name from the body. + + Two document layouts exist: + A) Side-by-side columns (two table cells): "Name:" and + "Name:" appear as two separate runs in the full text. + B) Stacked blocks: a "Empfänger/Abnehmer:" header precedes the buyer's + "Name:". + Strategy: collect every "Name:" value that looks like a person and is not + the seller (Drazena Rimac). The buyer is such a value. + """ + # Layout B: explicit Abnehmer/Empfänger header followed by Name:. + bm = re.search( + r"(?:Empf\w+nger|Abnehmer)[^:]*:\s*(?:[^:]*?\s)?Name\s*:\s*" + rf"(.+?)(?=\s*(?:{_ADDR_STOP})\s*:|$)", + text, re.IGNORECASE, + ) + if bm: + cand = _clean_person(bm.group(1)) + if cand and not _SELLER_RX.match(cand): + return cand + + # Layout A / fallback: scan all Name: values; pick the first non-seller one. + for m in re.finditer( + rf"Name\s*:\s*(.+?)(?=\s*(?:{_ADDR_STOP})\s*:|\s*Name\s*:|$)", + text, re.IGNORECASE, + ): + cand = _clean_person(m.group(1)) + if not cand or _SELLER_RX.match(cand): + continue + # Skip the animal block: animal Name is immediately followed by + # Geburtsdatum/Geschlecht/Farbschlag. + tail = text[m.end():m.end() + 40] + if re.match(r"\s*(?:Geburtsdatum|Geschlecht|Farbschlag)\s*:", tail, re.IGNORECASE): + continue + return cand + return "" + + +def parse_contract(docx_path: str) -> dict | None: + """Parse one .docx. Returns a contract record or None if not a contract.""" + fname = os.path.basename(docx_path) + try: + text = _full_text(docx_path) + paras = _paragraphs(docx_path) + except (zipfile.BadZipFile, KeyError, OSError): + return None # unreadable / not a real docx + + if any(m in text for m in NON_CONTRACT_MARKERS): + return None # Abstammungsnachweis / Geburtsurkunde — not a sale contract + + fn = parse_filename(fname) + + # --- animal block (body) --- + body_name = _label_value(text, r"Name") + # The first "Name:" in the body could be the seller's. The animal's name + # appears under "Tierdaten:". Prefer the Name that directly precedes + # Geburtsdatum/Geschlecht (the animal block). + animal_name = "" + # The animal block is headed by "Tierdaten:" (when present) and its Name is + # the LAST "Name:" before "Geburtsdatum:". Anchor on Tierdaten if present, + # then take the closest Name: to Geburtsdatum. + scope = text + ti = re.search(r"Tierdaten\s*:", text, re.IGNORECASE) + if ti: + scope = text[ti.end():] + am = re.search( + r"Name\s*:\s*(.*?)\s*Geburtsdatum\s*:", + scope, re.IGNORECASE, + ) + if am: + cand = am.group(1).strip() + # Greedy guard: if it still spans multiple labels, keep only the tail + # after the last embedded "Name:". + if "Name:" in cand or re.search(r"Name\s*:", cand): + cand = re.split(r"Name\s*:", cand)[-1].strip() + # Drop any leading address-block leftovers. + cand = re.split(rf"\s*(?:{_ADDR_STOP})\s*:", cand)[-1].strip() + animal_name = cand + # Body animal name is often blank (the call-name lives in the filename). + if not animal_name or len(animal_name) > 40: + animal_name = "" + + dob = _norm_date(_label_value(text, r"Geburtsdatum")) + gender_raw = _label_value(text, r"Geschlecht").lower() + if gender_raw.startswith("m"): + gender = "Male" + elif gender_raw.startswith("w"): + gender = "Female" + else: + gender = "" + + color = _label_value(text, r"Farbschlag") or _label_value(text, r"Farbe") + color = color.strip() + if not color and fn["color"]: + color = fn["color"] + + handover = _norm_date( + _label_value(text, r"Abgabedatum") or _label_value(text, r"Abgabe\s*am") + ) + + price = "" + for lab in (r"Gesamtpreis", r"Schutzgeb\w+", r"Kaufpreis", r"Preis"): + raw = _label_value(text, lab) + if raw: + price = _clean_money(raw) + if price: + break + + # contract date: trailing "Ort, [den ]DD.MM.YYYY" near signature line + contract_date = "" + cm = re.findall(r"[A-Za-zÄÖÜäöü.\- ]+,\s*(?:den\s*)?(\d{1,2}\.\d{1,2}\.\d{2,4})", + text) + if cm: + contract_date = _norm_date(cm[-1]) + + buyer = _extract_buyer(text, paras) + if not buyer and fn["buyer"]: + buyer = fn["buyer"] + + # Animal call-names: prefer filename (the call-names the breeder filed by), + # fall back to body animal name. + animals = list(fn["animals"]) + if not animals and animal_name: + animals = [animal_name] + + # Reject if we have neither a buyer nor any animal name — useless record. + if not buyer and not animals: + return None + + return { + "sourceFile": fname, + "buyer": buyer, + "animals": animals, + "animalNameBody": animal_name, + "color": color, + "gender": gender, + "dob": dob, + "handoverDate": handover, + "contractDate": contract_date, + "price": price, + } + + +def scan(dir_path: str, limit: int | None = None): + """Walk the share, parse every .docx. Returns (records, stats).""" + records = [] + stats = { + "files_seen": 0, "template_skipped": 0, "non_contract_skipped": 0, + "unreadable": 0, "parsed": 0, "unparseable": 0, + "with_buyer": 0, "with_handover": 0, "with_price": 0, + "with_color": 0, "with_dob": 0, "with_animals": 0, + } + for root, _, files in os.walk(dir_path): + for f in sorted(files): + if not f.lower().endswith(".docx"): + continue + stats["files_seen"] += 1 + low = f.lower() + if any(m in low for m in TEMPLATE_MARKERS): + stats["template_skipped"] += 1 + continue + path = os.path.join(root, f) + try: + rec = parse_contract(path) + except Exception: # never let one bad file kill the run + rec = None + if rec is None: + # Distinguish unreadable vs non-contract vs genuinely unparseable + try: + _ = _full_text(path) + txt = _ + if any(m in txt for m in NON_CONTRACT_MARKERS): + stats["non_contract_skipped"] += 1 + else: + stats["unparseable"] += 1 + except Exception: + stats["unreadable"] += 1 + continue + stats["parsed"] += 1 + if rec["buyer"]: + stats["with_buyer"] += 1 + if rec["handoverDate"]: + stats["with_handover"] += 1 + if rec["price"]: + stats["with_price"] += 1 + if rec["color"]: + stats["with_color"] += 1 + if rec["dob"]: + stats["with_dob"] += 1 + if rec["animals"]: + stats["with_animals"] += 1 + records.append(rec) + if limit and len(records) >= limit: + return records, stats + return records, stats + + +def main(): + try: + sys.stdout.reconfigure(encoding="utf-8", errors="replace") + except Exception: + pass + + ap = argparse.ArgumentParser(description="Abgabevertrag-Extraktor") + ap.add_argument("--dir", default=DEFAULT_DIR, help="Verträge-Ordner") + ap.add_argument("--limit", type=int, default=None, + help="Nur die ersten N Verträge (Stichprobe)") + args = ap.parse_args() + + if not os.path.isdir(args.dir): + print(f"Fehler: Ordner nicht gefunden: {args.dir}", file=sys.stderr) + sys.exit(1) + + os.makedirs(OUT, exist_ok=True) + print(f"Scanne: {args.dir}") + records, stats = scan(args.dir, args.limit) + + out_path = os.path.join(OUT, "contracts.json") + with open(out_path, "w", encoding="utf-8") as f: + json.dump(records, f, ensure_ascii=False, indent=2) + + print(f"Dateien gesehen: {stats['files_seen']}") + print(f" Vorlage übersprungen: {stats['template_skipped']}") + print(f" Nicht-Vertrag (AN): {stats['non_contract_skipped']}") + print(f" unlesbar (kein ZIP): {stats['unreadable']}") + print(f" unparsbar: {stats['unparseable']}") + print(f"Verträge geparst: {stats['parsed']}") + print(f" mit Käufer: {stats['with_buyer']}") + print(f" mit Tier(en): {stats['with_animals']}") + print(f" mit Geburtsdatum: {stats['with_dob']}") + print(f" mit Abgabedatum: {stats['with_handover']}") + print(f" mit Preis: {stats['with_price']}") + print(f" mit Farbschlag: {stats['with_color']}") + print(f"Ausgabe: {out_path}") + + +if __name__ == "__main__": + main() diff --git a/tools/import/merge_and_resolve.py b/tools/import/merge_and_resolve.py index d185f37..ce730ef 100644 --- a/tools/import/merge_and_resolve.py +++ b/tools/import/merge_and_resolve.py @@ -944,6 +944,168 @@ def _format_discard(d): return DISCARD_MARK + line + "." +def _append_history(prov_json, line): + """Append one history line to an existing Provenance JSON string and add the + contract source file. Returns the updated JSON string.""" + try: + prov = json.loads(prov_json) if prov_json else {} + except (ValueError, TypeError): + prov = {} + prov.setdefault("history", []) + if line not in prov["history"]: + prov["history"].append(line) + return json.dumps(prov, ensure_ascii=False) + + +def enrich_from_contracts(contracts, resolved_gerbils, contact_by_norm_name, + contact_id_map): + """Conservatively fold Abgabevertrag data into the resolved gerbils. + + For every parsed contract we (a) ensure the buyer exists as a (receiver) + contact, reusing the existing contact dedup/normalisation, and (b) try to + match each animal call-name to exactly one resolved gerbil that the breeder + bred ("…Chaoten"). On a confident match we set ReceiverContactId / + GoHomeDate / Status=GivenAway — but only where not already set differently — + and add a provenance history line. Ambiguous or absent matches are logged, + never guessed. + + Returns a stats dict. Mutates resolved_gerbils + contact_by_norm_name in + place. New buyer contacts are appended via contact_by_norm_name so the later + IsReceiver-flag pass picks them up automatically. + """ + stats = { + "contracts": len(contracts), "buyers_created": 0, "buyers_existing": 0, + "matched": 0, "ambiguous_skipped": 0, "no_match_skipped": 0, + "receiver_set": 0, "gohome_set": 0, "status_givenaway": 0, + "conflicts": 0, + } + if not contracts: + return stats + + # Index breeder-owned gerbils by dedup name key. Contracts only ever sell + # animals the breeder bred, so restrict candidates to her own stock to avoid + # colliding with same-named foreign-bred animals. + def is_own(g): + ob = (g.get("OriginBreeder") or "").lower() + return ("chaoten" in ob) or (g.get("OriginBreeder") is None) + + index = {} + for g in resolved_gerbils: + if not is_own(g): + continue + key = get_dedup_name_key(get_call_name(g.get("Name", ""))) + if key: + index.setdefault(key, []).append(g) + + def year_of(iso): + return iso[:4] if iso else None + + # Reject buyer values that are obviously label leakage / non-person noise + # (defends against any stale contracts.json produced before the parser fix). + _buyer_junk = re.compile( + r"^(?:stra\w+e|wohnort|fon|e-?mail|handy|festnetz|telefon|homepage|" + r"zuchtname|facebook|name)\s*:?\s*$|^[\d\s/]+$", re.IGNORECASE) + + for c in contracts: + fname = c.get("sourceFile", "") + buyer_raw = (c.get("buyer") or "").strip() + if buyer_raw and _buyer_junk.match(buyer_raw): + buyer_raw = "" + + # --- buyer contact (reuse curated normalisation/dedup) --- + buyer_global_id = None + if buyer_raw: + canon, keep = get_normalized_contact_name(buyer_raw) + if keep and canon: + norm = normalize_name(canon) + if norm in contact_by_norm_name: + gc = contact_by_norm_name[norm] + gc.setdefault("_source_files", set()).add(fname) + gc["_merged_count"] = gc.get("_merged_count", 1) + 1 + stats["buyers_existing"] += 1 + else: + gid = generate_guid(f"contact-{norm}") + contact_by_norm_name[norm] = { + "Id": gid, "Name": canon, "Email": None, "Phone": None, + "Address": None, "Notes": None, "NameSuffix": None, + "_source_files": {fname}, "_merged_count": 1, + } + stats["buyers_created"] += 1 + buyer_global_id = contact_by_norm_name[norm]["Id"] + + # --- match each animal call-name to a resolved gerbil --- + handover = parse_date(c.get("handoverDate")) + c_year = year_of(parse_date(c.get("dob"))) if c.get("dob") else None + c_color = (c.get("color") or "").strip().lower() + + for call in (c.get("animals") or []): + key = get_dedup_name_key(get_call_name(call)) + if not key: + continue + cands = index.get(key, []) + if not cands: + stats["no_match_skipped"] += 1 + continue + + # Corroborate when more than one candidate shares the call-name. + chosen = None + if len(cands) == 1: + chosen = cands[0] + else: + scored = [] + for g in cands: + score = 0 + g_year = year_of(g.get("DateOfBirth")) + if c_year and g_year and c_year == g_year: + score += 2 + if c_color and g.get("ColorVarietyId"): + # color match is corroboration; we don't have the name + # here, so only DOB drives disambiguation strongly. + pass + scored.append((score, g)) + scored.sort(key=lambda t: t[0], reverse=True) + if scored[0][0] >= 2 and (len(scored) == 1 or scored[0][0] > scored[1][0]): + chosen = scored[0][1] + else: + stats["ambiguous_skipped"] += 1 + continue + + stats["matched"] += 1 + + # --- set receiver, only if not already set differently --- + if buyer_global_id: + cur = chosen.get("ReceiverContactId") + if not cur: + chosen["ReceiverContactId"] = buyer_global_id + stats["receiver_set"] += 1 + chosen["Provenance"] = _append_history( + chosen.get("Provenance"), + f"Abgabe an „{buyer_raw}“ aus Vertrag {_quote_file(fname)} übernommen.") + elif cur != buyer_global_id: + stats["conflicts"] += 1 + chosen["Provenance"] = _append_history( + chosen.get("Provenance"), + f"Vertrag {_quote_file(fname)} nennt anderen Abnehmer „{buyer_raw}“ " + f"— bestehende Zuordnung beibehalten.") + + # --- set go-home date, only if empty --- + if handover and not chosen.get("GoHomeDate"): + chosen["GoHomeDate"] = handover + stats["gohome_set"] += 1 + chosen["Provenance"] = _append_history( + chosen.get("Provenance"), + f"Abgabedatum {handover} aus Vertrag {_quote_file(fname)} übernommen.") + + # --- status: derive GivenAway if we set a receiver and it isn't + # already a stronger state (Deceased). --- + if chosen.get("ReceiverContactId") and chosen.get("Status") not in ( + "Deceased", "GivenAway"): + chosen["Status"] = "GivenAway" + stats["status_givenaway"] += 1 + + return stats + + def main(): print("Loading color variety seeds...") variety_map = {} @@ -1176,6 +1338,17 @@ def main(): else: print(f"Warning: docx_litters.json not found at {docx_litters_path}") + # Abgabevertrag-Datensätze (aus extract_contracts.py). Optional: wenn die + # Datei fehlt, läuft der Import ohne Vertrags-Anreicherung normal weiter. + contracts = [] + contracts_path = os.path.join(OUTPUT_DIR, "contracts.json") + if os.path.exists(contracts_path): + with open(contracts_path, "r", encoding="utf-8") as f: + contracts = json.load(f) + print(f"Loaded {len(contracts)} Abgabeverträge.") + else: + print(f"Info: contracts.json not found at {contracts_path} (Vertrags-Anreicherung übersprungen)") + # Extract docx buyer contacts and add to raw_contacts for da in docx_animals: o_name = (da.get("owner") or "").strip() @@ -2685,6 +2858,28 @@ def main(): if n_discards: print(f"Datenherkunft: recorded {n_discards} discard line(s) across gerbils.") + # Abgabevertrag-Anreicherung: Käufer als Abnehmer-Kontakte anlegen und — + # konservativ — auf eindeutig passende Tiere ReceiverContactId/GoHomeDate/ + # Status=GivenAway setzen (nur falls noch nicht gesetzt). Provenance- + # Historie wird ergänzt. Neue Käuferkontakte landen in contact_by_norm_name + # und werden danach automatisch als IsReceiver markiert. + cstats = enrich_from_contracts( + contracts, resolved_gerbils, contact_by_norm_name, contact_id_map) + if contracts: + print("Abgabeverträge: " + f"{cstats['contracts']} geladen, {cstats['matched']} Tier-Treffer, " + f"{cstats['ambiguous_skipped']} mehrdeutig übersprungen, " + f"{cstats['no_match_skipped']} ohne Treffer.") + print(" Kontakte: " + f"{cstats['buyers_created']} neu, {cstats['buyers_existing']} bestehend.") + print(" Gesetzt: " + f"ReceiverContactId={cstats['receiver_set']}, " + f"GoHomeDate={cstats['gohome_set']}, " + f"Status=GivenAway={cstats['status_givenaway']}, " + f"Konflikte={cstats['conflicts']}.") + # Re-materialise contacts so freshly created buyer contacts are exported. + resolved_contacts = list(contact_by_norm_name.values()) + # Datenherkunft for litters: which source files contributed, whether this is # a Wurfchronik litter vs a Stammbaum-reconstructed ("virtual") litter, how # many raw records merged into it, plus human-readable notes. Accumulators diff --git a/tools/import/test_extract_contracts.py b/tools/import/test_extract_contracts.py new file mode 100644 index 0000000..3304327 --- /dev/null +++ b/tools/import/test_extract_contracts.py @@ -0,0 +1,189 @@ +"""Tests for extract_contracts.py — run: python test_extract_contracts.py + +Zero third-party deps (mirrors test_extract_docx.py). Builds tiny in-memory +.docx files (a zip with word/document.xml) so the tests run without the +network share. Covers: filename parsing, run-splitting de-mangling, both body +layouts (side-by-side / stacked), price/date normalisation, and skip rules. +""" +import io +import os +import sys +import zipfile + +try: + sys.stdout.reconfigure(encoding="utf-8", errors="replace") +except Exception: + pass + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import extract_contracts as ec + +failed = 0 + + +def check(name, cond): + global failed + print(("ok: " if cond else "FAIL: ") + name) + if not cond: + failed += 1 + + +def make_docx(paragraphs): + """paragraphs: list of lists of run-strings → bytes of a .docx zip.""" + body = [] + for runs in paragraphs: + rs = "".join(f"{r}" for r in runs) + body.append(f"{rs}") + xml = ('' + + "".join(body) + "") + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as z: + z.writestr("word/document.xml", xml) + return buf.getvalue() + + +def write_tmp(name, data): + d = os.path.join(os.path.dirname(os.path.abspath(__file__)), "_test_tmp") + os.makedirs(d, exist_ok=True) + p = os.path.join(d, name) + with open(p, "wb") as f: + f.write(data) + return p + + +# --- _norm_date ----------------------------------------------------------- +check("date 4-digit year", ec._norm_date("18.02.2014") == "2014-02-18") +check("date 2-digit year", ec._norm_date("09.12.14") == "2014-12-09") +check("date embedded", ec._norm_date("Bonn, 20.09.2014") == "2014-09-20") +check("date invalid", ec._norm_date("99.99.9999") == "") +check("date empty", ec._norm_date("") == "") + +# --- _clean_money --------------------------------------------------------- +check("money comma", ec._clean_money("27,50 Euro (Überweisung)") == "27,50") +check("money dot→comma", ec._clean_money("25.00 €") == "25,00") +check("money plain", ec._clean_money("28 Euro") == "28") +check("money none", ec._clean_money("kostenlos") == "") + +# --- parse_filename ------------------------------------------------------- +fn = ec.parse_filename( + "Zucht der kleinen Chaoten _ Agouti (Kathlin.Baxter) - Stephan Füchsle_.docx") +check("fn color", fn["color"] == "Agouti") +check("fn animals", fn["animals"] == ["Kathlin", "Baxter"]) +check("fn buyer", fn["buyer"] == "Stephan Füchsle") + +fn2 = ec.parse_filename( + "Zucht der kleinen Chaoten _ dd Polar Sp (Velvet.Vance Jr.)- Alexandra Wendler_.docx") +check("fn multiword color", fn2["color"] == "dd Polar Sp") +check("fn multiword animal", fn2["animals"] == ["Velvet", "Vance Jr"]) +check("fn buyer 2", fn2["buyer"] == "Alexandra Wendler") + +fn3 = ec.parse_filename("Zucht der kleinen Chaoten _ Ethan - Stefanie Stoica_.docx") +check("fn no-paren animal", fn3["animals"] == ["Ethan"]) +check("fn no-paren buyer", fn3["buyer"] == "Stefanie Stoica") + +fn4 = ec.parse_filename("Zucht der kleinen Chaoten _Azrael_.docx") +check("fn name-only animal", fn4["animals"] == ["Azrael"]) +check("fn name-only no buyer", fn4["buyer"] == "") + +# --- run-splitting de-mangling (the core text-extraction fix) ------------- +data = make_docx([["Name:", "F", "r", "au Josephin Kiefer"]]) +p = write_tmp("split.docx", data) +check("run-split joins to 'Frau Josephin Kiefer'", + ec._full_text(p) == "Name:Frau Josephin Kiefer") + +money = make_docx([["Schutzgebühr: ", "3", "0,00 €"]]) +p = write_tmp("money.docx", money) +check("run-split price intact", + ec._clean_money(ec._label_value(ec._full_text(p), r"Schutzgeb\w+")) == "30,00") + +# --- full contract: stacked layout (B) ------------------------------------ +stacked = make_docx([ + ["Vermittlungsvertrag"], + ["Abgebender/Züchter:"], + ["Name:", "Frau Drazena Rimac"], + ["Straße:", "New-York-Str. 30"], + ["Empfänger/Abnehmer:"], + ["Name:", "Familie Tanja und Thorsten Kurz"], + ["Straße:", "Ebelstraße 2"], + ["Tierdaten:"], + ["Name:", "Einstein"], + ["Geburtsdatum:", "10.08.2019"], + ["Geschlecht:", "männlich"], + ["Farbschlag: ", "Schwarz"], + ["Abgabedatum:", "31.12.2019"], + ["Schutzgebühr: ", "25,00 €"], + ["Butzbach, den 31.12.2019"], +]) +p = write_tmp("stacked.docx", + make_docx_b := stacked) +rec = ec.parse_contract(p) +check("stacked buyer", rec["buyer"] == "Familie Tanja und Thorsten Kurz") +check("stacked not seller", rec["buyer"] != "Frau Drazena Rimac") +check("stacked animal name", rec["animalNameBody"] == "Einstein") +check("stacked dob", rec["dob"] == "2019-08-10") +check("stacked gender", rec["gender"] == "Male") +check("stacked color", rec["color"] == "Schwarz") +check("stacked handover", rec["handoverDate"] == "2019-12-31") +check("stacked price", rec["price"] == "25,00") +check("stacked contract date", rec["contractDate"] == "2019-12-31") + +# --- full contract: side-by-side columns (A) ------------------------------ +# Two columns render as separate paragraphs (table cells). +sidebyside = make_docx([ + ["Vermittlungsvertrag"], + ["Züchter:"], ["Abnehmer:"], + ["Name:", "Drazena Rimac"], ["Name:", "Andrea Thesing"], + ["Straße:", "Alten-Busecker-Str. 57"], ["Straße:", "Veilchenweg 32"], + ["Tierdaten:"], + ["Name:", "Sally"], + ["Geburtsdatum:", "03.02.2014"], + ["Geschlecht:", "weiblich"], + ["Farbschlag:", "Kohlfuchs-hell"], + ["Abgabedatum:", "20.09.2014"], + ["Gesamtpreis: ", "27,00 Euro (Überweisung)"], + ["Bonn, 20.09.2014"], +]) +p = write_tmp("sidebyside.docx", sidebyside) +rec = ec.parse_contract(p) +check("sbs buyer is Andrea Thesing", rec["buyer"] == "Andrea Thesing") +check("sbs buyer not seller", rec["buyer"] != "Drazena Rimac") +check("sbs animal", rec["animalNameBody"] == "Sally") +check("sbs gender female", rec["gender"] == "Female") +check("sbs price", rec["price"] == "27,00") + +# --- skip rules ----------------------------------------------------------- +abstammung = make_docx([ + ["Abstammungsnachweis"], + ["Name:"], ["Grace"], + ["Geburtsdatum:", "04.10.2015"], +]) +p = write_tmp("abst.docx", abstammung) +check("Abstammungsnachweis skipped", ec.parse_contract(p) is None) + +# bad zip +badp = write_tmp("bad.docx", b"not a zip") +check("bad zip → None", ec.parse_contract(badp) is None) + +# --- filename color fallback when body has no Farbschlag ------------------ +nocolor = make_docx([ + ["Vermittlungsvertrag"], ["Abnehmer:"], ["Name:", "Max Mustermann"], + ["Tierdaten:"], ["Name:", "Bello"], + ["Geburtsdatum:", "01.01.2020"], ["Geschlecht:", "männlich"], + ["Gesamtpreis: ", "20,00 €"], +]) +p = write_tmp("Zucht der kleinen Chaoten _ Agouti (Bello)- Max Mustermann_.docx", + nocolor) +rec = ec.parse_contract(p) +check("color falls back to filename", rec["color"] == "Agouti") +check("animals from filename", rec["animals"] == ["Bello"]) + +# cleanup tmp +import shutil +shutil.rmtree(os.path.join(os.path.dirname(os.path.abspath(__file__)), "_test_tmp"), + ignore_errors=True) + +print() +if failed: + print(f"{failed} test(s) FAILED") + sys.exit(1) +print("All extract_contracts tests passed.")