Compare commits
2 Commits
2befe53df9
...
5e14124322
| Author | SHA1 | Date | |
|---|---|---|---|
| 5e14124322 | |||
| aa473b764e |
@@ -135,7 +135,11 @@ export function seedDb(): MockDb {
|
||||
'Geburtsdatum (12.03.2025) aus „Stammbaum von Krümel.xlsx“.',
|
||||
'Genotyp aus „Stammbaum von Krümel.xlsx“.',
|
||||
'Auch in „Wurfchronik-Detail.docx“ gefunden → Datensätze zusammengeführt.',
|
||||
// VERWORFEN: abweichender Wert aus einer Quelle wurde verworfen (Grund + Ersatz).
|
||||
'⚠ Geburtsdatum 14.06.2015 aus „Wurfchronik-Detail.docx“ verworfen — abweichend; 12.03.2025 aus „Stammbaum von Krümel.xlsx“ verwendet (Mehrheit).',
|
||||
'Eltern über Position im Stammbaum erkannt (Quelle: „Stammbaum von Krümel.xlsx“).',
|
||||
// VERWORFEN: unplausibler Elternteil verworfen, kein Ersatz verwendet.
|
||||
'⚠ Vater „Jayjay“ (*19.06.2013) verworfen — unplausibel (9 Jahre älter als das Kind); kein Ersatz.',
|
||||
'Angaben aus der Wurfchronik übernommen.',
|
||||
],
|
||||
}),
|
||||
|
||||
@@ -21,6 +21,17 @@ test('Tierakte: "Datenherkunft"-Button öffnet den Nachverfolgungs-Dialog', asyn
|
||||
'Auch in „Wurfchronik-Detail.docx“ gefunden → Datensätze zusammengeführt.',
|
||||
)
|
||||
|
||||
// VERWORFEN: eine Quelle wurde verworfen — mit Grund und verwendetem Ersatz.
|
||||
await expect(dialog).toContainText(p.discardLegend)
|
||||
await expect(dialog).toContainText(
|
||||
'⚠ Geburtsdatum 14.06.2015 aus „Wurfchronik-Detail.docx“ verworfen — abweichend; 12.03.2025 aus „Stammbaum von Krümel.xlsx“ verwendet (Mehrheit).',
|
||||
)
|
||||
await expect(dialog).toContainText(
|
||||
'⚠ Vater „Jayjay“ (*19.06.2013) verworfen — unplausibel (9 Jahre älter als das Kind); kein Ersatz.',
|
||||
)
|
||||
// Die Verwerfen-Zeile ist visuell als solche markiert (eigene CSS-Klasse).
|
||||
await expect(dialog.locator('.provenance__history-step--discard').first()).toBeVisible()
|
||||
|
||||
// Quelldateien werden angezeigt.
|
||||
await expect(dialog).toContainText(p.sourceFilesTitle)
|
||||
await expect(dialog).toContainText('Stammbaum von Krümel.xlsx')
|
||||
|
||||
@@ -12,6 +12,11 @@ import { de } from '../strings/de'
|
||||
import type { EntityProvenance } from '../api/types'
|
||||
import './provenanceDialog.css'
|
||||
|
||||
/** Leading marker the Python merge prefixes onto discard ("verworfen") history
|
||||
* lines (data thrown away with reason + replacement). Kept in sync with
|
||||
* DISCARD_MARK in tools/import/merge_and_resolve.py. */
|
||||
const DISCARD_PREFIX = '⚠ '
|
||||
|
||||
interface ProvenanceDialogProps {
|
||||
open: boolean
|
||||
onClose: () => void
|
||||
@@ -92,12 +97,27 @@ export default function ProvenanceDialog({ open, onClose, provenance, entityLabe
|
||||
{p.history.length > 0 && (
|
||||
<section className="provenance__section">
|
||||
<div className="provenance__section-title">{t.historyTitle}</div>
|
||||
{p.history.some((s) => s.startsWith(DISCARD_PREFIX)) && (
|
||||
<p className="provenance__discard-legend">{t.discardLegend}</p>
|
||||
)}
|
||||
<ol className="provenance__history">
|
||||
{p.history.map((step, i) => (
|
||||
<li key={`${i}-${step}`} className="provenance__history-step">
|
||||
{step}
|
||||
</li>
|
||||
))}
|
||||
{p.history.map((step, i) => {
|
||||
// Discard lines (Python prefixes them with „⚠ “) are styled
|
||||
// distinctly so a thrown-away value stands out from accepted facts.
|
||||
const isDiscard = step.startsWith(DISCARD_PREFIX)
|
||||
return (
|
||||
<li
|
||||
key={`${i}-${step}`}
|
||||
className={
|
||||
isDiscard
|
||||
? 'provenance__history-step provenance__history-step--discard'
|
||||
: 'provenance__history-step'
|
||||
}
|
||||
>
|
||||
{step}
|
||||
</li>
|
||||
)
|
||||
})}
|
||||
</ol>
|
||||
</section>
|
||||
)}
|
||||
|
||||
@@ -163,6 +163,19 @@
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
/* Verworfene Werte (Datenherkunft): hervorgehoben, damit klar wird, dass hier
|
||||
Daten aus einer Quelle verworfen wurden — mit Grund und Ersatz. */
|
||||
.provenance__history-step--discard {
|
||||
color: var(--color-danger, #9a3412);
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
.provenance__discard-legend {
|
||||
margin: 0 0 0.5rem;
|
||||
font-size: 0.78rem;
|
||||
color: var(--color-text-muted, #888);
|
||||
}
|
||||
|
||||
.provenance__actions {
|
||||
display: flex;
|
||||
justify-content: flex-end;
|
||||
|
||||
@@ -896,6 +896,9 @@ export const de = {
|
||||
},
|
||||
/** Überschrift des chronologischen, dateibezogenen Verlaufs (Primärinhalt). */
|
||||
historyTitle: 'Verlauf',
|
||||
/** Legende für die mit „⚠“ markierten Verlaufszeilen, in denen Daten aus
|
||||
* einer Quelle verworfen wurden (mit Grund und verwendetem Ersatz). */
|
||||
discardLegend: '⚠ markiert verworfene Quelldaten (mit Grund und verwendetem Ersatz).',
|
||||
/** Überschriften / Feldbeschriftungen. */
|
||||
sourceFilesTitle: 'Quelldateien',
|
||||
sourceFilesCount: (n: number) => (n === 1 ? 'aus 1 Quelle' : `aus ${n} Quellen`),
|
||||
|
||||
@@ -51,12 +51,25 @@ preserving everything: `genotype.mapped8locus`, `genotype.rawGenotype` (verbatim
|
||||
|
||||
```sh
|
||||
cd tools/import
|
||||
python extract.py # uses the default source paths
|
||||
python extract.py # xlsx → animals.json / litters.json
|
||||
python extract.py --stammbaeume "<dir>" --wurfchronik "<file.xlsx>"
|
||||
python extract_docx.py # Wurfchronik-Detail.docx → docx_*.json
|
||||
python extract_contracts.py # Abgabeverträge (.docx) → contracts.json
|
||||
python merge_and_resolve.py # → resolved_import.json (DB-ready)
|
||||
```
|
||||
|
||||
Requires Python 3. **Re-runnable / idempotent** — re-run when more files arrive
|
||||
(Wurfchronik `Teil2+`, or new charts).
|
||||
Requires Python 3 (zero third-party deps). **Re-runnable / idempotent** — re-run
|
||||
when more files arrive (Wurfchronik `Teil2+`, new charts, or new contracts).
|
||||
|
||||
`extract_contracts.py` scans the breeder's sale-contract share
|
||||
(`\\truenas\…\Verträge`, ~1.4k `.docx`) and emits one record per contract
|
||||
(buyer, animal call-names, Farbschlag, dates, price, source filename). It skips
|
||||
the blank template, `Abstammungsnachweis`/`Geburtsurkunde` documents, and any
|
||||
file that is not a readable `.docx`. `merge_and_resolve.py` then conservatively
|
||||
folds contracts into the resolved data: buyers become receiver `Contacts`, and
|
||||
unambiguously matched gerbils get `ReceiverContactId` / `GoHomeDate` /
|
||||
`Status=GivenAway` (only where not already set), with a provenance history line.
|
||||
Ambiguous / unmatched animals are counted and skipped, never guessed.
|
||||
|
||||
## Output (`tools/import/output/`, git-ignored except the report)
|
||||
|
||||
@@ -64,6 +77,9 @@ Requires Python 3. **Re-runnable / idempotent** — re-run when more files arriv
|
||||
|---|---|
|
||||
| `animals.json` | deduped animals with genotype, parentRefs, photos, sourceFiles |
|
||||
| `litters.json` | litters from the Wurfchronik |
|
||||
| `docx_animals.json` / `docx_litters.json` | Wurfchronik-Detail.docx rows |
|
||||
| `contracts.json` | one record per Abgabevertrag (buyer, animals, dates, price) |
|
||||
| `resolved_import.json` | merged DB-ready payload consumed by `IngestResolvedService` |
|
||||
| `photos/<slug>/…` | extracted, anchor-mapped images |
|
||||
| `review-report.md` | **human review deliverable** (committed) |
|
||||
|
||||
|
||||
@@ -774,6 +774,9 @@ def dedup(animals):
|
||||
# FEAT-8c: machine-readable quarantine marker so the API loader can skip
|
||||
# conflicting records without parsing the German review report.
|
||||
"conflict": False,
|
||||
# DOB-Remap (manuelle Entscheidung): original/corrected birthdate so the
|
||||
# downstream Datenherkunft can record the discarded original value.
|
||||
"dobRemap": next((a["dobRemap"] for a in grp if a.get("dobRemap")), None),
|
||||
}
|
||||
merged.append(out)
|
||||
# conflict: same animal, GENUINELY disagreeing genotype (presence-vs-absence is NOT a
|
||||
@@ -1037,6 +1040,9 @@ def apply_dob_remaps(raw_animals, path):
|
||||
dob = norm_dob(a.get("dob", ""))
|
||||
new = remaps_full.get((nc, zc, dob)) or remaps_name.get((nc, dob))
|
||||
if new and a.get("dob") != new:
|
||||
# Keep the discarded original DOB so the provenance/Datenherkunft can
|
||||
# show „Geburtsdatum X verworfen, per Entscheidung auf Y geändert.“
|
||||
a["dobRemap"] = {"original": a.get("dob"), "corrected": new}
|
||||
a["dob"] = new
|
||||
n += 1
|
||||
return n
|
||||
|
||||
446
tools/import/extract_contracts.py
Normal file
446
tools/import/extract_contracts.py
Normal file
@@ -0,0 +1,446 @@
|
||||
#!/usr/bin/env python3
|
||||
"""FEAT-Contracts Stage 1 — Abgabeverträge (.docx) extrahieren.
|
||||
|
||||
Liest die Sammlung der Vermittlungs-/Abgabeverträge ("Zucht der kleinen
|
||||
Chaoten _ … (Tier.Namen) - <Käufer>_.docx") vom Netzlaufwerk und erzeugt:
|
||||
output/contracts.json — strukturierte Vertrags-Datensätze
|
||||
|
||||
Reines stdlib-Python (kein pip), nach Vorbild von extract_docx.py: .docx ist
|
||||
ein ZIP mit word/document.xml; daraus werden die "Label: Wert"-Zeilen des
|
||||
Vertragskörpers gelesen. Der Dateiname dient als Zusatz-Signal für die
|
||||
Tier-Rufnamen und den Farbschlag, wenn der Körper sie nicht hergibt.
|
||||
|
||||
Zuverlässig extrahierbar (beobachtete Abdeckung über 250er-Stichprobe):
|
||||
Name (Tier): 100 % — Body-Label "Name:"
|
||||
Geschlecht: 100 % — Body-Label "Geschlecht:"
|
||||
Geburtsdatum: 98 % — Body-Label "Geburtsdatum:"
|
||||
Abgabedatum: 78 % — Body-Label "Abgabedatum:"
|
||||
Preis: 78 % — "Gesamtpreis/Schutzgebühr/Kaufpreis"
|
||||
Käufer (Body): 79 % — Block "Abnehmer/Empfänger" → "Name:"
|
||||
Farbschlag(Body): 13 % — Body-Label "Farbschlag:" (oft im Dateinamen)
|
||||
|
||||
Übersprungen werden:
|
||||
* die Vorlage "Vertrags mustter new.docx"
|
||||
* Abstammungsnachweise / Geburtsurkunden (kein Verkauf, eigener Doc-Typ)
|
||||
* Dateien, die kein lesbares ZIP sind (alte .doc als .docx getarnt)
|
||||
|
||||
Ausführung: python extract_contracts.py [--dir PFAD] [--limit N]
|
||||
Idempotent: mehrfaches Ausführen überschreibt output/contracts.json.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import json
|
||||
import zipfile
|
||||
import argparse
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
DEFAULT_DIR = r"\\truenas\Datengrab\Rennmäuse\Verträge"
|
||||
OUT = os.path.join(HERE, "output")
|
||||
|
||||
TEMPLATE_MARKERS = ("mustter", "muster new")
|
||||
# Doc types that are NOT sale contracts (skip):
|
||||
NON_CONTRACT_MARKERS = ("Abstammungsnachweis", "Geburtsurkunde")
|
||||
|
||||
# --- Body label regexes (work on whitespace-collapsed plain text) ----------
|
||||
# Generic "Label: value up to next known label or end".
|
||||
_KNOWN_LABELS = (
|
||||
r"Name|Geburtsdatum|Geschlecht|Farbschlag|Farbe|Abgabedatum|Abgabe\s*am|"
|
||||
r"Gesamtpreis|Schutzgeb\w+|Kaufpreis|Preis|Zuchtname|Zuchtbuchnummer|"
|
||||
r"Stra\w+e|Wohnort|Fon|Festnetz|Handy|Handynummer|Telefon|E-?Mail|Homepage|"
|
||||
r"Facebook|Mutter|Vater|Linie|Wurf|Bemerkung|Z\w+chter\w*|Abnehmer|Empf\w+nger"
|
||||
)
|
||||
|
||||
|
||||
def _norm_date(d: str) -> str:
|
||||
"""German DD.MM.YY[YY] → ISO YYYY-MM-DD (or '' if unparseable)."""
|
||||
if not d:
|
||||
return ""
|
||||
m = re.search(r"(\d{1,2})\.(\d{1,2})\.(\d{2,4})", d)
|
||||
if not m:
|
||||
return ""
|
||||
day, mon, yr = m.group(1), m.group(2), m.group(3)
|
||||
if len(yr) == 2:
|
||||
yr = "20" + yr
|
||||
try:
|
||||
di, mi, yi = int(day), int(mon), int(yr)
|
||||
if not (1 <= di <= 31 and 1 <= mi <= 12 and 1900 <= yi <= 2100):
|
||||
return ""
|
||||
except ValueError:
|
||||
return ""
|
||||
return f"{yi:04d}-{mi:02d}-{di:02d}"
|
||||
|
||||
|
||||
def _unescape(t: str) -> str:
|
||||
for a, b in (("&", "&"), ("<", "<"), (">", ">"),
|
||||
("'", "'"), (""", '"'), (" ", " ")):
|
||||
t = t.replace(a, b)
|
||||
return t
|
||||
|
||||
|
||||
def _para_text(p_xml: str) -> str:
|
||||
"""Plain text of one <w:p>.
|
||||
|
||||
IMPORTANT: Word frequently splits a single word across multiple <w:r>/<w:t>
|
||||
runs (formatting/spell-check artefacts). Run boundaries are NOT word
|
||||
boundaries, so we concatenate <w:t> contents directly (no separator) and
|
||||
only turn explicit tabs/breaks into spaces. This avoids mangling
|
||||
"Frau"→"F r au" or "30,00"→"3 0,00".
|
||||
"""
|
||||
p_xml = re.sub(r"<w:tab\b[^>]*/?>", " ", p_xml)
|
||||
p_xml = re.sub(r"<w:br\b[^>]*/?>", " ", p_xml)
|
||||
texts = re.findall(r"<w:t\b[^>]*>(.*?)</w:t>", p_xml, re.DOTALL)
|
||||
t = _unescape("".join(texts))
|
||||
return re.sub(r"\s+", " ", t).strip()
|
||||
|
||||
|
||||
def _full_text(docx_path: str) -> str:
|
||||
"""Return whitespace-collapsed plain text of the document body.
|
||||
|
||||
Paragraphs (and table cells, also wrapped in <w:p>) are joined by a single
|
||||
space so adjacent labels stay separable.
|
||||
"""
|
||||
with zipfile.ZipFile(docx_path) as z:
|
||||
xml = z.read("word/document.xml").decode("utf-8", errors="replace")
|
||||
paras = [_para_text(p) for p in re.findall(r"<w:p[ >].*?</w:p>", xml, re.DOTALL)]
|
||||
return re.sub(r"\s+", " ", " ".join(p for p in paras if p)).strip()
|
||||
|
||||
|
||||
def _paragraphs(docx_path: str) -> list[str]:
|
||||
"""Return per-paragraph plain text (preserves the Label/value line breaks)."""
|
||||
with zipfile.ZipFile(docx_path) as z:
|
||||
xml = z.read("word/document.xml").decode("utf-8", errors="replace")
|
||||
out = []
|
||||
for p in re.findall(r"<w:p[ >].*?</w:p>", xml, re.DOTALL):
|
||||
t = _para_text(p)
|
||||
if t:
|
||||
out.append(t)
|
||||
return out
|
||||
|
||||
|
||||
def _label_value(text: str, label_rx: str) -> str:
|
||||
"""Find 'Label: value' in collapsed text; stop at the next known label."""
|
||||
rx = re.compile(
|
||||
rf"(?:{label_rx})\s*:\s*(.+?)(?=\s*(?:{_KNOWN_LABELS})\s*:|$)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
m = rx.search(text)
|
||||
return m.group(1).strip() if m else ""
|
||||
|
||||
|
||||
def _clean_money(raw: str) -> str:
|
||||
"""'27,50 Euro (Überweisung)' → '27,50'. Returns '' if no number."""
|
||||
m = re.search(r"(\d+(?:[.,]\d{1,2})?)", raw)
|
||||
if not m:
|
||||
return ""
|
||||
return m.group(1).replace(".", ",")
|
||||
|
||||
|
||||
def parse_filename(fname: str) -> dict:
|
||||
"""Best-effort structured pieces from the contract filename.
|
||||
|
||||
Returns dict with keys: color, animals (list of call-names), buyer
|
||||
(may be empty). Robust to the many messy variants observed.
|
||||
"""
|
||||
base = re.sub(r"\.docx$", "", fname, flags=re.IGNORECASE)
|
||||
# Strip the leading cattery prefix and the leading/trailing underscores.
|
||||
base = re.sub(r"^\s*(?:Zucht der kleinen Chaoten|Clan[^_]*Chaoten)\s*",
|
||||
"", base, flags=re.IGNORECASE)
|
||||
base = base.strip().strip("_").strip()
|
||||
|
||||
color = ""
|
||||
animals: list[str] = []
|
||||
buyer = ""
|
||||
|
||||
# Animals are inside the (…) group, dot-separated call-names.
|
||||
pm = re.search(r"\(([^)]*)\)", base)
|
||||
if pm:
|
||||
inner = pm.group(1).strip()
|
||||
# split on dot (call-name separator) but keep multi-word names
|
||||
animals = [a.strip() for a in inner.split(".") if a.strip()]
|
||||
# color = text before "("
|
||||
color = base[:pm.start()].strip(" -–_")
|
||||
# buyer = text after ")"
|
||||
after = base[pm.end():].strip()
|
||||
bm = re.match(r"\s*[-–]\s*(.+)", after)
|
||||
if bm:
|
||||
buyer = bm.group(1).strip().strip("_").strip()
|
||||
else:
|
||||
# No parens. Two shapes:
|
||||
# "<token> - <Buyer>" (token is animal-name or color)
|
||||
# "<AnimalName>" (just a name)
|
||||
dm = re.split(r"\s*[-–]\s*", base, maxsplit=1)
|
||||
if len(dm) == 2 and dm[1].strip():
|
||||
color = "" # ambiguous; treat the left token as an animal name
|
||||
animals = [dm[0].strip().strip("_")] if dm[0].strip() else []
|
||||
buyer = dm[1].strip().strip("_").strip()
|
||||
else:
|
||||
tok = base.strip().strip("_").strip()
|
||||
# comma list like "Speedy,Agouti" → first is the name
|
||||
if tok:
|
||||
animals = [tok.split(",")[0].strip()]
|
||||
|
||||
# Clean buyer: drop trailing numbering "2", file-version noise
|
||||
buyer = re.sub(r"\s*\(?\d+\)?$", "", buyer).strip() if buyer else ""
|
||||
return {"color": color, "animals": animals, "buyer": buyer}
|
||||
|
||||
|
||||
# Seller is always the breeder Drazena Rimac (the cattery owner); never a buyer.
|
||||
_SELLER_RX = re.compile(r"^(?:(?:Herr|Frau|Familie)\s+)?Drazena\b", re.IGNORECASE)
|
||||
_ADDR_STOP = r"Stra\w+e|Wohnort|Fon|E-?Mail|Handy|Festnetz|Telefon|Homepage|Zuchtname|Facebook"
|
||||
|
||||
|
||||
_BUYER_REJECT_RX = re.compile(
|
||||
r"^(?:Stra\w+e|Wohnort|Fon|E-?Mail|Handy|Festnetz|Telefon|Homepage|"
|
||||
r"Zuchtname|Facebook|Name)\s*:?\s*$|^[\d\s/]+$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _clean_person(name: str) -> str:
|
||||
name = re.sub(r"\s+", " ", name).strip().strip(",").strip()
|
||||
# Reject label leakage / non-person artefacts (e.g. "Straße:", phone runs).
|
||||
if _BUYER_REJECT_RX.match(name):
|
||||
return ""
|
||||
return name
|
||||
|
||||
|
||||
def _extract_buyer(text: str, paras: list[str]) -> str:
|
||||
"""Buyer (Abnehmer/Empfänger) name from the body.
|
||||
|
||||
Two document layouts exist:
|
||||
A) Side-by-side columns (two table cells): "Name:<seller>" and
|
||||
"Name:<buyer>" appear as two separate runs in the full text.
|
||||
B) Stacked blocks: a "Empfänger/Abnehmer:" header precedes the buyer's
|
||||
"Name:".
|
||||
Strategy: collect every "Name:" value that looks like a person and is not
|
||||
the seller (Drazena Rimac). The buyer is such a value.
|
||||
"""
|
||||
# Layout B: explicit Abnehmer/Empfänger header followed by Name:.
|
||||
bm = re.search(
|
||||
r"(?:Empf\w+nger|Abnehmer)[^:]*:\s*(?:[^:]*?\s)?Name\s*:\s*"
|
||||
rf"(.+?)(?=\s*(?:{_ADDR_STOP})\s*:|$)",
|
||||
text, re.IGNORECASE,
|
||||
)
|
||||
if bm:
|
||||
cand = _clean_person(bm.group(1))
|
||||
if cand and not _SELLER_RX.match(cand):
|
||||
return cand
|
||||
|
||||
# Layout A / fallback: scan all Name: values; pick the first non-seller one.
|
||||
for m in re.finditer(
|
||||
rf"Name\s*:\s*(.+?)(?=\s*(?:{_ADDR_STOP})\s*:|\s*Name\s*:|$)",
|
||||
text, re.IGNORECASE,
|
||||
):
|
||||
cand = _clean_person(m.group(1))
|
||||
if not cand or _SELLER_RX.match(cand):
|
||||
continue
|
||||
# Skip the animal block: animal Name is immediately followed by
|
||||
# Geburtsdatum/Geschlecht/Farbschlag.
|
||||
tail = text[m.end():m.end() + 40]
|
||||
if re.match(r"\s*(?:Geburtsdatum|Geschlecht|Farbschlag)\s*:", tail, re.IGNORECASE):
|
||||
continue
|
||||
return cand
|
||||
return ""
|
||||
|
||||
|
||||
def parse_contract(docx_path: str) -> dict | None:
|
||||
"""Parse one .docx. Returns a contract record or None if not a contract."""
|
||||
fname = os.path.basename(docx_path)
|
||||
try:
|
||||
text = _full_text(docx_path)
|
||||
paras = _paragraphs(docx_path)
|
||||
except (zipfile.BadZipFile, KeyError, OSError):
|
||||
return None # unreadable / not a real docx
|
||||
|
||||
if any(m in text for m in NON_CONTRACT_MARKERS):
|
||||
return None # Abstammungsnachweis / Geburtsurkunde — not a sale contract
|
||||
|
||||
fn = parse_filename(fname)
|
||||
|
||||
# --- animal block (body) ---
|
||||
body_name = _label_value(text, r"Name")
|
||||
# The first "Name:" in the body could be the seller's. The animal's name
|
||||
# appears under "Tierdaten:". Prefer the Name that directly precedes
|
||||
# Geburtsdatum/Geschlecht (the animal block).
|
||||
animal_name = ""
|
||||
# The animal block is headed by "Tierdaten:" (when present) and its Name is
|
||||
# the LAST "Name:" before "Geburtsdatum:". Anchor on Tierdaten if present,
|
||||
# then take the closest Name: to Geburtsdatum.
|
||||
scope = text
|
||||
ti = re.search(r"Tierdaten\s*:", text, re.IGNORECASE)
|
||||
if ti:
|
||||
scope = text[ti.end():]
|
||||
am = re.search(
|
||||
r"Name\s*:\s*(.*?)\s*Geburtsdatum\s*:",
|
||||
scope, re.IGNORECASE,
|
||||
)
|
||||
if am:
|
||||
cand = am.group(1).strip()
|
||||
# Greedy guard: if it still spans multiple labels, keep only the tail
|
||||
# after the last embedded "Name:".
|
||||
if "Name:" in cand or re.search(r"Name\s*:", cand):
|
||||
cand = re.split(r"Name\s*:", cand)[-1].strip()
|
||||
# Drop any leading address-block leftovers.
|
||||
cand = re.split(rf"\s*(?:{_ADDR_STOP})\s*:", cand)[-1].strip()
|
||||
animal_name = cand
|
||||
# Body animal name is often blank (the call-name lives in the filename).
|
||||
if not animal_name or len(animal_name) > 40:
|
||||
animal_name = ""
|
||||
|
||||
dob = _norm_date(_label_value(text, r"Geburtsdatum"))
|
||||
gender_raw = _label_value(text, r"Geschlecht").lower()
|
||||
if gender_raw.startswith("m"):
|
||||
gender = "Male"
|
||||
elif gender_raw.startswith("w"):
|
||||
gender = "Female"
|
||||
else:
|
||||
gender = ""
|
||||
|
||||
color = _label_value(text, r"Farbschlag") or _label_value(text, r"Farbe")
|
||||
color = color.strip()
|
||||
if not color and fn["color"]:
|
||||
color = fn["color"]
|
||||
|
||||
handover = _norm_date(
|
||||
_label_value(text, r"Abgabedatum") or _label_value(text, r"Abgabe\s*am")
|
||||
)
|
||||
|
||||
price = ""
|
||||
for lab in (r"Gesamtpreis", r"Schutzgeb\w+", r"Kaufpreis", r"Preis"):
|
||||
raw = _label_value(text, lab)
|
||||
if raw:
|
||||
price = _clean_money(raw)
|
||||
if price:
|
||||
break
|
||||
|
||||
# contract date: trailing "Ort, [den ]DD.MM.YYYY" near signature line
|
||||
contract_date = ""
|
||||
cm = re.findall(r"[A-Za-zÄÖÜäöü.\- ]+,\s*(?:den\s*)?(\d{1,2}\.\d{1,2}\.\d{2,4})",
|
||||
text)
|
||||
if cm:
|
||||
contract_date = _norm_date(cm[-1])
|
||||
|
||||
buyer = _extract_buyer(text, paras)
|
||||
if not buyer and fn["buyer"]:
|
||||
buyer = fn["buyer"]
|
||||
|
||||
# Animal call-names: prefer filename (the call-names the breeder filed by),
|
||||
# fall back to body animal name.
|
||||
animals = list(fn["animals"])
|
||||
if not animals and animal_name:
|
||||
animals = [animal_name]
|
||||
|
||||
# Reject if we have neither a buyer nor any animal name — useless record.
|
||||
if not buyer and not animals:
|
||||
return None
|
||||
|
||||
return {
|
||||
"sourceFile": fname,
|
||||
"buyer": buyer,
|
||||
"animals": animals,
|
||||
"animalNameBody": animal_name,
|
||||
"color": color,
|
||||
"gender": gender,
|
||||
"dob": dob,
|
||||
"handoverDate": handover,
|
||||
"contractDate": contract_date,
|
||||
"price": price,
|
||||
}
|
||||
|
||||
|
||||
def scan(dir_path: str, limit: int | None = None):
|
||||
"""Walk the share, parse every .docx. Returns (records, stats)."""
|
||||
records = []
|
||||
stats = {
|
||||
"files_seen": 0, "template_skipped": 0, "non_contract_skipped": 0,
|
||||
"unreadable": 0, "parsed": 0, "unparseable": 0,
|
||||
"with_buyer": 0, "with_handover": 0, "with_price": 0,
|
||||
"with_color": 0, "with_dob": 0, "with_animals": 0,
|
||||
}
|
||||
for root, _, files in os.walk(dir_path):
|
||||
for f in sorted(files):
|
||||
if not f.lower().endswith(".docx"):
|
||||
continue
|
||||
stats["files_seen"] += 1
|
||||
low = f.lower()
|
||||
if any(m in low for m in TEMPLATE_MARKERS):
|
||||
stats["template_skipped"] += 1
|
||||
continue
|
||||
path = os.path.join(root, f)
|
||||
try:
|
||||
rec = parse_contract(path)
|
||||
except Exception: # never let one bad file kill the run
|
||||
rec = None
|
||||
if rec is None:
|
||||
# Distinguish unreadable vs non-contract vs genuinely unparseable
|
||||
try:
|
||||
_ = _full_text(path)
|
||||
txt = _
|
||||
if any(m in txt for m in NON_CONTRACT_MARKERS):
|
||||
stats["non_contract_skipped"] += 1
|
||||
else:
|
||||
stats["unparseable"] += 1
|
||||
except Exception:
|
||||
stats["unreadable"] += 1
|
||||
continue
|
||||
stats["parsed"] += 1
|
||||
if rec["buyer"]:
|
||||
stats["with_buyer"] += 1
|
||||
if rec["handoverDate"]:
|
||||
stats["with_handover"] += 1
|
||||
if rec["price"]:
|
||||
stats["with_price"] += 1
|
||||
if rec["color"]:
|
||||
stats["with_color"] += 1
|
||||
if rec["dob"]:
|
||||
stats["with_dob"] += 1
|
||||
if rec["animals"]:
|
||||
stats["with_animals"] += 1
|
||||
records.append(rec)
|
||||
if limit and len(records) >= limit:
|
||||
return records, stats
|
||||
return records, stats
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
ap = argparse.ArgumentParser(description="Abgabevertrag-Extraktor")
|
||||
ap.add_argument("--dir", default=DEFAULT_DIR, help="Verträge-Ordner")
|
||||
ap.add_argument("--limit", type=int, default=None,
|
||||
help="Nur die ersten N Verträge (Stichprobe)")
|
||||
args = ap.parse_args()
|
||||
|
||||
if not os.path.isdir(args.dir):
|
||||
print(f"Fehler: Ordner nicht gefunden: {args.dir}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
os.makedirs(OUT, exist_ok=True)
|
||||
print(f"Scanne: {args.dir}")
|
||||
records, stats = scan(args.dir, args.limit)
|
||||
|
||||
out_path = os.path.join(OUT, "contracts.json")
|
||||
with open(out_path, "w", encoding="utf-8") as f:
|
||||
json.dump(records, f, ensure_ascii=False, indent=2)
|
||||
|
||||
print(f"Dateien gesehen: {stats['files_seen']}")
|
||||
print(f" Vorlage übersprungen: {stats['template_skipped']}")
|
||||
print(f" Nicht-Vertrag (AN): {stats['non_contract_skipped']}")
|
||||
print(f" unlesbar (kein ZIP): {stats['unreadable']}")
|
||||
print(f" unparsbar: {stats['unparseable']}")
|
||||
print(f"Verträge geparst: {stats['parsed']}")
|
||||
print(f" mit Käufer: {stats['with_buyer']}")
|
||||
print(f" mit Tier(en): {stats['with_animals']}")
|
||||
print(f" mit Geburtsdatum: {stats['with_dob']}")
|
||||
print(f" mit Abgabedatum: {stats['with_handover']}")
|
||||
print(f" mit Preis: {stats['with_price']}")
|
||||
print(f" mit Farbschlag: {stats['with_color']}")
|
||||
print(f"Ausgabe: {out_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -713,8 +713,63 @@ def pick_parent_ref(parent_refs, role, child_dob, avoid_name=None, gender_of=Non
|
||||
return role_refs[order[0]]
|
||||
|
||||
|
||||
def explain_pick_rejections(parent_refs, role, child_dob, chosen, avoid_name=None,
|
||||
gender_of=None):
|
||||
"""Explain why other refs for `role` lost to `chosen` in pick_parent_ref.
|
||||
|
||||
Returns a list of discard dicts (for _format_discard) — one per distinct
|
||||
rejected candidate name that was beaten for a clear reason (wrong sex, age-
|
||||
impossible, or duplicate of the other parent). Mirrors pick_parent_ref's
|
||||
ranking so the history can explain the same decision it made.
|
||||
"""
|
||||
role_refs = [p for p in parent_refs if p.get("roleGuess") == role]
|
||||
if not role_refs or chosen is None:
|
||||
return []
|
||||
avoid = normalize_name(avoid_name) if avoid_name else None
|
||||
expected = "male" if role == "father" else "female"
|
||||
role_de = "Vaterrolle" if role == "father" else "Mutterrolle"
|
||||
cand_de = "Vater-Kandidat" if role == "father" else "Mutter-Kandidat"
|
||||
chosen_name = chosen.get("name")
|
||||
repl_disp = chosen_name
|
||||
if chosen.get("dob"):
|
||||
repl_disp = f"{chosen_name} (*{_de_date(parse_date(chosen.get('dob'))) or chosen.get('dob')})"
|
||||
|
||||
out = []
|
||||
seen = set()
|
||||
for p in role_refs:
|
||||
name = p.get("name")
|
||||
if not name or normalize_name(name) == normalize_name(chosen_name or ""):
|
||||
continue
|
||||
key = normalize_name(name)
|
||||
if key in seen:
|
||||
continue
|
||||
g = gender_of(name) if gender_of else None
|
||||
dob = p.get("dob")
|
||||
reason = None
|
||||
if avoid is not None and key == avoid:
|
||||
reason = "bereits als anderer Elternteil gewählt"
|
||||
elif g in ("male", "female") and g != expected:
|
||||
reason = f"falsches Geschlecht für die {role_de}"
|
||||
elif dob and not parent_age_plausible(dob, child_dob):
|
||||
reason = "unplausibles Alter für diesen Wurf"
|
||||
if reason is None:
|
||||
continue
|
||||
seen.add(key)
|
||||
disp = name
|
||||
if dob:
|
||||
disp = f"{name} (*{_de_date(parse_date(dob)) or dob})"
|
||||
out.append({
|
||||
"label": cand_de,
|
||||
"value": f"„{disp}“",
|
||||
"reason": reason,
|
||||
"replacement": f"„{repl_disp}“",
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def _build_gerbil_history(records, best_g, field_source, parent_method=None,
|
||||
any_decision=False, any_conflict=False, conflict_notes=None):
|
||||
any_decision=False, any_conflict=False, conflict_notes=None,
|
||||
field_discards=None, parent_discards=None):
|
||||
"""Build an ordered, file-attributed German history for a resolved gerbil.
|
||||
|
||||
Reads like a chronological log:
|
||||
@@ -773,6 +828,13 @@ def _build_gerbil_history(records, best_g, field_source, parent_method=None,
|
||||
else:
|
||||
history.append("In weiterem Datensatz gefunden → Datensätze zusammengeführt.")
|
||||
|
||||
# Discarded field values from majority-vote conflict resolution: the LOSING
|
||||
# values, their file, and what won instead.
|
||||
for d in (field_discards or []):
|
||||
line = _format_discard(d)
|
||||
if line not in history:
|
||||
history.append(line)
|
||||
|
||||
# Parent derivation.
|
||||
if parent_method:
|
||||
method_label = {
|
||||
@@ -791,6 +853,14 @@ def _build_gerbil_history(records, best_g, field_source, parent_method=None,
|
||||
else:
|
||||
history.append(f"Eltern über {method_label} erkannt.")
|
||||
|
||||
# Discarded parent candidates / links (pick_parent_ref rejections, role
|
||||
# normalization drops, parent-age sanity check). Threaded in after the merge
|
||||
# via the gerbil's _discarded list so they read in chronological order.
|
||||
for d in (parent_discards or []):
|
||||
line = _format_discard(d)
|
||||
if line not in history:
|
||||
history.append(line)
|
||||
|
||||
# Manual decisions / conflicts.
|
||||
if any_decision:
|
||||
history.append("Zuordnung per manueller Entscheidung getroffen.")
|
||||
@@ -821,6 +891,221 @@ def _de_gender(g):
|
||||
return {"male": "männlich", "female": "weiblich"}.get(g, g)
|
||||
|
||||
|
||||
# Leading marker that visually flags a discard ("data was thrown away") line in
|
||||
# the history timeline. The frontend keys discard styling off this marker.
|
||||
DISCARD_MARK = "⚠ "
|
||||
|
||||
|
||||
def _format_discard(d):
|
||||
"""Render one discard record into a German history line.
|
||||
|
||||
A discard record is a dict describing a value/candidate the pipeline threw
|
||||
away. Recognised keys:
|
||||
• text — a fully pre-formatted line (used verbatim, marker added)
|
||||
• label — German field label (e.g. „Geburtsdatum“)
|
||||
• value — the discarded value (already display-formatted)
|
||||
• file — source file the discarded value came from (attributed)
|
||||
• reason — why it was dropped (e.g. „abweichend“, „unplausibel …“)
|
||||
• replacement — what was used instead (already display-formatted)
|
||||
• repl_file — source file the replacement came from
|
||||
Produces lines like:
|
||||
„⚠ Geburtsdatum 14.06.2015 aus ‚A.xlsx‘ verworfen — abweichend;
|
||||
14.06.2017 aus ‚B.xlsx‘ verwendet (Mehrheit).“
|
||||
Generic across entity types so litters/contacts can reuse it.
|
||||
"""
|
||||
if d.get("text"):
|
||||
return DISCARD_MARK + d["text"]
|
||||
parts = []
|
||||
label = d.get("label")
|
||||
value = d.get("value")
|
||||
if label and value is not None:
|
||||
parts.append(f"{label} {value}")
|
||||
elif label:
|
||||
parts.append(str(label))
|
||||
elif value is not None:
|
||||
parts.append(str(value))
|
||||
head = " ".join(parts) if parts else "Wert"
|
||||
if d.get("file"):
|
||||
head += f" aus {_quote_file(d['file'])}"
|
||||
line = f"{head} verworfen"
|
||||
if d.get("reason"):
|
||||
line += f" — {d['reason']}"
|
||||
repl = d.get("replacement")
|
||||
if repl is not None and repl != "":
|
||||
instead = str(repl)
|
||||
if d.get("repl_file"):
|
||||
instead += f" aus {_quote_file(d['repl_file'])}"
|
||||
suffix = d.get("replacement_note")
|
||||
line += f"; {instead} verwendet"
|
||||
if suffix:
|
||||
line += f" ({suffix})"
|
||||
elif d.get("no_replacement"):
|
||||
line += "; kein Ersatz"
|
||||
return DISCARD_MARK + line + "."
|
||||
|
||||
|
||||
def _append_history(prov_json, line):
|
||||
"""Append one history line to an existing Provenance JSON string and add the
|
||||
contract source file. Returns the updated JSON string."""
|
||||
try:
|
||||
prov = json.loads(prov_json) if prov_json else {}
|
||||
except (ValueError, TypeError):
|
||||
prov = {}
|
||||
prov.setdefault("history", [])
|
||||
if line not in prov["history"]:
|
||||
prov["history"].append(line)
|
||||
return json.dumps(prov, ensure_ascii=False)
|
||||
|
||||
|
||||
def enrich_from_contracts(contracts, resolved_gerbils, contact_by_norm_name,
|
||||
contact_id_map):
|
||||
"""Conservatively fold Abgabevertrag data into the resolved gerbils.
|
||||
|
||||
For every parsed contract we (a) ensure the buyer exists as a (receiver)
|
||||
contact, reusing the existing contact dedup/normalisation, and (b) try to
|
||||
match each animal call-name to exactly one resolved gerbil that the breeder
|
||||
bred ("…Chaoten"). On a confident match we set ReceiverContactId /
|
||||
GoHomeDate / Status=GivenAway — but only where not already set differently —
|
||||
and add a provenance history line. Ambiguous or absent matches are logged,
|
||||
never guessed.
|
||||
|
||||
Returns a stats dict. Mutates resolved_gerbils + contact_by_norm_name in
|
||||
place. New buyer contacts are appended via contact_by_norm_name so the later
|
||||
IsReceiver-flag pass picks them up automatically.
|
||||
"""
|
||||
stats = {
|
||||
"contracts": len(contracts), "buyers_created": 0, "buyers_existing": 0,
|
||||
"matched": 0, "ambiguous_skipped": 0, "no_match_skipped": 0,
|
||||
"receiver_set": 0, "gohome_set": 0, "status_givenaway": 0,
|
||||
"conflicts": 0,
|
||||
}
|
||||
if not contracts:
|
||||
return stats
|
||||
|
||||
# Index breeder-owned gerbils by dedup name key. Contracts only ever sell
|
||||
# animals the breeder bred, so restrict candidates to her own stock to avoid
|
||||
# colliding with same-named foreign-bred animals.
|
||||
def is_own(g):
|
||||
ob = (g.get("OriginBreeder") or "").lower()
|
||||
return ("chaoten" in ob) or (g.get("OriginBreeder") is None)
|
||||
|
||||
index = {}
|
||||
for g in resolved_gerbils:
|
||||
if not is_own(g):
|
||||
continue
|
||||
key = get_dedup_name_key(get_call_name(g.get("Name", "")))
|
||||
if key:
|
||||
index.setdefault(key, []).append(g)
|
||||
|
||||
def year_of(iso):
|
||||
return iso[:4] if iso else None
|
||||
|
||||
# Reject buyer values that are obviously label leakage / non-person noise
|
||||
# (defends against any stale contracts.json produced before the parser fix).
|
||||
_buyer_junk = re.compile(
|
||||
r"^(?:stra\w+e|wohnort|fon|e-?mail|handy|festnetz|telefon|homepage|"
|
||||
r"zuchtname|facebook|name)\s*:?\s*$|^[\d\s/]+$", re.IGNORECASE)
|
||||
|
||||
for c in contracts:
|
||||
fname = c.get("sourceFile", "")
|
||||
buyer_raw = (c.get("buyer") or "").strip()
|
||||
if buyer_raw and _buyer_junk.match(buyer_raw):
|
||||
buyer_raw = ""
|
||||
|
||||
# --- buyer contact (reuse curated normalisation/dedup) ---
|
||||
buyer_global_id = None
|
||||
if buyer_raw:
|
||||
canon, keep = get_normalized_contact_name(buyer_raw)
|
||||
if keep and canon:
|
||||
norm = normalize_name(canon)
|
||||
if norm in contact_by_norm_name:
|
||||
gc = contact_by_norm_name[norm]
|
||||
gc.setdefault("_source_files", set()).add(fname)
|
||||
gc["_merged_count"] = gc.get("_merged_count", 1) + 1
|
||||
stats["buyers_existing"] += 1
|
||||
else:
|
||||
gid = generate_guid(f"contact-{norm}")
|
||||
contact_by_norm_name[norm] = {
|
||||
"Id": gid, "Name": canon, "Email": None, "Phone": None,
|
||||
"Address": None, "Notes": None, "NameSuffix": None,
|
||||
"_source_files": {fname}, "_merged_count": 1,
|
||||
}
|
||||
stats["buyers_created"] += 1
|
||||
buyer_global_id = contact_by_norm_name[norm]["Id"]
|
||||
|
||||
# --- match each animal call-name to a resolved gerbil ---
|
||||
handover = parse_date(c.get("handoverDate"))
|
||||
c_year = year_of(parse_date(c.get("dob"))) if c.get("dob") else None
|
||||
c_color = (c.get("color") or "").strip().lower()
|
||||
|
||||
for call in (c.get("animals") or []):
|
||||
key = get_dedup_name_key(get_call_name(call))
|
||||
if not key:
|
||||
continue
|
||||
cands = index.get(key, [])
|
||||
if not cands:
|
||||
stats["no_match_skipped"] += 1
|
||||
continue
|
||||
|
||||
# Corroborate when more than one candidate shares the call-name.
|
||||
chosen = None
|
||||
if len(cands) == 1:
|
||||
chosen = cands[0]
|
||||
else:
|
||||
scored = []
|
||||
for g in cands:
|
||||
score = 0
|
||||
g_year = year_of(g.get("DateOfBirth"))
|
||||
if c_year and g_year and c_year == g_year:
|
||||
score += 2
|
||||
if c_color and g.get("ColorVarietyId"):
|
||||
# color match is corroboration; we don't have the name
|
||||
# here, so only DOB drives disambiguation strongly.
|
||||
pass
|
||||
scored.append((score, g))
|
||||
scored.sort(key=lambda t: t[0], reverse=True)
|
||||
if scored[0][0] >= 2 and (len(scored) == 1 or scored[0][0] > scored[1][0]):
|
||||
chosen = scored[0][1]
|
||||
else:
|
||||
stats["ambiguous_skipped"] += 1
|
||||
continue
|
||||
|
||||
stats["matched"] += 1
|
||||
|
||||
# --- set receiver, only if not already set differently ---
|
||||
if buyer_global_id:
|
||||
cur = chosen.get("ReceiverContactId")
|
||||
if not cur:
|
||||
chosen["ReceiverContactId"] = buyer_global_id
|
||||
stats["receiver_set"] += 1
|
||||
chosen["Provenance"] = _append_history(
|
||||
chosen.get("Provenance"),
|
||||
f"Abgabe an „{buyer_raw}“ aus Vertrag {_quote_file(fname)} übernommen.")
|
||||
elif cur != buyer_global_id:
|
||||
stats["conflicts"] += 1
|
||||
chosen["Provenance"] = _append_history(
|
||||
chosen.get("Provenance"),
|
||||
f"Vertrag {_quote_file(fname)} nennt anderen Abnehmer „{buyer_raw}“ "
|
||||
f"— bestehende Zuordnung beibehalten.")
|
||||
|
||||
# --- set go-home date, only if empty ---
|
||||
if handover and not chosen.get("GoHomeDate"):
|
||||
chosen["GoHomeDate"] = handover
|
||||
stats["gohome_set"] += 1
|
||||
chosen["Provenance"] = _append_history(
|
||||
chosen.get("Provenance"),
|
||||
f"Abgabedatum {handover} aus Vertrag {_quote_file(fname)} übernommen.")
|
||||
|
||||
# --- status: derive GivenAway if we set a receiver and it isn't
|
||||
# already a stronger state (Deceased). ---
|
||||
if chosen.get("ReceiverContactId") and chosen.get("Status") not in (
|
||||
"Deceased", "GivenAway"):
|
||||
chosen["Status"] = "GivenAway"
|
||||
stats["status_givenaway"] += 1
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def main():
|
||||
print("Loading color variety seeds...")
|
||||
variety_map = {}
|
||||
@@ -864,6 +1149,14 @@ def main():
|
||||
else:
|
||||
print(f"Warning: Seeds path not found at {SEEDS_PATH}")
|
||||
|
||||
# Reverse map (variety GUID → human name) for discard/replacement history
|
||||
# lines that need to show a colour value rather than a raw GUID. First wins
|
||||
# so we keep the canonical lower-case catalog name.
|
||||
variety_id_to_name = {}
|
||||
for name_lower, vid in variety_map.items():
|
||||
if vid not in variety_id_to_name:
|
||||
variety_id_to_name[vid] = name_lower
|
||||
|
||||
md_files = sorted([f for f in os.listdir(DIR_PATH) if f.lower().endswith('.md')])
|
||||
print(f"Found {len(md_files)} markdown files in {DIR_PATH}.")
|
||||
|
||||
@@ -1045,6 +1338,17 @@ def main():
|
||||
else:
|
||||
print(f"Warning: docx_litters.json not found at {docx_litters_path}")
|
||||
|
||||
# Abgabevertrag-Datensätze (aus extract_contracts.py). Optional: wenn die
|
||||
# Datei fehlt, läuft der Import ohne Vertrags-Anreicherung normal weiter.
|
||||
contracts = []
|
||||
contracts_path = os.path.join(OUTPUT_DIR, "contracts.json")
|
||||
if os.path.exists(contracts_path):
|
||||
with open(contracts_path, "r", encoding="utf-8") as f:
|
||||
contracts = json.load(f)
|
||||
print(f"Loaded {len(contracts)} Abgabeverträge.")
|
||||
else:
|
||||
print(f"Info: contracts.json not found at {contracts_path} (Vertrags-Anreicherung übersprungen)")
|
||||
|
||||
# Extract docx buyer contacts and add to raw_contacts
|
||||
for da in docx_animals:
|
||||
o_name = (da.get("owner") or "").strip()
|
||||
@@ -1187,7 +1491,20 @@ def main():
|
||||
mother_ref = pick_parent_ref(parent_refs, "mother", child_dob_raw,
|
||||
avoid_name=father_ref.get("name") if father_ref else None,
|
||||
gender_of=gender_of_name)
|
||||
|
||||
|
||||
# Record rejected parent-ref candidates (wrong sex / age-impossible /
|
||||
# duplicate of the other role) so this animal's gerbil history can
|
||||
# explain which Stammbaum positions were discarded and what won instead.
|
||||
pick_discards = []
|
||||
pick_discards += explain_pick_rejections(parent_refs, "father", child_dob_raw,
|
||||
father_ref, gender_of=gender_of_name)
|
||||
pick_discards += explain_pick_rejections(
|
||||
parent_refs, "mother", child_dob_raw, mother_ref,
|
||||
avoid_name=father_ref.get("name") if father_ref else None,
|
||||
gender_of=gender_of_name)
|
||||
if pick_discards:
|
||||
a["_pick_discards"] = pick_discards
|
||||
|
||||
a["_mapped_litter_scoped_id"] = None
|
||||
if father_ref and mother_ref:
|
||||
f_name = get_normalized_gerbil_name(father_ref.get("name"))
|
||||
@@ -1623,6 +1940,23 @@ def main():
|
||||
"_old_id": rg.get("Id") or rg.get("id")
|
||||
})
|
||||
|
||||
# Build the discard list for one stammbaum animal: rejected parent-ref
|
||||
# candidates (pick_parent_ref) plus a manual DOB-remap decision, if any.
|
||||
def _stammbaum_discards(a):
|
||||
discards = list(a.get("_pick_discards", []))
|
||||
remap = a.get("dobRemap")
|
||||
if remap and remap.get("original") and remap.get("corrected"):
|
||||
orig = _de_date(parse_date(remap["original"])) or remap["original"]
|
||||
corr = _de_date(parse_date(remap["corrected"])) or remap["corrected"]
|
||||
if orig != corr:
|
||||
discards.append({
|
||||
"label": PROV_FIELD_LABELS["DateOfBirth"],
|
||||
"value": orig,
|
||||
"reason": "per manueller Entscheidung korrigiert",
|
||||
"replacement": corr,
|
||||
})
|
||||
return discards
|
||||
|
||||
# Map and append stammbaum animals to all_processed_gerbils
|
||||
for a in stammbaum_only_animals:
|
||||
a_id = a["id"]
|
||||
@@ -1731,7 +2065,10 @@ def main():
|
||||
"_eff_dob": dob_val or "2010-01-01",
|
||||
"_birth_date": dob_val,
|
||||
"_filename": a.get("sourceFiles", ["Stammbaum"])[0],
|
||||
"_old_id": a_id
|
||||
"_old_id": a_id,
|
||||
# Rejected Stammbaum parent-ref candidates for this animal (filled by
|
||||
# pick_parent_ref above) — surfaced in this gerbil's discard history.
|
||||
"_discarded": _stammbaum_discards(a),
|
||||
})
|
||||
|
||||
# Map and append docx animals to all_processed_gerbils
|
||||
@@ -1885,7 +2222,8 @@ def main():
|
||||
|
||||
return True
|
||||
|
||||
def build_provenance(records, best_g, extra_notes=None, field_source=None):
|
||||
def build_provenance(records, best_g, extra_notes=None, field_source=None,
|
||||
field_discards=None):
|
||||
"""Aggregate data-provenance across every raw record merged into one
|
||||
resolved gerbil. Returns a JSON string (stored on the Gerbil entity as a
|
||||
nullable text column) so the Rennmausakte can show where the entry came
|
||||
@@ -1937,6 +2275,8 @@ def main():
|
||||
any_decision=any_decision,
|
||||
any_conflict=any_conflict,
|
||||
conflict_notes=extra_notes or [],
|
||||
field_discards=field_discards,
|
||||
parent_discards=best_g.get("_discarded"),
|
||||
)
|
||||
|
||||
return build_entity_provenance(
|
||||
@@ -1971,7 +2311,9 @@ def main():
|
||||
if is_placeholder:
|
||||
# Placeholders: do NOT merge, keep all separate
|
||||
for g in group:
|
||||
g["Provenance"] = build_provenance([g], g)
|
||||
# Provenance is built in a final pass (after parent resolution),
|
||||
# so parent-link discards land in the right gerbil's history.
|
||||
g["_prov_args"] = ([g], g, None, None, None)
|
||||
resolved_gerbils.append(g)
|
||||
gerbil_id_map[g["Id"]] = g["Id"]
|
||||
continue
|
||||
@@ -1992,7 +2334,7 @@ def main():
|
||||
for sub in sub_groups:
|
||||
if len(sub) == 1:
|
||||
g = sub[0]
|
||||
g["Provenance"] = build_provenance([g], g)
|
||||
g["_prov_args"] = ([g], g, None, None, None)
|
||||
resolved_gerbils.append(g)
|
||||
gerbil_id_map[g["Id"]] = g["Id"]
|
||||
continue
|
||||
@@ -2019,6 +2361,17 @@ def main():
|
||||
if best_g["Notes"]:
|
||||
merged_notes.append(best_g["Notes"])
|
||||
|
||||
# Carry over discard records (rejected parent-refs / dob remaps) from
|
||||
# every merged record so none are lost when the primary changes.
|
||||
merged_discards = list(best_g.get("_discarded") or [])
|
||||
for g in sub:
|
||||
if g is best_g:
|
||||
continue
|
||||
for d in (g.get("_discarded") or []):
|
||||
if d not in merged_discards:
|
||||
merged_discards.append(d)
|
||||
best_g["_discarded"] = merged_discards
|
||||
|
||||
# Merge photos
|
||||
merged_photos = list(best_g.get("_photos", []))
|
||||
|
||||
@@ -2094,8 +2447,21 @@ def main():
|
||||
if not any(kw in g["Notes"].lower() for kw in ["parent listed", "mutter von", "vater von", "dam of", "sire of"]):
|
||||
merged_notes.append(g["Notes"])
|
||||
|
||||
# Display formatter per field for discard/replacement lines.
|
||||
def _disp(field, val):
|
||||
if val is None or val == "" or val == "unknown":
|
||||
return None
|
||||
if field in ("DateOfBirth", "DateOfDeath"):
|
||||
return _de_date(val)
|
||||
if field == "Gender":
|
||||
return _de_gender(val)
|
||||
if field == "ColorVarietyId":
|
||||
return variety_id_to_name.get(val, "Farbschlag")
|
||||
return str(val)
|
||||
|
||||
# Reconcile fields based on number of source files supporting them
|
||||
conflict_notes = []
|
||||
field_discards = []
|
||||
for field in ["DateOfBirth", "DateOfDeath", "Gender", "Genotype", "ColorVarietyId"]:
|
||||
votes = {}
|
||||
for g in sub:
|
||||
@@ -2118,6 +2484,25 @@ def main():
|
||||
winner = next((g for g in sub if g.get(field) == best_val), None)
|
||||
if winner is not None:
|
||||
field_source[field] = winner
|
||||
# Record each LOSING value: which file it came from, that it
|
||||
# was discarded as differing, and that the majority value (and
|
||||
# its file) was used instead.
|
||||
if len(votes) > 1:
|
||||
repl_disp = _disp(field, best_val)
|
||||
repl_file = _primary_file_of(winner) if winner is not None else None
|
||||
for lose_val in votes:
|
||||
if lose_val == best_val:
|
||||
continue
|
||||
loser = next((g for g in sub if g.get(field) == lose_val), None)
|
||||
field_discards.append({
|
||||
"label": PROV_FIELD_LABELS.get(field, field),
|
||||
"value": _disp(field, lose_val),
|
||||
"file": _primary_file_of(loser) if loser is not None else None,
|
||||
"reason": "abweichend",
|
||||
"replacement": repl_disp,
|
||||
"repl_file": repl_file,
|
||||
"replacement_note": "Mehrheit",
|
||||
})
|
||||
# Keep helper fields in sync if we changed DateOfBirth
|
||||
if field == "DateOfBirth":
|
||||
best_g["_birth_date"] = best_val
|
||||
@@ -2127,9 +2512,9 @@ def main():
|
||||
best_g["Notes"] = " | ".join(merged_notes)
|
||||
|
||||
best_g["_photos"] = merged_photos
|
||||
best_g["Provenance"] = build_provenance(
|
||||
sub, best_g, extra_notes=conflict_notes, field_source=field_source
|
||||
)
|
||||
# Defer provenance to a final pass so parent-link discards (added
|
||||
# later by the parent-age / role-normalization passes) are included.
|
||||
best_g["_prov_args"] = (sub, best_g, conflict_notes, field_source, field_discards)
|
||||
|
||||
# Print merge trace
|
||||
print(f"Deduplicated same-animal name '{best_g['Name']}': merged {len(sub)} entries across files: {', '.join(sources)}")
|
||||
@@ -2294,6 +2679,25 @@ def main():
|
||||
del l["_mother_name"]
|
||||
del l["_filename"]
|
||||
|
||||
# Children-of-litter lookup so a dropped parent link can be explained in the
|
||||
# offspring's Datenherkunft (the discard is most meaningful on the child).
|
||||
children_by_litter = {}
|
||||
for g in resolved_gerbils:
|
||||
lid = g.get("LitterId")
|
||||
if lid:
|
||||
children_by_litter.setdefault(lid, []).append(g)
|
||||
|
||||
def _add_child_discard(litter, discard):
|
||||
"""Attach a parent-link discard to every child gerbil of the litter."""
|
||||
for child in children_by_litter.get(litter["Id"], []):
|
||||
dl = child.setdefault("_discarded", [])
|
||||
if discard not in dl:
|
||||
dl.append(discard)
|
||||
|
||||
def _pname(pid):
|
||||
p = gerbil_by_id_final.get(pid)
|
||||
return p.get("Name") if p else None
|
||||
|
||||
# Role normalization: assign each resolved parent to the role matching its
|
||||
# gender, eliminate self-pairings (same animal in both roles), and never let
|
||||
# impossible duplicates survive (two males / two females). This corrects
|
||||
@@ -2301,9 +2705,23 @@ def main():
|
||||
# (one parent of "unknown" gender, or a self-paired litter).
|
||||
role_fixes = 0
|
||||
for l in resolved_litters:
|
||||
before = (l.get("FatherId"), l.get("MotherId"))
|
||||
father, mother = assign_parent_roles(l.get("FatherId"), l.get("MotherId"), _final_gender)
|
||||
if (l.get("FatherId"), l.get("MotherId")) != (father, mother):
|
||||
if before != (father, mother):
|
||||
role_fixes += 1
|
||||
after = {father, mother}
|
||||
# A parent id present before but gone after was dropped by role
|
||||
# normalization (self-pairing or two-of-the-same-sex). Explain it.
|
||||
for pid in before:
|
||||
if pid and pid not in after:
|
||||
pname = _pname(pid)
|
||||
if before[0] == before[1]:
|
||||
reason = "Selbstverpaarung — Tier kann nicht beide Elternteile sein"
|
||||
else:
|
||||
reason = "ein Wurf hat nur einen Vater und eine Mutter"
|
||||
_add_child_discard(l, {
|
||||
"text": f"Elternteil „{pname}“ verworfen — {reason}",
|
||||
})
|
||||
l["FatherId"] = father
|
||||
l["MotherId"] = mother
|
||||
if role_fixes:
|
||||
@@ -2323,6 +2741,23 @@ def main():
|
||||
p = gerbil_by_id_final.get(pid)
|
||||
if p and not parent_age_plausible(p.get("DateOfBirth"), ldate):
|
||||
age_drops.append((l.get("Name"), role, p.get("Name"), p.get("DateOfBirth"), ldate))
|
||||
# Explain the drop in each child's history: which parent, its DOB,
|
||||
# why (implausible age), and that no replacement was used.
|
||||
role_de = "Vater" if role == "FatherId" else "Mutter"
|
||||
pdob_disp = _de_date(p.get("DateOfBirth")) or "unbekannt"
|
||||
age_reason = "unplausibles Alter für diesen Wurf"
|
||||
pd = date_to_days(parse_date(p.get("DateOfBirth"))) if p.get("DateOfBirth") else None
|
||||
ld = date_to_days(parse_date(ldate)) if ldate else None
|
||||
if pd is not None and ld is not None:
|
||||
years = abs(ld - pd) / 365.25
|
||||
if ld - pd <= 0:
|
||||
age_reason = "unplausibel (nicht vor dem Kind geboren)"
|
||||
else:
|
||||
age_reason = f"unplausibel ({years:.0f} Jahre älter als das Kind)"
|
||||
_add_child_discard(l, {
|
||||
"text": (f"{role_de} „{p.get('Name')}“ (*{pdob_disp}) verworfen "
|
||||
f"— {age_reason}; kein Ersatz"),
|
||||
})
|
||||
l[role] = None
|
||||
if age_drops:
|
||||
print(f"Parent-age sanity check: dropped {len(age_drops)} implausible parent link(s):")
|
||||
@@ -2402,6 +2837,49 @@ def main():
|
||||
resolved_litters = deduped2
|
||||
litter_by_scoped_id = {l["Id"]: l for l in resolved_litters}
|
||||
|
||||
# Final gerbil-provenance pass: now that parent links are fully resolved and
|
||||
# all discards (majority-vote conflicts during dedup; rejected parent-refs;
|
||||
# role-normalization drops; parent-age drops) are attached to each gerbil's
|
||||
# _discarded list, render the provenance JSON with the discard history lines.
|
||||
n_discards = 0
|
||||
for g in resolved_gerbils:
|
||||
args = g.pop("_prov_args", None)
|
||||
if g.get("_discarded"):
|
||||
n_discards += len(g["_discarded"])
|
||||
if args is not None:
|
||||
records, best_g, conflict_notes, field_source, field_discards = args
|
||||
g["Provenance"] = build_provenance(
|
||||
records, best_g, extra_notes=conflict_notes,
|
||||
field_source=field_source, field_discards=field_discards,
|
||||
)
|
||||
else:
|
||||
g["Provenance"] = build_provenance([g], g)
|
||||
g.pop("_discarded", None)
|
||||
if n_discards:
|
||||
print(f"Datenherkunft: recorded {n_discards} discard line(s) across gerbils.")
|
||||
|
||||
# Abgabevertrag-Anreicherung: Käufer als Abnehmer-Kontakte anlegen und —
|
||||
# konservativ — auf eindeutig passende Tiere ReceiverContactId/GoHomeDate/
|
||||
# Status=GivenAway setzen (nur falls noch nicht gesetzt). Provenance-
|
||||
# Historie wird ergänzt. Neue Käuferkontakte landen in contact_by_norm_name
|
||||
# und werden danach automatisch als IsReceiver markiert.
|
||||
cstats = enrich_from_contracts(
|
||||
contracts, resolved_gerbils, contact_by_norm_name, contact_id_map)
|
||||
if contracts:
|
||||
print("Abgabeverträge: "
|
||||
f"{cstats['contracts']} geladen, {cstats['matched']} Tier-Treffer, "
|
||||
f"{cstats['ambiguous_skipped']} mehrdeutig übersprungen, "
|
||||
f"{cstats['no_match_skipped']} ohne Treffer.")
|
||||
print(" Kontakte: "
|
||||
f"{cstats['buyers_created']} neu, {cstats['buyers_existing']} bestehend.")
|
||||
print(" Gesetzt: "
|
||||
f"ReceiverContactId={cstats['receiver_set']}, "
|
||||
f"GoHomeDate={cstats['gohome_set']}, "
|
||||
f"Status=GivenAway={cstats['status_givenaway']}, "
|
||||
f"Konflikte={cstats['conflicts']}.")
|
||||
# Re-materialise contacts so freshly created buyer contacts are exported.
|
||||
resolved_contacts = list(contact_by_norm_name.values())
|
||||
|
||||
# Datenherkunft for litters: which source files contributed, whether this is
|
||||
# a Wurfchronik litter vs a Stammbaum-reconstructed ("virtual") litter, how
|
||||
# many raw records merged into it, plus human-readable notes. Accumulators
|
||||
|
||||
189
tools/import/test_extract_contracts.py
Normal file
189
tools/import/test_extract_contracts.py
Normal file
@@ -0,0 +1,189 @@
|
||||
"""Tests for extract_contracts.py — run: python test_extract_contracts.py
|
||||
|
||||
Zero third-party deps (mirrors test_extract_docx.py). Builds tiny in-memory
|
||||
.docx files (a zip with word/document.xml) so the tests run without the
|
||||
network share. Covers: filename parsing, run-splitting de-mangling, both body
|
||||
layouts (side-by-side / stacked), price/date normalisation, and skip rules.
|
||||
"""
|
||||
import io
|
||||
import os
|
||||
import sys
|
||||
import zipfile
|
||||
|
||||
try:
|
||||
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import extract_contracts as ec
|
||||
|
||||
failed = 0
|
||||
|
||||
|
||||
def check(name, cond):
|
||||
global failed
|
||||
print(("ok: " if cond else "FAIL: ") + name)
|
||||
if not cond:
|
||||
failed += 1
|
||||
|
||||
|
||||
def make_docx(paragraphs):
|
||||
"""paragraphs: list of lists of run-strings → bytes of a .docx zip."""
|
||||
body = []
|
||||
for runs in paragraphs:
|
||||
rs = "".join(f"<w:r><w:t>{r}</w:t></w:r>" for r in runs)
|
||||
body.append(f"<w:p>{rs}</w:p>")
|
||||
xml = ('<?xml version="1.0"?><w:document xmlns:w="x"><w:body>'
|
||||
+ "".join(body) + "</w:body></w:document>")
|
||||
buf = io.BytesIO()
|
||||
with zipfile.ZipFile(buf, "w") as z:
|
||||
z.writestr("word/document.xml", xml)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def write_tmp(name, data):
|
||||
d = os.path.join(os.path.dirname(os.path.abspath(__file__)), "_test_tmp")
|
||||
os.makedirs(d, exist_ok=True)
|
||||
p = os.path.join(d, name)
|
||||
with open(p, "wb") as f:
|
||||
f.write(data)
|
||||
return p
|
||||
|
||||
|
||||
# --- _norm_date -----------------------------------------------------------
|
||||
check("date 4-digit year", ec._norm_date("18.02.2014") == "2014-02-18")
|
||||
check("date 2-digit year", ec._norm_date("09.12.14") == "2014-12-09")
|
||||
check("date embedded", ec._norm_date("Bonn, 20.09.2014") == "2014-09-20")
|
||||
check("date invalid", ec._norm_date("99.99.9999") == "")
|
||||
check("date empty", ec._norm_date("") == "")
|
||||
|
||||
# --- _clean_money ---------------------------------------------------------
|
||||
check("money comma", ec._clean_money("27,50 Euro (Überweisung)") == "27,50")
|
||||
check("money dot→comma", ec._clean_money("25.00 €") == "25,00")
|
||||
check("money plain", ec._clean_money("28 Euro") == "28")
|
||||
check("money none", ec._clean_money("kostenlos") == "")
|
||||
|
||||
# --- parse_filename -------------------------------------------------------
|
||||
fn = ec.parse_filename(
|
||||
"Zucht der kleinen Chaoten _ Agouti (Kathlin.Baxter) - Stephan Füchsle_.docx")
|
||||
check("fn color", fn["color"] == "Agouti")
|
||||
check("fn animals", fn["animals"] == ["Kathlin", "Baxter"])
|
||||
check("fn buyer", fn["buyer"] == "Stephan Füchsle")
|
||||
|
||||
fn2 = ec.parse_filename(
|
||||
"Zucht der kleinen Chaoten _ dd Polar Sp (Velvet.Vance Jr.)- Alexandra Wendler_.docx")
|
||||
check("fn multiword color", fn2["color"] == "dd Polar Sp")
|
||||
check("fn multiword animal", fn2["animals"] == ["Velvet", "Vance Jr"])
|
||||
check("fn buyer 2", fn2["buyer"] == "Alexandra Wendler")
|
||||
|
||||
fn3 = ec.parse_filename("Zucht der kleinen Chaoten _ Ethan - Stefanie Stoica_.docx")
|
||||
check("fn no-paren animal", fn3["animals"] == ["Ethan"])
|
||||
check("fn no-paren buyer", fn3["buyer"] == "Stefanie Stoica")
|
||||
|
||||
fn4 = ec.parse_filename("Zucht der kleinen Chaoten _Azrael_.docx")
|
||||
check("fn name-only animal", fn4["animals"] == ["Azrael"])
|
||||
check("fn name-only no buyer", fn4["buyer"] == "")
|
||||
|
||||
# --- run-splitting de-mangling (the core text-extraction fix) -------------
|
||||
data = make_docx([["Name:", "F", "r", "au Josephin Kiefer"]])
|
||||
p = write_tmp("split.docx", data)
|
||||
check("run-split joins to 'Frau Josephin Kiefer'",
|
||||
ec._full_text(p) == "Name:Frau Josephin Kiefer")
|
||||
|
||||
money = make_docx([["Schutzgebühr: ", "3", "0,00 €"]])
|
||||
p = write_tmp("money.docx", money)
|
||||
check("run-split price intact",
|
||||
ec._clean_money(ec._label_value(ec._full_text(p), r"Schutzgeb\w+")) == "30,00")
|
||||
|
||||
# --- full contract: stacked layout (B) ------------------------------------
|
||||
stacked = make_docx([
|
||||
["Vermittlungsvertrag"],
|
||||
["Abgebender/Züchter:"],
|
||||
["Name:", "Frau Drazena Rimac"],
|
||||
["Straße:", "New-York-Str. 30"],
|
||||
["Empfänger/Abnehmer:"],
|
||||
["Name:", "Familie Tanja und Thorsten Kurz"],
|
||||
["Straße:", "Ebelstraße 2"],
|
||||
["Tierdaten:"],
|
||||
["Name:", "Einstein"],
|
||||
["Geburtsdatum:", "10.08.2019"],
|
||||
["Geschlecht:", "männlich"],
|
||||
["Farbschlag: ", "Schwarz"],
|
||||
["Abgabedatum:", "31.12.2019"],
|
||||
["Schutzgebühr: ", "25,00 €"],
|
||||
["Butzbach, den 31.12.2019"],
|
||||
])
|
||||
p = write_tmp("stacked.docx",
|
||||
make_docx_b := stacked)
|
||||
rec = ec.parse_contract(p)
|
||||
check("stacked buyer", rec["buyer"] == "Familie Tanja und Thorsten Kurz")
|
||||
check("stacked not seller", rec["buyer"] != "Frau Drazena Rimac")
|
||||
check("stacked animal name", rec["animalNameBody"] == "Einstein")
|
||||
check("stacked dob", rec["dob"] == "2019-08-10")
|
||||
check("stacked gender", rec["gender"] == "Male")
|
||||
check("stacked color", rec["color"] == "Schwarz")
|
||||
check("stacked handover", rec["handoverDate"] == "2019-12-31")
|
||||
check("stacked price", rec["price"] == "25,00")
|
||||
check("stacked contract date", rec["contractDate"] == "2019-12-31")
|
||||
|
||||
# --- full contract: side-by-side columns (A) ------------------------------
|
||||
# Two columns render as separate paragraphs (table cells).
|
||||
sidebyside = make_docx([
|
||||
["Vermittlungsvertrag"],
|
||||
["Züchter:"], ["Abnehmer:"],
|
||||
["Name:", "Drazena Rimac"], ["Name:", "Andrea Thesing"],
|
||||
["Straße:", "Alten-Busecker-Str. 57"], ["Straße:", "Veilchenweg 32"],
|
||||
["Tierdaten:"],
|
||||
["Name:", "Sally"],
|
||||
["Geburtsdatum:", "03.02.2014"],
|
||||
["Geschlecht:", "weiblich"],
|
||||
["Farbschlag:", "Kohlfuchs-hell"],
|
||||
["Abgabedatum:", "20.09.2014"],
|
||||
["Gesamtpreis: ", "27,00 Euro (Überweisung)"],
|
||||
["Bonn, 20.09.2014"],
|
||||
])
|
||||
p = write_tmp("sidebyside.docx", sidebyside)
|
||||
rec = ec.parse_contract(p)
|
||||
check("sbs buyer is Andrea Thesing", rec["buyer"] == "Andrea Thesing")
|
||||
check("sbs buyer not seller", rec["buyer"] != "Drazena Rimac")
|
||||
check("sbs animal", rec["animalNameBody"] == "Sally")
|
||||
check("sbs gender female", rec["gender"] == "Female")
|
||||
check("sbs price", rec["price"] == "27,00")
|
||||
|
||||
# --- skip rules -----------------------------------------------------------
|
||||
abstammung = make_docx([
|
||||
["Abstammungsnachweis"],
|
||||
["Name:"], ["Grace"],
|
||||
["Geburtsdatum:", "04.10.2015"],
|
||||
])
|
||||
p = write_tmp("abst.docx", abstammung)
|
||||
check("Abstammungsnachweis skipped", ec.parse_contract(p) is None)
|
||||
|
||||
# bad zip
|
||||
badp = write_tmp("bad.docx", b"not a zip")
|
||||
check("bad zip → None", ec.parse_contract(badp) is None)
|
||||
|
||||
# --- filename color fallback when body has no Farbschlag ------------------
|
||||
nocolor = make_docx([
|
||||
["Vermittlungsvertrag"], ["Abnehmer:"], ["Name:", "Max Mustermann"],
|
||||
["Tierdaten:"], ["Name:", "Bello"],
|
||||
["Geburtsdatum:", "01.01.2020"], ["Geschlecht:", "männlich"],
|
||||
["Gesamtpreis: ", "20,00 €"],
|
||||
])
|
||||
p = write_tmp("Zucht der kleinen Chaoten _ Agouti (Bello)- Max Mustermann_.docx",
|
||||
nocolor)
|
||||
rec = ec.parse_contract(p)
|
||||
check("color falls back to filename", rec["color"] == "Agouti")
|
||||
check("animals from filename", rec["animals"] == ["Bello"])
|
||||
|
||||
# cleanup tmp
|
||||
import shutil
|
||||
shutil.rmtree(os.path.join(os.path.dirname(os.path.abspath(__file__)), "_test_tmp"),
|
||||
ignore_errors=True)
|
||||
|
||||
print()
|
||||
if failed:
|
||||
print(f"{failed} test(s) FAILED")
|
||||
sys.exit(1)
|
||||
print("All extract_contracts tests passed.")
|
||||
@@ -277,6 +277,74 @@ check("_de_date: ISO → DD.MM.YYYY", m._de_date("2022-03-27") == "27.03.2022")
|
||||
check("_de_date: passes through non-ISO", m._de_date("unbekannt") == "unbekannt")
|
||||
|
||||
|
||||
# ── Discard history: a discarded source value records reason + replacement ──
|
||||
# _format_discard: majority-vote conflict (losing value + file → winner + file).
|
||||
_d_mehr = m._format_discard({
|
||||
"label": "Geburtsdatum", "value": "14.06.2015", "file": "A.xlsx",
|
||||
"reason": "abweichend", "replacement": "14.06.2017", "repl_file": "B.xlsx",
|
||||
"replacement_note": "Mehrheit",
|
||||
})
|
||||
check("discard: starts with warning marker", _d_mehr.startswith(m.DISCARD_MARK))
|
||||
check("discard(majority): names losing value + its file",
|
||||
"Geburtsdatum 14.06.2015 aus „A.xlsx“ verworfen" in _d_mehr)
|
||||
check("discard(majority): states the reason", "— abweichend" in _d_mehr)
|
||||
check("discard(majority): names replacement + its file + note",
|
||||
"14.06.2017 aus „B.xlsx“ verwendet (Mehrheit)." in _d_mehr)
|
||||
|
||||
# _format_discard: a parent dropped with NO replacement.
|
||||
_d_noerepl = m._format_discard({
|
||||
"text": "Vater „Jayjay“ (*19.06.2013) verworfen — unplausibel (9 Jahre älter "
|
||||
"als das Kind); kein Ersatz",
|
||||
})
|
||||
check("discard(text): verbatim text gets the warning marker",
|
||||
_d_noerepl == m.DISCARD_MARK + "Vater „Jayjay“ (*19.06.2013) verworfen — "
|
||||
"unplausibel (9 Jahre älter als das Kind); kein Ersatz")
|
||||
|
||||
# explain_pick_rejections: a wrong-sex father candidate is explained (Molly case).
|
||||
_picks = m.explain_pick_rejections(
|
||||
molly_refs, "father", "13.09.2021", fr, gender_of=gof,
|
||||
)
|
||||
_pick_father = next((d for d in _picks if "Danielle" in (d.get("value") or "")), None)
|
||||
check("pick-reject: wrong-sex father candidate is recorded",
|
||||
_pick_father is not None)
|
||||
check("pick-reject: reason = wrong sex for the father role",
|
||||
_pick_father and "falsches Geschlecht für die Vaterrolle" in _pick_father["reason"])
|
||||
check("pick-reject: replacement names the chosen Hagrid",
|
||||
_pick_father and "Hagrid" in (_pick_father.get("replacement") or ""))
|
||||
|
||||
# explain_pick_rejections: an age-impossible candidate is explained.
|
||||
_age_refs = [pref("Old", "father", "2010-01-01"), pref("Dad", "father", "2021-01-01")]
|
||||
_chosen = m.pick_parent_ref(_age_refs, "father", "2022-03-27")
|
||||
_age_picks = m.explain_pick_rejections(_age_refs, "father", "2022-03-27", _chosen)
|
||||
check("pick-reject(age): age-impossible candidate recorded with reason",
|
||||
any("unplausibles Alter" in d["reason"] for d in _age_picks))
|
||||
|
||||
# _build_gerbil_history threads field_discards (after merge) and parent_discards
|
||||
# (after the parent line) into the timeline.
|
||||
g_disc = {
|
||||
"Name": "Solice", "DateOfBirth": "2022-03-27", "Gender": "male",
|
||||
"Genotype": None, "ColorVarietyId": None, "DateOfDeath": None,
|
||||
"ImportSource": "Stammbaum.xlsx", "_filename": "Stammbaum.xlsx",
|
||||
"parentRefs": [],
|
||||
"_discarded": [{"text": "Vater „Jayjay“ verworfen — unplausibel; kein Ersatz"}],
|
||||
}
|
||||
h_disc = m._build_gerbil_history(
|
||||
[g_disc], g_disc, {},
|
||||
field_discards=[{
|
||||
"label": "Geschlecht", "value": "weiblich", "file": "Wurfchronik.docx",
|
||||
"reason": "abweichend", "replacement": "männlich", "repl_file": "Stammbaum.xlsx",
|
||||
"replacement_note": "Mehrheit",
|
||||
}],
|
||||
parent_discards=g_disc["_discarded"],
|
||||
)
|
||||
check("history: field-discard line present (majority vote)",
|
||||
any("Geschlecht weiblich aus „Wurfchronik.docx“ verworfen" in s for s in h_disc))
|
||||
check("history: parent-discard line present (dropped parent)",
|
||||
any("Vater „Jayjay“ verworfen" in s for s in h_disc))
|
||||
check("history: discard lines carry the warning marker",
|
||||
all(s.startswith(m.DISCARD_MARK) for s in h_disc if "verworfen" in s))
|
||||
|
||||
|
||||
if check.failed:
|
||||
print(f"\n{check.failed} test(s) FAILED")
|
||||
sys.exit(1)
|
||||
|
||||
Reference in New Issue
Block a user