tools/import/ (Python, zero-dep migration tooling, not product code): - xlsx_util.py: dependency-free .xlsx reader (shared strings, cells, drawing anchors) - genotype.py: notation -> frozen 8-locus mapping + verbatim rawGenotype + unmappedTokens; '-' -> '?' - extract.py: 10 Stammbaum charts + Wurfchronik -> animals.json/litters.json + anchor-mapped photos; dedup on normalise(name)+DOB -> German review-report.md (no DB load) Run: 889 raw -> 587 unique animals, 24 conflicts, 310 ambiguous, 123 photos, 752 litters. Output gitignored except review-report.md. Re-runnable per file (Wurfchronik Teil2+). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
136 lines
4.4 KiB
Python
136 lines
4.4 KiB
Python
"""Minimal dependency-free .xlsx reader (xlsx = zip of XML).
|
|
|
|
We only need: shared strings, cell text by reference, and image/drawing anchors.
|
|
Using stdlib zipfile + regex keeps this migration tooling free of openpyxl so it
|
|
runs anywhere Python 3 is present.
|
|
"""
|
|
import zipfile
|
|
import re
|
|
import html
|
|
import os
|
|
|
|
_T = re.compile(r"<t[^>]*>(.*?)</t>", re.S)
|
|
_SI = re.compile(r"<si>(.*?)</si>", re.S)
|
|
_CELL = re.compile(
|
|
r'<c\s+([^>]*?)>(?:<f[^>]*>.*?</f>)?(?:<v>(.*?)</v>|<is>(.*?)</is>)?</c>', re.S)
|
|
_ATTR = re.compile(r'(\w+)="([^"]*)"')
|
|
|
|
|
|
def col_to_num(col):
|
|
n = 0
|
|
for ch in col:
|
|
n = n * 26 + (ord(ch) - 64)
|
|
return n
|
|
|
|
|
|
def num_to_col(n):
|
|
s = ""
|
|
while n > 0:
|
|
n, r = divmod(n - 1, 26)
|
|
s = chr(65 + r) + s
|
|
return s
|
|
|
|
|
|
def shared_strings(z):
|
|
try:
|
|
raw = z.read("xl/sharedStrings.xml").decode("utf-8")
|
|
except KeyError:
|
|
return []
|
|
return [html.unescape("".join(_T.findall(si))).strip() for si in _SI.findall(raw)]
|
|
|
|
|
|
def sheet_paths(z):
|
|
"""Return worksheet xml paths in workbook order (best effort)."""
|
|
paths = sorted(n for n in z.namelist()
|
|
if re.match(r"xl/worksheets/sheet\d+\.xml$", n))
|
|
return paths
|
|
|
|
|
|
def read_cells(z, sheet_path, ss=None):
|
|
"""Return {(colnum, row): text} for a worksheet."""
|
|
if ss is None:
|
|
ss = shared_strings(z)
|
|
raw = z.read(sheet_path).decode("utf-8")
|
|
cells = {}
|
|
for attrs, v, istr in _CELL.findall(raw):
|
|
a = dict(_ATTR.findall(attrs))
|
|
ref = a.get("r")
|
|
if not ref:
|
|
continue
|
|
m = re.match(r"([A-Z]+)(\d+)", ref)
|
|
if not m:
|
|
continue
|
|
col, row = m.group(1), int(m.group(2))
|
|
typ = a.get("t")
|
|
if typ == "s" and v != "":
|
|
try:
|
|
text = ss[int(v)]
|
|
except (ValueError, IndexError):
|
|
text = ""
|
|
elif typ == "inlineStr" and istr:
|
|
text = html.unescape("".join(_T.findall(istr)))
|
|
elif v != "":
|
|
text = html.unescape(v)
|
|
else:
|
|
continue
|
|
if text.strip():
|
|
cells[(col_to_num(col), row)] = text.strip()
|
|
return cells
|
|
|
|
|
|
def header_row(cells):
|
|
"""Return {colnum: header_text} for the topmost row that has text."""
|
|
if not cells:
|
|
return {}
|
|
top = min(r for (_, r) in cells)
|
|
return {c: t for (c, r), t in cells.items() if r == top}
|
|
|
|
|
|
def image_anchors(z):
|
|
"""Yield (sheet_path, from_col, from_row, media_zip_path) for every image anchor.
|
|
|
|
drawing rels map rId -> media target; the worksheet rels map the drawing to a
|
|
sheet. We resolve sheet via worksheet _rels where possible, else attribute all
|
|
anchors to the single worksheet (these files are single-sheet charts).
|
|
"""
|
|
out = []
|
|
# sheet -> drawing
|
|
sheet_drawing = {}
|
|
for sp in sheet_paths(z):
|
|
rels = "xl/worksheets/_rels/" + os.path.basename(sp) + ".rels"
|
|
try:
|
|
r = z.read(rels).decode("utf-8")
|
|
except KeyError:
|
|
continue
|
|
for rid, tgt in re.findall(r'Id="([^"]+)"[^>]*Target="([^"]+)"', r):
|
|
if "drawing" in tgt:
|
|
dpath = os.path.normpath(os.path.join(
|
|
"xl/worksheets", tgt)).replace("\\", "/")
|
|
sheet_drawing[sp] = dpath
|
|
for sp, dpath in sheet_drawing.items():
|
|
try:
|
|
d = z.read(dpath).decode("utf-8")
|
|
except KeyError:
|
|
continue
|
|
drels = os.path.join(os.path.dirname(dpath), "_rels",
|
|
os.path.basename(dpath) + ".rels").replace("\\", "/")
|
|
relmap = {}
|
|
try:
|
|
rr = z.read(drels).decode("utf-8")
|
|
for rid, tgt in re.findall(r'Id="([^"]+)"[^>]*Target="([^"]+)"', rr):
|
|
relmap[rid] = os.path.normpath(os.path.join(
|
|
os.path.dirname(dpath), tgt)).replace("\\", "/")
|
|
except KeyError:
|
|
pass
|
|
for anc in re.findall(r"<xdr:(?:two|one)CellAnchor.*?</xdr:(?:two|one)CellAnchor>", d, re.S):
|
|
fm = re.search(
|
|
r"<xdr:from>.*?<xdr:col>(\d+)</xdr:col>.*?<xdr:row>(\d+)</xdr:row>", anc, re.S)
|
|
emb = re.search(r'r:embed="([^"]+)"', anc)
|
|
if fm and emb:
|
|
media = relmap.get(emb.group(1))
|
|
if media:
|
|
# xdr col/row are 0-based; convert to 1-based to match cell refs
|
|
out.append((sp, int(fm.group(1)) + 1,
|
|
int(fm.group(2)) + 1, media))
|
|
return out
|