FEAT-8b: spreadsheet import tooling (stages 1-2) + review report
tools/import/ (Python, zero-dep migration tooling, not product code): - xlsx_util.py: dependency-free .xlsx reader (shared strings, cells, drawing anchors) - genotype.py: notation -> frozen 8-locus mapping + verbatim rawGenotype + unmappedTokens; '-' -> '?' - extract.py: 10 Stammbaum charts + Wurfchronik -> animals.json/litters.json + anchor-mapped photos; dedup on normalise(name)+DOB -> German review-report.md (no DB load) Run: 889 raw -> 587 unique animals, 24 conflicts, 310 ambiguous, 123 photos, 752 litters. Output gitignored except review-report.md. Re-runnable per file (Wurfchronik Teil2+). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
135
tools/import/xlsx_util.py
Normal file
135
tools/import/xlsx_util.py
Normal file
@@ -0,0 +1,135 @@
|
||||
"""Minimal dependency-free .xlsx reader (xlsx = zip of XML).
|
||||
|
||||
We only need: shared strings, cell text by reference, and image/drawing anchors.
|
||||
Using stdlib zipfile + regex keeps this migration tooling free of openpyxl so it
|
||||
runs anywhere Python 3 is present.
|
||||
"""
|
||||
import zipfile
|
||||
import re
|
||||
import html
|
||||
import os
|
||||
|
||||
_T = re.compile(r"<t[^>]*>(.*?)</t>", re.S)
|
||||
_SI = re.compile(r"<si>(.*?)</si>", re.S)
|
||||
_CELL = re.compile(
|
||||
r'<c\s+([^>]*?)>(?:<f[^>]*>.*?</f>)?(?:<v>(.*?)</v>|<is>(.*?)</is>)?</c>', re.S)
|
||||
_ATTR = re.compile(r'(\w+)="([^"]*)"')
|
||||
|
||||
|
||||
def col_to_num(col):
|
||||
n = 0
|
||||
for ch in col:
|
||||
n = n * 26 + (ord(ch) - 64)
|
||||
return n
|
||||
|
||||
|
||||
def num_to_col(n):
|
||||
s = ""
|
||||
while n > 0:
|
||||
n, r = divmod(n - 1, 26)
|
||||
s = chr(65 + r) + s
|
||||
return s
|
||||
|
||||
|
||||
def shared_strings(z):
|
||||
try:
|
||||
raw = z.read("xl/sharedStrings.xml").decode("utf-8")
|
||||
except KeyError:
|
||||
return []
|
||||
return [html.unescape("".join(_T.findall(si))).strip() for si in _SI.findall(raw)]
|
||||
|
||||
|
||||
def sheet_paths(z):
|
||||
"""Return worksheet xml paths in workbook order (best effort)."""
|
||||
paths = sorted(n for n in z.namelist()
|
||||
if re.match(r"xl/worksheets/sheet\d+\.xml$", n))
|
||||
return paths
|
||||
|
||||
|
||||
def read_cells(z, sheet_path, ss=None):
|
||||
"""Return {(colnum, row): text} for a worksheet."""
|
||||
if ss is None:
|
||||
ss = shared_strings(z)
|
||||
raw = z.read(sheet_path).decode("utf-8")
|
||||
cells = {}
|
||||
for attrs, v, istr in _CELL.findall(raw):
|
||||
a = dict(_ATTR.findall(attrs))
|
||||
ref = a.get("r")
|
||||
if not ref:
|
||||
continue
|
||||
m = re.match(r"([A-Z]+)(\d+)", ref)
|
||||
if not m:
|
||||
continue
|
||||
col, row = m.group(1), int(m.group(2))
|
||||
typ = a.get("t")
|
||||
if typ == "s" and v != "":
|
||||
try:
|
||||
text = ss[int(v)]
|
||||
except (ValueError, IndexError):
|
||||
text = ""
|
||||
elif typ == "inlineStr" and istr:
|
||||
text = html.unescape("".join(_T.findall(istr)))
|
||||
elif v != "":
|
||||
text = html.unescape(v)
|
||||
else:
|
||||
continue
|
||||
if text.strip():
|
||||
cells[(col_to_num(col), row)] = text.strip()
|
||||
return cells
|
||||
|
||||
|
||||
def header_row(cells):
|
||||
"""Return {colnum: header_text} for the topmost row that has text."""
|
||||
if not cells:
|
||||
return {}
|
||||
top = min(r for (_, r) in cells)
|
||||
return {c: t for (c, r), t in cells.items() if r == top}
|
||||
|
||||
|
||||
def image_anchors(z):
|
||||
"""Yield (sheet_path, from_col, from_row, media_zip_path) for every image anchor.
|
||||
|
||||
drawing rels map rId -> media target; the worksheet rels map the drawing to a
|
||||
sheet. We resolve sheet via worksheet _rels where possible, else attribute all
|
||||
anchors to the single worksheet (these files are single-sheet charts).
|
||||
"""
|
||||
out = []
|
||||
# sheet -> drawing
|
||||
sheet_drawing = {}
|
||||
for sp in sheet_paths(z):
|
||||
rels = "xl/worksheets/_rels/" + os.path.basename(sp) + ".rels"
|
||||
try:
|
||||
r = z.read(rels).decode("utf-8")
|
||||
except KeyError:
|
||||
continue
|
||||
for rid, tgt in re.findall(r'Id="([^"]+)"[^>]*Target="([^"]+)"', r):
|
||||
if "drawing" in tgt:
|
||||
dpath = os.path.normpath(os.path.join(
|
||||
"xl/worksheets", tgt)).replace("\\", "/")
|
||||
sheet_drawing[sp] = dpath
|
||||
for sp, dpath in sheet_drawing.items():
|
||||
try:
|
||||
d = z.read(dpath).decode("utf-8")
|
||||
except KeyError:
|
||||
continue
|
||||
drels = os.path.join(os.path.dirname(dpath), "_rels",
|
||||
os.path.basename(dpath) + ".rels").replace("\\", "/")
|
||||
relmap = {}
|
||||
try:
|
||||
rr = z.read(drels).decode("utf-8")
|
||||
for rid, tgt in re.findall(r'Id="([^"]+)"[^>]*Target="([^"]+)"', rr):
|
||||
relmap[rid] = os.path.normpath(os.path.join(
|
||||
os.path.dirname(dpath), tgt)).replace("\\", "/")
|
||||
except KeyError:
|
||||
pass
|
||||
for anc in re.findall(r"<xdr:(?:two|one)CellAnchor.*?</xdr:(?:two|one)CellAnchor>", d, re.S):
|
||||
fm = re.search(
|
||||
r"<xdr:from>.*?<xdr:col>(\d+)</xdr:col>.*?<xdr:row>(\d+)</xdr:row>", anc, re.S)
|
||||
emb = re.search(r'r:embed="([^"]+)"', anc)
|
||||
if fm and emb:
|
||||
media = relmap.get(emb.group(1))
|
||||
if media:
|
||||
# xdr col/row are 0-based; convert to 1-based to match cell refs
|
||||
out.append((sp, int(fm.group(1)) + 1,
|
||||
int(fm.group(2)) + 1, media))
|
||||
return out
|
||||
Reference in New Issue
Block a user