Add detection: skew, system anchors, ink extents, cuts, staff height
Systems anchor on the vertical bracket and expand over nearby ink, per ADR 0006. Two corrections the corpus forced: - Expansion measures distance from the bracket, never from the growing extent. A chaining expansion hops between the closely-stacked lines of a title block and walks the whole way up the page — on Engel p1 it swallowed the title into system 1 and the copyright footer into system 3. - Reach is 1.5 staff heights, which takes in the lyrics below the last staff and the tempo mark and INTRO box above the first, while leaving the title block and footer out. Verified against the corpus. Ketun joululaulu: 2 systems per page except p7 (3) and p12 (1), staff 48px throughout, skew -2.7 to +1.2 per page, and p2 system 1 spans 179-1071 where its bracket is 177-994 — the difference being the bottom voice's lyric line. Engel: 2-3 systems per page, staff 70-72px. Elaman nalka: 3 systems per page, 0 skew, staff 118px at 600 DPI. The overlay dump is what made both bugs visible, and stays as the tool for diagnosing a page that comes out wrong. Closes #4, #5, #6, #7, #8, #9
This commit is contained in:
Esa Kataja
+38
-1
@@ -5,7 +5,12 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
from . import __version__
|
||||
import numpy as np
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from . import __version__, overlay
|
||||
from .detect import detect_page
|
||||
from .pdf import SourceType, open_source, page_raster
|
||||
|
||||
|
||||
@@ -20,6 +25,31 @@ def _info(args: argparse.Namespace) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def _detect(args: argparse.Namespace) -> int:
|
||||
source = open_source(args.pdf, SourceType(args.type) if args.type else None)
|
||||
out = Path(args.out)
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
pages = range(len(source)) if args.page is None else [args.page - 1]
|
||||
|
||||
for i in pages:
|
||||
gray = page_raster(source, i)
|
||||
detection = detect_page(gray)
|
||||
staves = [s.staff_height for s in detection.systems if s.staff_height]
|
||||
note = f", staff {np.median(staves):.0f}px" if staves else ""
|
||||
print(
|
||||
f"p{i + 1:<3} skew {detection.skew:+.2f}° "
|
||||
f"{len(detection.systems)} systems{note}"
|
||||
f"{' (no bracket)' if detection.bracketless else ''}"
|
||||
)
|
||||
for n, system in enumerate(detection.systems, 1):
|
||||
print(f" sys{n}: {system.top}–{system.bottom} h={system.height}")
|
||||
overlay.write(gray, detection, out / f"{source.path.stem}-p{i + 1:02}.png")
|
||||
|
||||
print(f"overlays written to {out}/")
|
||||
source.close()
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="noteman-slicer",
|
||||
@@ -37,6 +67,13 @@ def main(argv: list[str] | None = None) -> int:
|
||||
)
|
||||
info.set_defaults(func=_info)
|
||||
|
||||
det = sub.add_parser("detect", help="run detection and write debug overlays")
|
||||
det.add_argument("pdf")
|
||||
det.add_argument("--out", default="overlays", help="output directory")
|
||||
det.add_argument("--page", type=int, help="single 1-based page instead of all")
|
||||
det.add_argument("--type", choices=[t.value for t in SourceType])
|
||||
det.set_defaults(func=_detect)
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
return args.func(args)
|
||||
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
"""Detection: skew, systems, cuts, staff height.
|
||||
|
||||
Everything here is a *suggestion* the user confirms or edits (ADR 0004).
|
||||
Nothing downstream may assume a result is right.
|
||||
|
||||
Systems are anchored on the vertical bracket that spans their staves, not on
|
||||
gaps in the row-darkness profile: a row profile cannot tell an inter-staff gap
|
||||
from an inter-system gap, and gets the count wrong on every page of a
|
||||
multi-voice choral score (ADR 0006). The row profile is still needed, to expand
|
||||
each anchor to its true ink extent — a bracket stops at the last staff line,
|
||||
but the slice must include the lyrics printed below it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
SKEW_LIMIT_DEG = 5.0
|
||||
SKEW_COARSE_STEP = 1.0
|
||||
SKEW_FINE_STEP = 0.1
|
||||
_SKEW_WORK_SCALE = 0.25
|
||||
|
||||
_INK = 128 # below this is ink, above is paper
|
||||
_ANCHOR_KERNEL = 0.03 # vertical open kernel, as a fraction of page height
|
||||
_ANCHOR_MIN = 0.04 # a bracket is at least this tall, as a fraction of page
|
||||
_PROFILE_FLOOR = 0.02 # ink-run threshold, as a fraction of the profile peak
|
||||
_EXPAND_REACH = 1.5 # how far past the bracket a system's ink reaches, in staff heights
|
||||
|
||||
|
||||
@dataclass
|
||||
class System:
|
||||
"""One line of music: the ink extent that becomes a slice."""
|
||||
|
||||
top: int
|
||||
bottom: int
|
||||
staff_height: float | None = None
|
||||
|
||||
@property
|
||||
def height(self) -> int:
|
||||
return self.bottom - self.top
|
||||
|
||||
|
||||
@dataclass
|
||||
class PageDetection:
|
||||
skew: float
|
||||
systems: list[System] = field(default_factory=list)
|
||||
cuts: list[int] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def bracketless(self) -> bool:
|
||||
"""True when no bracket was found and the row profile was used alone."""
|
||||
return not self.systems or all(s.staff_height is None for s in self.systems)
|
||||
|
||||
|
||||
def row_darkness(gray: np.ndarray) -> np.ndarray:
|
||||
return (255 - gray.astype(np.float32)).sum(axis=1)
|
||||
|
||||
|
||||
def deskew_angle(gray: np.ndarray) -> float:
|
||||
"""Angle maximising row-darkness variance — staff lines are the signal.
|
||||
|
||||
Coarse then fine, on a downscaled copy: 31 warps instead of 101.
|
||||
"""
|
||||
work = cv2.resize(gray, None, fx=_SKEW_WORK_SCALE, fy=_SKEW_WORK_SCALE,
|
||||
interpolation=cv2.INTER_AREA)
|
||||
|
||||
def score(angle: float) -> float:
|
||||
return float(row_darkness(_rotate(work, angle, cv2.INTER_LINEAR)).var())
|
||||
|
||||
coarse = np.arange(-SKEW_LIMIT_DEG, SKEW_LIMIT_DEG + 1e-9, SKEW_COARSE_STEP)
|
||||
best = max(coarse, key=score)
|
||||
fine = np.arange(best - SKEW_COARSE_STEP, best + SKEW_COARSE_STEP + 1e-9, SKEW_FINE_STEP)
|
||||
fine = fine[np.abs(fine) <= SKEW_LIMIT_DEG]
|
||||
return round(float(max(fine, key=score)), 2)
|
||||
|
||||
|
||||
def _rotate(gray: np.ndarray, angle: float, flags: int = cv2.INTER_CUBIC) -> np.ndarray:
|
||||
if angle == 0.0:
|
||||
return gray
|
||||
h, w = gray.shape
|
||||
m = cv2.getRotationMatrix2D((w / 2, h / 2), angle, 1.0)
|
||||
return cv2.warpAffine(gray, m, (w, h), flags=flags, borderValue=255)
|
||||
|
||||
|
||||
def deskew(gray: np.ndarray, angle: float) -> np.ndarray:
|
||||
return _rotate(gray, angle)
|
||||
|
||||
|
||||
def system_anchors(gray: np.ndarray) -> list[tuple[int, int]]:
|
||||
"""y-extents of the vertical brackets, one per system."""
|
||||
h = gray.shape[0]
|
||||
binary = (gray < _INK).astype(np.uint8)
|
||||
kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (1, max(3, int(h * _ANCHOR_KERNEL))))
|
||||
strokes = cv2.morphologyEx(binary, cv2.MORPH_OPEN, kernel)
|
||||
|
||||
count, _, stats, _ = cv2.connectedComponentsWithStats(strokes, 8)
|
||||
tall = [
|
||||
(stats[i, cv2.CC_STAT_TOP], stats[i, cv2.CC_STAT_TOP] + stats[i, cv2.CC_STAT_HEIGHT])
|
||||
for i in range(1, count)
|
||||
if stats[i, cv2.CC_STAT_HEIGHT] > h * _ANCHOR_MIN
|
||||
]
|
||||
|
||||
# Tallest first, keeping only strokes that don't overlap one already kept:
|
||||
# a system's barlines all overlap its bracket, so each system yields one.
|
||||
anchors: list[tuple[int, int]] = []
|
||||
for top, bottom in sorted(tall, key=lambda s: s[1] - s[0], reverse=True):
|
||||
if any(not (bottom < a[0] or top > a[1]) for a in anchors):
|
||||
continue
|
||||
anchors.append((top, bottom))
|
||||
return sorted(anchors)
|
||||
|
||||
|
||||
def ink_runs(gray: np.ndarray) -> list[tuple[int, int]]:
|
||||
"""Rows containing ink, despeckled — specks are the known failure mode."""
|
||||
profile = row_darkness(cv2.medianBlur(gray, 3))
|
||||
if profile.max() <= 0:
|
||||
return []
|
||||
inked = profile > profile.max() * _PROFILE_FLOOR
|
||||
|
||||
runs: list[tuple[int, int]] = []
|
||||
start: int | None = None
|
||||
for i, on in enumerate(inked):
|
||||
if on and start is None:
|
||||
start = i
|
||||
elif not on and start is not None:
|
||||
runs.append((start, i))
|
||||
start = None
|
||||
if start is not None:
|
||||
runs.append((start, len(inked)))
|
||||
return runs
|
||||
|
||||
|
||||
def staff_height(gray: np.ndarray, top: int, bottom: int) -> float | None:
|
||||
"""Distance between a staff's outer lines, from staff-line spacing."""
|
||||
profile = row_darkness(gray[top:bottom])
|
||||
if profile.size == 0 or profile.max() <= 0:
|
||||
return None
|
||||
peaks = np.where(profile > profile.max() * 0.55)[0]
|
||||
if peaks.size < 2:
|
||||
return None
|
||||
|
||||
centres = []
|
||||
run = [peaks[0]]
|
||||
for prev, cur in zip(peaks, peaks[1:]):
|
||||
if cur - prev > 3:
|
||||
centres.append(float(np.mean(run)))
|
||||
run = []
|
||||
run.append(cur)
|
||||
centres.append(float(np.mean(run)))
|
||||
if len(centres) < 2:
|
||||
return None
|
||||
|
||||
gaps = np.diff(centres)
|
||||
# Keep intra-staff gaps; the big ones are the spaces between staves.
|
||||
intra = gaps[gaps < np.median(gaps) * 2]
|
||||
if intra.size == 0:
|
||||
return None
|
||||
return float(np.median(intra) * 4) # 5 lines, 4 spaces
|
||||
|
||||
|
||||
def _expand(
|
||||
runs: list[tuple[int, int]],
|
||||
top: int,
|
||||
bottom: int,
|
||||
max_gap: float,
|
||||
others: list[tuple[int, int]],
|
||||
) -> tuple[int, int]:
|
||||
"""Grow a bracket span over the ink close to it.
|
||||
|
||||
Lyrics printed below the last staff, and the tempo mark or section label
|
||||
printed above the first, sit within about a staff height of the bracket and
|
||||
get absorbed. A title block or a copyright footer is far further away and
|
||||
does not.
|
||||
|
||||
Distance is measured from the *bracket*, never from the growing extent: a
|
||||
title block's credit lines are stacked closely enough that a chaining
|
||||
expansion hops from one to the next and walks the whole way up the page.
|
||||
"""
|
||||
lo, hi = top, bottom
|
||||
for start, end in runs:
|
||||
if any(end > o[0] and start < o[1] for o in others):
|
||||
continue # belongs to a different system
|
||||
if end > top and start < bottom: # overlaps the bracket itself
|
||||
lo, hi = min(lo, start), max(hi, end)
|
||||
elif end <= top and top - end <= max_gap:
|
||||
lo = min(lo, start)
|
||||
elif start >= bottom and start - bottom <= max_gap:
|
||||
hi = max(hi, end)
|
||||
return lo, hi
|
||||
|
||||
|
||||
def detect_page(gray: np.ndarray, skew: float | None = None) -> PageDetection:
|
||||
"""Full proposal for one page raster. `gray` is the *unrotated* page."""
|
||||
angle = deskew_angle(gray) if skew is None else skew
|
||||
straight = deskew(gray, angle)
|
||||
|
||||
runs = ink_runs(straight)
|
||||
anchors = system_anchors(straight)
|
||||
|
||||
if anchors:
|
||||
systems = []
|
||||
for i, (top, bottom) in enumerate(anchors):
|
||||
# Staff height is measured on the bracket span, before expansion,
|
||||
# so a swallowed title block can't distort it.
|
||||
height = staff_height(straight, top, bottom)
|
||||
others = [a for j, a in enumerate(anchors) if j != i]
|
||||
reach = (height or gray.shape[0] * 0.02) * _EXPAND_REACH
|
||||
lo, hi = _expand(runs, top, bottom, reach, others)
|
||||
systems.append(System(top=lo, bottom=hi, staff_height=height))
|
||||
else:
|
||||
# No bracket: a single-staff melody or lead sheet, where every ink run
|
||||
# genuinely is its own system.
|
||||
systems = [System(top=t, bottom=b) for t, b in runs]
|
||||
|
||||
cuts = [
|
||||
(systems[i].bottom + systems[i + 1].top) // 2 for i in range(len(systems) - 1)
|
||||
]
|
||||
return PageDetection(skew=angle, systems=systems, cuts=cuts)
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Debug overlay: what detection proposed, drawn on the page.
|
||||
|
||||
The fastest way to judge a detection change, and the tool for working out why
|
||||
song #40 came out wrong. Kept after release for that reason.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
from .detect import PageDetection
|
||||
|
||||
_SYSTEM = (0, 160, 0)
|
||||
_CUT = (0, 0, 255)
|
||||
_PROFILE = (220, 120, 0)
|
||||
_PREVIEW_WIDTH = 1100
|
||||
|
||||
|
||||
def draw(gray: np.ndarray, detection: PageDetection) -> np.ndarray:
|
||||
"""Straightened page with systems boxed, cuts lined, row profile down the side."""
|
||||
from .detect import deskew, row_darkness
|
||||
|
||||
straight = deskew(gray, detection.skew)
|
||||
vis = cv2.cvtColor(straight, cv2.COLOR_GRAY2BGR)
|
||||
h, w = straight.shape
|
||||
thickness = max(1, w // 700)
|
||||
|
||||
profile = row_darkness(straight)
|
||||
if profile.max() > 0:
|
||||
scaled = (profile / profile.max() * (w * 0.08)).astype(int)
|
||||
for y in range(0, h, max(1, h // 900)):
|
||||
cv2.line(vis, (0, y), (int(scaled[y]), y), _PROFILE, 1)
|
||||
|
||||
for i, system in enumerate(detection.systems):
|
||||
cv2.rectangle(vis, (2, system.top), (w - 3, system.bottom), _SYSTEM, thickness)
|
||||
label = f"{i + 1}"
|
||||
if system.staff_height:
|
||||
label += f" staff {system.staff_height:.0f}px"
|
||||
cv2.putText(vis, label, (int(w * 0.10), system.top + int(h * 0.02)),
|
||||
cv2.FONT_HERSHEY_SIMPLEX, w / 1400, _SYSTEM, thickness)
|
||||
|
||||
for y in detection.cuts:
|
||||
cv2.line(vis, (0, y), (w, y), _CUT, thickness)
|
||||
|
||||
return vis
|
||||
|
||||
|
||||
def write(gray: np.ndarray, detection: PageDetection, path: Path) -> Path:
|
||||
vis = draw(gray, detection)
|
||||
height = int(vis.shape[0] * _PREVIEW_WIDTH / vis.shape[1])
|
||||
cv2.imwrite(str(path), cv2.resize(vis, (_PREVIEW_WIDTH, height), interpolation=cv2.INTER_AREA))
|
||||
return path
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Runnable check for detection, on a synthetic page.
|
||||
|
||||
Draws the structure that matters — a bracket per system, staves, lyrics close
|
||||
below, and a title and footer far away — so the check is about the algorithm
|
||||
rather than about any one scan. Run with `python tests/test_detect.py`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from noteman_slicer.detect import deskew, deskew_angle, detect_page # noqa: E402
|
||||
|
||||
W, H = 1000, 1400
|
||||
STAFF_GAP = 15 # → staff height 60, so expansion reaches 90px past a bracket
|
||||
|
||||
|
||||
def _system(page: np.ndarray, top: int) -> tuple[int, int]:
|
||||
"""Two staves joined by a bracket, with a lyric line below. Returns its span."""
|
||||
bottom = top + 200
|
||||
page[top:bottom, 100:104] = 0 # the bracket
|
||||
for staff_top in (top, top + 140):
|
||||
for i in range(5):
|
||||
y = staff_top + i * STAFF_GAP
|
||||
page[y : y + 2, 110:900] = 0
|
||||
page[staff_top + 90 : staff_top + 105, 200:800] = 0 # lyrics under the staff
|
||||
return top, bottom
|
||||
|
||||
|
||||
def _page() -> np.ndarray:
|
||||
page = np.full((H, W), 255, np.uint8)
|
||||
page[50:70, 300:700] = 0 # title, far above system 1
|
||||
_system(page, 200)
|
||||
_system(page, 700)
|
||||
page[1350:1365, 100:600] = 0 # footer, far below system 2
|
||||
return page
|
||||
|
||||
|
||||
def main() -> int:
|
||||
page = _page()
|
||||
|
||||
det = detect_page(page)
|
||||
assert len(det.systems) == 2, f"expected 2 systems, got {len(det.systems)}"
|
||||
assert len(det.cuts) == 1, det.cuts
|
||||
|
||||
first, second = det.systems
|
||||
# The bracket spans 200–400; the lyric line under the lower staff reaches
|
||||
# ~445 and must be absorbed.
|
||||
assert first.top == 200, first.top
|
||||
assert 400 < first.bottom < 500, first.bottom
|
||||
assert second.top == 700, second.top
|
||||
|
||||
# The title and footer are far from any bracket and must not be swallowed —
|
||||
# the bug that a chaining expansion reintroduces.
|
||||
assert first.top > 70, "title block was swallowed"
|
||||
assert second.bottom < 1350, "footer was swallowed"
|
||||
|
||||
# The cut falls between the two systems, in the whitespace.
|
||||
assert first.bottom < det.cuts[0] < second.top, det.cuts
|
||||
|
||||
assert first.staff_height is not None
|
||||
assert abs(first.staff_height - STAFF_GAP * 4) < STAFF_GAP, first.staff_height
|
||||
|
||||
# Skew is recovered to within one fine step.
|
||||
for angle in (-1.5, 0.8):
|
||||
found = deskew_angle(deskew(page, angle))
|
||||
assert abs(found + angle) <= 0.15, f"skew {angle}: got {found}"
|
||||
|
||||
# No brackets: every ink run is its own system.
|
||||
bare = np.full((H, W), 255, np.uint8)
|
||||
for y in (200, 500, 800):
|
||||
bare[y : y + 20, 100:900] = 0
|
||||
assert len(detect_page(bare).systems) == 3
|
||||
assert detect_page(bare).bracketless
|
||||
|
||||
print("ok")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user