Add optional bilevel shrinking of the archived PDF

Scanned scores are black ink on white paper stored as 8-bit greyscale or
RGB, which costs several times what the same page costs as a bilevel
image. Across an 11-song corpus this is 27.6 MB to 7.7 MB; Engel's
bundle goes from 5997 KB to 2092 KB with byte-identical slices, since
only the archived copy changes.

Three approaches were measured and discarded first, which is worth
recording because two of them are the obvious ones. Converting RGB to
greyscale and re-encoding makes these files 20-86% LARGER: the source
JPEGs are already near 0.7 bits per pixel, so re-encoding adds
generation loss and spends more bits than the original did, and dropping
chroma recovers nothing because JPEG already subsamples it. Lossless
structural optimisation gains 0.1%, because images are 99% of every file
and there are no duplicates. Downsampling works but 300 DPI is print
resolution, and the PDF exists to be printed.

Two failure modes were found by looking at output rather than at byte
counts, and both are now refused:

- A scan at ~115 DPI came back with broken staff lines. Guarded on
  resolution as the image is *placed on the page*, so a tiled scan with
  126 small images still qualifies where a pixel count would reject it.
- Cover artwork was flattened to grey. Guarded on chroma: artwork
  measures 44% off-grey against 3% for sensor tint on a greyscale scan.
  The first threshold of 2% was a false positive that cost 685 KB on one
  song for nothing; 10% sits in the gap with room either side.

Exposed as a button rather than a checkbox. It reports what it skipped
and why, and shows a before/after crop, because the failure it can
produce is obvious at a glance and invisible in a size figure. Off by
default: this is lossy on the copy kept for printing.

noteman_slicer/bundle.py

@@ -110,7 +110,15 @@ def write(project: Project, source: Source, path: Path) -> Path:
             zipfile.ZIP_DEFLATED,
         )
         if project.source.exists():
-            zf.write(project.source, "original.pdf")
+            pdf = project.source.read_bytes()
+            if project.optimise_pdf:
+                import pymupdf
+
+                from .pdfopt import optimise
+
+                shrunk, _ = optimise(pymupdf.open(project.source), len(pdf))
+                pdf = shrunk or pdf  # empty means it found no saving
+            zf.writestr("original.pdf", pdf, zipfile.ZIP_STORED)
         for name, data in zip(names, images):
             zf.writestr(name, data, zipfile.ZIP_STORED)
 

noteman_slicer/editor.py

@@ -34,6 +34,7 @@ from PySide6.QtWidgets import (
     QGraphicsScene,
     QGraphicsView,
     QComboBox,
+    QDialog,
     QHBoxLayout,
     QLabel,
     QLineEdit,
@@ -205,20 +206,6 @@ class PageView(QGraphicsView):
         text.setPos(x, y)
         return x + box.width() * scale + pad * 3
 
-        for i, cut in enumerate(self.page.cuts):
-            colour = _CUT_ACTIVE if i == self.selected_cut else _CUT
-            pen = QPen(colour, 2)
-            pen.setCosmetic(True)
-            points = [QPointF(x * w, y * h) for x, y in cut.points]
-            for a, b in zip(points, points[1:]):
-                scene.addLine(a.x(), a.y(), b.x(), b.y(), pen)
-            if i == self.selected_cut:
-                r = HIT * 1.5 / max(self.transform().m11(), 1e-6)
-                for p in points:
-                    scene.addEllipse(
-                        p.x() - r, p.y() - r, r * 2, r * 2, QPen(Qt.NoPen), QBrush(_VERTEX)
-                    )
-
     def _slice_polygon(self, slot: int, w: int, h: int) -> QPolygonF:
         above, below = self.page.bounds(slot)
         top = [(0.0, 0.0), (1.0, 0.0)] if above is None else above.points
@@ -573,6 +560,11 @@ class Editor(QMainWindow):
                 edit.setFixedWidth(90)
             meta_form.addRow(f"{field.replace('_', ' ').title()}{' *' if required else ''}", edit)
 
+        self.optimise = QPushButton("Shrink the original PDF…")
+        self.optimise.setToolTip("Convert scanned pages to bilevel in the archived PDF")
+        self.optimise.clicked.connect(self._optimise_pdf)
+        meta_layout.addWidget(self.optimise)
+
         self.summary = QLabel()
         self.summary.setWordWrap(True)
         box.addWidget(self.summary)
@@ -765,6 +757,67 @@ class Editor(QMainWindow):
         }
         self.autosave.start()
 
+    def _optimise_pdf(self) -> None:
+        """Offer to shrink the archived PDF, showing the result before agreeing.
+
+        A before/after crop rather than a checkbox: the failure this can produce
+        — broken staff lines on a coarse scan — is obvious at a glance and
+        invisible in a byte count.
+        """
+        import pymupdf
+
+        from .pdfopt import Report, optimise, preview
+
+        self.statusBar().showMessage("examining the PDF…")
+        QApplication.processEvents()
+        source = self.project.source
+        data, report = optimise(pymupdf.open(source), source.stat().st_size)
+        self.statusBar().clearMessage()
+
+        if not data:
+            QMessageBox.information(self, "Nothing to shrink", report.summary())
+            self.project.optimise_pdf = False
+            return
+
+        dialog = QDialog(self)
+        dialog.setWindowTitle("Shrink the original PDF")
+        layout = QVBoxLayout(dialog)
+        text = QLabel(report.summary() + "\n\nThe slices are unaffected — only the archived PDF.")
+        text.setWordWrap(True)
+        layout.addWidget(text)
+
+        crop = preview(pymupdf.open(source), pymupdf.open(stream=data, filetype="pdf"))
+        crop = np.ascontiguousarray(crop)
+        h, w, _ = crop.shape
+        image = QImage(crop.data, w, h, w * 3, QImage.Format_BGR888).copy()
+        label = QLabel()
+        label.setPixmap(QPixmap.fromImage(image))
+        area = QScrollArea()
+        area.setWidget(label)
+        area.setWidgetResizable(True)
+        area.setMinimumHeight(420)
+        layout.addWidget(area)
+        layout.addWidget(QLabel("Original above, shrunk below. Check the staff lines."))
+
+        buttons = QHBoxLayout()
+        use = QPushButton("Use the smaller PDF")
+        use.clicked.connect(dialog.accept)
+        keep = QPushButton("Keep the original")
+        keep.clicked.connect(dialog.reject)
+        buttons.addWidget(use)
+        buttons.addWidget(keep)
+        layout.addLayout(buttons)
+        dialog.resize(1100, 700)
+
+        self.project.optimise_pdf = dialog.exec() == QDialog.Accepted
+        self._touched()
+        self.statusBar().showMessage(
+            "the bundle will carry the shrunk PDF"
+            if self.project.optimise_pdf
+            else "the bundle will carry the original PDF",
+            4000,
+        )
+
     def _reset_rect(self) -> None:
         """Back to what detection proposed for this page.
 

noteman_slicer/pdfopt.py

@@ -0,0 +1,172 @@
+"""Optional shrinking of the original PDF carried in a bundle.
+
+Scanned scores are usually black ink on white paper stored as 8-bit greyscale
+or RGB, which costs several times what the same page costs as a bilevel image.
+Converting them is worth 7–9× on a real corpus.
+
+Two things it must not do, both found by looking at output rather than at
+numbers:
+
+  * A page that is genuinely coloured — cover artwork — loses its artwork.
+  * A scan too coarse to have more than about one pixel per staff line comes
+    back with the staff lines broken.
+
+Both are detectable before converting, so both are skipped. Everything skipped
+is reported, so a caller can say what was left alone and why.
+
+This affects only the archival copy of the score. Slices are cut from the
+original before any of this and are unchanged either way.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+import cv2
+import numpy as np
+import pymupdf
+
+# Below this many pixels per inch as the image is *placed on the page*, staff
+# lines are about a pixel wide and thresholding breaks them. Measured against a
+# corpus where the one failure sat at ~115 DPI and the successes at 260+.
+MIN_DPI = 200
+
+# An image is "coloured" when this share of sampled pixels are off-grey by
+# more than _CHROMA. The two populations are far apart: measured on a corpus,
+# cover artwork sits at 44% while a greyscale scan's sensor tint reaches 3%.
+# Ten percent sits in the gap with room on both sides.
+_CHROMA = 24
+_COLOUR_SHARE = 0.10
+
+_BLOCK = 31  # adaptive threshold window
+_OFFSET = 15
+
+
+@dataclass
+class Report:
+    before: int = 0
+    after: int = 0
+    converted: int = 0
+    skipped: dict[str, int] = field(default_factory=dict)
+
+    @property
+    def ratio(self) -> float:
+        return self.after / self.before if self.before else 1.0
+
+    def skip(self, reason: str) -> None:
+        self.skipped[reason] = self.skipped.get(reason, 0) + 1
+
+    def summary(self) -> str:
+        if not self.converted:
+            return "nothing to optimise — every image is already bilevel, coloured or too coarse"
+        parts = [
+            f"{self.before / 1024:.0f} KB → {self.after / 1024:.0f} KB "
+            f"({self.ratio * 100:.0f}%), {self.converted} images converted"
+        ]
+        for reason, count in sorted(self.skipped.items()):
+            parts.append(f"{count} left alone: {reason}")
+        return "\n".join(parts)
+
+
+def _is_coloured(image: np.ndarray) -> bool:
+    if image.ndim != 3 or image.shape[2] < 3:
+        return False
+    sample = image[::4, ::4, :3].astype(np.int16)
+    spread = sample.max(axis=2) - sample.min(axis=2)
+    return float((spread > _CHROMA).mean()) > _COLOUR_SHARE
+
+
+def _placed_dpi(page: pymupdf.Page, item, width: int) -> float:
+    """Pixels per inch of an image as it appears on the page.
+
+    Not the pixel count: a page split into tiles has small images at a high
+    resolution, and a full-page image can be large yet coarse.
+    """
+    try:
+        bbox = pymupdf.Rect(page.get_image_bbox(item))
+    except (ValueError, RuntimeError):
+        return float("inf")
+    inches = abs(bbox.width) / 72.0
+    return width / inches if inches > 0 else float("inf")
+
+
+def optimise(doc: pymupdf.Document, source_bytes: int) -> tuple[bytes, Report]:
+    """Return the optimised PDF and a report of what was done.
+
+    `doc` is modified in place, so pass a copy or reopen afterwards.
+    """
+    report = Report(before=source_bytes)
+
+    for page in doc:
+        for item in page.get_images(full=True):
+            xref = item[0]
+            info = doc.extract_image(xref)
+
+            if info.get("bpc") == 1:
+                report.skip("already bilevel")
+                continue
+
+            if _placed_dpi(page, item, info["width"]) < MIN_DPI:
+                report.skip(f"below {MIN_DPI} DPI, staff lines would break")
+                continue
+
+            raw = cv2.imdecode(np.frombuffer(info["image"], np.uint8), cv2.IMREAD_UNCHANGED)
+            if raw is None:
+                report.skip("unreadable encoding")
+                continue
+
+            if _is_coloured(raw):
+                report.skip("coloured artwork")
+                continue
+
+            gray = cv2.cvtColor(raw, cv2.COLOR_BGR2GRAY) if raw.ndim == 3 else raw
+            bilevel = cv2.adaptiveThreshold(
+                gray, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, _BLOCK, _OFFSET
+            )
+            ok, buffer = cv2.imencode(".png", bilevel, [cv2.IMWRITE_PNG_COMPRESSION, 9])
+            if not ok:
+                report.skip("re-encoding failed")
+                continue
+            try:
+                page.replace_image(xref, stream=buffer.tobytes())
+            except (ValueError, RuntimeError):
+                report.skip("could not be replaced")
+                continue
+            report.converted += 1
+
+    data = doc.tobytes(garbage=4, deflate=True, clean=True)
+    # Never hand back something larger than what came in.
+    if len(data) >= source_bytes:
+        report.after = source_bytes
+        report.converted = 0
+        report.skip("no saving available")
+        return b"", report
+
+    report.after = len(data)
+    return data, report
+
+
+def preview(original: pymupdf.Document, optimised: pymupdf.Document, dpi: int = 260):
+    """A stacked before/after crop of the first page, for eyeballing the result.
+
+    The numbers cannot show the failure this guards against — a broken staff
+    line is obvious at a glance and invisible in a byte count.
+    """
+    rect = original[0].rect
+    clip = pymupdf.Rect(
+        rect.x0 + rect.width * 0.08,
+        rect.y0 + rect.height * 0.20,
+        rect.x0 + rect.width * 0.58,
+        rect.y0 + rect.height * 0.33,
+    )
+
+    def render(doc: pymupdf.Document) -> np.ndarray:
+        pix = doc[0].get_pixmap(dpi=dpi, clip=clip)
+        image = np.frombuffer(pix.samples, np.uint8).reshape(pix.height, pix.width, pix.n)
+        return image[:, :, :3] if pix.n >= 3 else cv2.cvtColor(image[:, :, 0], cv2.COLOR_GRAY2BGR)
+
+    before, after = render(original), render(optimised)
+    h = min(before.shape[0], after.shape[0])
+    w = min(before.shape[1], after.shape[1])
+    divider = np.full((4, w, 3), 128, np.uint8)
+    return np.vstack([before[:h, :w], divider, after[:h, :w]])

noteman_slicer/project.py

@@ -222,6 +222,10 @@ class Project:
     key: str = "c"
     time: str = "4/4"
     clefs: list[str] = field(default_factory=list)
+    # Shrink the archival PDF carried in the bundle by converting its scanned
+    # pages to bilevel. Off by default: it is lossy on the copy kept for
+    # printing, and on some scans it breaks staff lines.
+    optimise_pdf: bool = False
     path: Path | None = None
     # Set once the song has been exported. A project is spent at that point:
     # opening the PDF again starts a fresh session from detection rather than
@@ -308,6 +312,7 @@ class Project:
             "key": self.key,
             "time": self.time,
             "clefs": self.clefs,
+            "optimise_pdf": self.optimise_pdf,
             "pages": [
                 {
                     "skew": page.skew,
@@ -415,6 +420,7 @@ class Project:
             key=data.get("key", "c"),
             time=data.get("time", "4/4"),
             clefs=data.get("clefs", []),
+            optimise_pdf=data.get("optimise_pdf", False),
         )
 
     def source_changed(self) -> bool:

tests/test_pdfopt.py

@@ -0,0 +1,96 @@
+"""Runnable check for optional PDF shrinking, including what it refuses to do."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import numpy as np
+import pymupdf
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from noteman_slicer.pdfopt import MIN_DPI, optimise  # noqa: E402
+
+A4_PT = (595, 842)
+
+
+def _pdf(path: Path, width: int, height: int, *, colour: bool = False, bilevel: bool = False):
+    """One full-page image of ruled lines, at the given pixel size."""
+    art = np.full((height, width, 3), 255, np.uint8)
+    for i in range(6):
+        y = int(height * (0.2 + i * 0.03))
+        art[y : y + max(1, height // 900), int(width * 0.1) : int(width * 0.9)] = 0
+    if colour:
+        art[: height // 3, :, 0] = 40  # a strong blue cast over the top third
+        art[: height // 3, :, 1] = 90
+    grey = art[:, :, 0] if not colour else None
+
+    doc = pymupdf.open()
+    page = doc.new_page(width=A4_PT[0], height=A4_PT[1])
+    if bilevel:
+        pix = pymupdf.Pixmap(pymupdf.csGRAY, width, height, bytearray(grey.tobytes()), False)
+        page.insert_image(page.rect, pixmap=pix)
+        doc.save(path, garbage=4, deflate=True)
+        # Re-save through a 1-bit PNG so the stored image really is bilevel.
+        import cv2
+
+        ok, buf = cv2.imencode(".png", (grey > 127).astype(np.uint8) * 255)
+        doc2 = pymupdf.open()
+        p2 = doc2.new_page(width=A4_PT[0], height=A4_PT[1])
+        p2.insert_image(p2.rect, stream=buf.tobytes())
+        doc2.save(path, garbage=4, deflate=True)
+        return
+    stream = art if colour else np.dstack([grey] * 3)
+    import cv2
+
+    ok, buf = cv2.imencode(".jpg", stream, [cv2.IMWRITE_JPEG_QUALITY, 92])
+    page.insert_image(page.rect, stream=buf.tobytes())
+    doc.save(path, garbage=4, deflate=True)
+
+
+def main() -> int:
+    tmp = Path(__file__).with_name("_tmp")
+    tmp.mkdir(exist_ok=True)
+
+    # A4 is 8.26in wide, so 2480px is ~300 DPI and 800px is ~97 DPI.
+    fine, coarse, colour = tmp / "fine.pdf", tmp / "coarse.pdf", tmp / "colour.pdf"
+    _pdf(fine, 2480, 3508)
+    _pdf(coarse, 800, 1130)
+    _pdf(colour, 2480, 3508, colour=True)
+
+    data, report = optimise(pymupdf.open(fine), fine.stat().st_size)
+    assert report.converted == 1, report.summary()
+    assert data, "a greyscale scan at 300 DPI should shrink"
+    assert report.ratio < 0.9, report.ratio
+    # The result must still be a readable PDF of the same page count.
+    assert len(pymupdf.open(stream=data, filetype="pdf")) == 1
+
+    # Too coarse: staff lines would break, so it is left alone.
+    _, report = optimise(pymupdf.open(coarse), coarse.stat().st_size)
+    assert report.converted == 0, report.summary()
+    assert any("DPI" in reason for reason in report.skipped), report.skipped
+
+    # Genuine colour: artwork is not thrown away.
+    _, report = optimise(pymupdf.open(colour), colour.stat().st_size)
+    assert report.converted == 0, report.summary()
+    assert any("colour" in reason for reason in report.skipped), report.skipped
+
+    # A no-op run reports honestly rather than returning something bigger.
+    empty = pymupdf.open()
+    empty.new_page()
+    data, report = optimise(empty, 1)
+    assert data == b"" and report.converted == 0
+    assert report.ratio == 1.0
+
+    assert MIN_DPI >= 150, "the floor exists to protect thin staff lines"
+
+    for f in (fine, coarse, colour):
+        f.unlink(missing_ok=True)
+    tmp.rmdir()
+    print("ok")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())