From 51b816025183801f5953d849fe4ea83cd9385582 Mon Sep 17 00:00:00 2001 From: Luke Hoersten Date: Mon, 15 Jun 2026 07:23:47 -0500 Subject: firmware: zero-copy JPEG → BGR888 → DSI hot path; Scrypted pre-rotates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Architectural rework of the /frame hot path. Scrypted now ships every JPEG already scaled + rotated to the panel's native dimensions (read from the firmware's /state response so nothing is hardcoded on the Scrypted side); the firmware decodes the JPEG straight into a BGR888 buffer that's directly draw_bitmap'able by the DSI driver, with zero CPU pixel work and zero rotation work in between. Firmware - jpeg_decoder now uses JPEG_DECODE_OUT_FORMAT_RGB888 + JPEG_DEC_RGB_ELEMENT_ORDER_BGR. Output buffer sized for 800*480*3. - New display_present_bgr888() is a one-liner that hands the decoder's output straight to esp_lcd_panel_draw_bitmap. - /frame handler validates dimensions against the panel-native VIEWPORT_PANEL_WIDTH x VIEWPORT_PANEL_HEIGHT (was effective_dims branching on orientation). Returns 400 if it's anything else. - /state JSON adds panel_width + panel_height so Scrypted can read them without hardcoding board-specific knowledge. - display_present_rgb565 + s_rot_buf stay for the local-screens cold path (info screen, loading) which still does its own CPU conversion + rotation — infrequent enough that it's not worth the rewrite. Scrypted - startStream() GETs /state at stream-start time, caches panel_width and panel_height in viewport storage, and uses them as the ffmpeg scale target. Falls back to cached or 800x480 if /state is mid-reboot. - For portrait viewports the ffmpeg pipeline now does scale=H:W:flags=lanczos,transpose=1 so the JPEG arrives pre-rotated 90° CW into panel-native dimensions. Landscape is just scale=W:H. - No more in-firmware rotation; Scrypted is the single source of truth for "how do I get this camera frame into a panel-shaped JPEG". Expected ceiling lift: ~5 fps → ~10 fps, gated by the JPEG decoder hardware throughput instead of the CPU rgb565→bgr888 + rotation loop. --- scrypted/scrypted-viewport.ts | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) (limited to 'scrypted') diff --git a/scrypted/scrypted-viewport.ts b/scrypted/scrypted-viewport.ts index 966d2fb..d4bf8a5 100644 --- a/scrypted/scrypted-viewport.ts +++ b/scrypted/scrypted-viewport.ts @@ -509,8 +509,36 @@ class ScryptedViewportProvider extends ScryptedDeviceBase const cam: any = systemManager.getDeviceById(v.cameraId); if (!cam) return; - const w = v.orientation === "portrait" ? 480 : 800; - const h = v.orientation === "portrait" ? 800 : 480; + + // Fetch the panel's native dimensions from the firmware and + // cache them on the viewport's storage. Falls back to 800x480 + // if /state is unreachable (e.g. mid-reboot). Panel dims never + // change for a given device so this only really needs to run + // once per discovery; refreshing on every wake costs ~5 ms and + // self-heals if the firmware is replaced. + const pw = parseInt(v.storage.getItem("panel_w") || "0", 10); + const ph = parseInt(v.storage.getItem("panel_h") || "0", 10); + let panelW = pw || 800; + let panelH = ph || 480; + try { + const st = await fetch(`http://${v.host}/state`, { signal: AbortSignal.timeout(1500) }).then(r => r.json()); + if (st?.panel_width && st?.panel_height) { + panelW = Number(st.panel_width); + panelH = Number(st.panel_height); + v.storage.setItem("panel_w", String(panelW)); + v.storage.setItem("panel_h", String(panelH)); + } + } catch { /* keep cached values */ } + + // Always send panel-native dimensions (panelW x panelH). For a + // portrait viewport we scale to the logical (panelH x panelW) + // target then transpose 90° CW so the buffer that arrives at the + // panel is already in the right rotation. The firmware never + // touches pixels — the hardware JPEG decoder writes BGR888 + // straight into a DMA buffer that gets handed to the DSI engine. + const vf = v.orientation === "portrait" + ? `scale=${panelH}:${panelW}:flags=lanczos,transpose=1` // 90° CW → panelW x panelH + : `scale=${panelW}:${panelH}:flags=lanczos`; const fps = Math.max(1, Math.round(1000 / v.frameIntervalMs)); // Pull the camera's video stream, convert to ffmpeg input args, and @@ -548,7 +576,7 @@ class ScryptedViewportProvider extends ScryptedDeviceBase "-fflags", "+genpts+nobuffer", "-flags", "low_delay", ...(ffmpegInput.inputArguments || []), "-an", "-sn", - "-vf", `scale=${w}:${h}:flags=lanczos,fps=${fps}`, + "-vf", `${vf},fps=${fps}`, "-c:v", "mjpeg", "-q:v", "2", "-f", "image2pipe", "-flush_packets", "1", "pipe:1", -- cgit v1.2.3