diff options
| -rw-r--r-- | main/stream_server.c | 81 | ||||
| -rw-r--r-- | scrypted/scrypted-viewport.ts | 149 |
2 files changed, 175 insertions, 55 deletions
diff --git a/main/stream_server.c b/main/stream_server.c index cebe3cb..7876968 100644 --- a/main/stream_server.c +++ b/main/stream_server.c @@ -67,6 +67,15 @@ static void handle_client(int fd, const char *peer) uint64_t frames_decoded = 0; int64_t t_window_start = esp_timer_get_time(); uint64_t bytes_in_window = 0; + int64_t t_prev_paint_done = 0; // for idle-gap measurement + + // Per-window min/max accumulators (microseconds). + int64_t recv_min = INT64_MAX, recv_max = 0, recv_sum = 0; + int64_t dec_min = INT64_MAX, dec_max = 0, dec_sum = 0; + int64_t pnt_min = INT64_MAX, pnt_max = 0, pnt_sum = 0; + int64_t idle_min = INT64_MAX, idle_max = 0, idle_sum = 0; + int64_t lock_min = INT64_MAX, lock_max = 0, lock_sum = 0; + uint64_t window_samples = 0; while (1) { uint8_t hdr[HEADER_BYTES]; @@ -86,12 +95,18 @@ static void handle_client(int fd, const char *peer) } int64_t t_entry = esp_timer_get_time(); + // Idle gap: time between the previous paint completing and + // this header landing. Large idle = ffmpeg/network is the + // bottleneck; near-zero idle = we're the bottleneck, source + // is queued up waiting. + int64_t idle_us = t_prev_paint_done ? (t_entry - t_prev_paint_done) : 0; if (!jpeg_decoder_try_lock(2000)) { ESP_LOGW(TAG, "decoder busy 2s — skipping seq %u", (unsigned)seq); if (drain_n(fd, jpeg_len) != ESP_OK) return; continue; } + int64_t t_lock_done = esp_timer_get_time(); uint8_t *in = jpeg_decoder_input_buffer(); if (read_n(fd, in, jpeg_len) != ESP_OK) { @@ -154,27 +169,65 @@ static void handle_client(int fd, const char *peer) state_machine_frame_painted(); frames_decoded++; - // Every 30 frames log a per-stage breakdown + window throughput - // — same shape as the old /frame log so the serial output stays - // useful for debugging without the script side. - if (frames_decoded % 30 == 0) { + // Stage timings for this frame, in microseconds. + int64_t lock_us = t_lock_done - t_entry; + int64_t recv_us = t_recv - t_lock_done; + int64_t dec_us = t_decode - t_recv; + int64_t pnt_us = t_paint - t_decode; + + if (lock_us < lock_min) lock_min = lock_us; + if (lock_us > lock_max) lock_max = lock_us; + lock_sum += lock_us; + if (recv_us < recv_min) recv_min = recv_us; + if (recv_us > recv_max) recv_max = recv_us; + recv_sum += recv_us; + if (dec_us < dec_min) dec_min = dec_us; + if (dec_us > dec_max) dec_max = dec_us; + dec_sum += dec_us; + if (pnt_us < pnt_min) pnt_min = pnt_us; + if (pnt_us > pnt_max) pnt_max = pnt_us; + pnt_sum += pnt_us; + if (idle_us > 0) { + if (idle_us < idle_min) idle_min = idle_us; + if (idle_us > idle_max) idle_max = idle_us; + idle_sum += idle_us; + } + window_samples++; + t_prev_paint_done = t_paint; + + // Every 30 frames log a windowed min/avg/max breakdown + + // sustained throughput. Idle = gap between previous paint + // and next header arriving (= upstream slack). lock = mutex + // acquire (should be ~0 with single client). recv = body + // bytes off the wire. dec = HW JPEG. paint = backbuffer flip. + if (frames_decoded % 30 == 0 && window_samples > 0) { int64_t now = esp_timer_get_time(); double win_s = (now - t_window_start) / 1.0e6; double mb_per_s = (win_s > 0) ? ((double)bytes_in_window / win_s) / (1024.0 * 1024.0) : 0.0; + double fps = (win_s > 0) ? (double)window_samples / win_s : 0.0; ESP_LOGI(TAG, - "frame %llu seq=%u: recv=%lldus dec=%lldus paint=%lldus " - "total=%lldus (jpeg=%uKB) window=%llumiB/%.1fs (%.2fMB/s)", - (unsigned long long)frames_decoded, (unsigned)seq, - (long long)(t_recv - t_entry), - (long long)(t_decode - t_recv), - (long long)(t_paint - t_decode), - (long long)(t_paint - t_entry), - (unsigned)(jpeg_len / 1024), - (unsigned long long)(bytes_in_window / (1024 * 1024)), - win_s, mb_per_s); + "%llu frames over %.1fs: %.1ffps %.2fMB/s avg-jpeg=%uKB | " + "lock min/avg/max=%lld/%lld/%lldus | " + "recv min/avg/max=%lld/%lld/%lldus | " + "dec min/avg/max=%lld/%lld/%lldus | " + "paint min/avg/max=%lld/%lld/%lldus | " + "idle min/avg/max=%lld/%lld/%lldus", + (unsigned long long)window_samples, win_s, fps, mb_per_s, + (unsigned)((bytes_in_window / window_samples) / 1024), + (long long)lock_min, (long long)(lock_sum / window_samples), (long long)lock_max, + (long long)recv_min, (long long)(recv_sum / window_samples), (long long)recv_max, + (long long)dec_min, (long long)(dec_sum / window_samples), (long long)dec_max, + (long long)pnt_min, (long long)(pnt_sum / window_samples), (long long)pnt_max, + (long long)(idle_min == INT64_MAX ? 0 : idle_min), + (long long)(idle_sum / window_samples), + (long long)idle_max); t_window_start = now; bytes_in_window = 0; + recv_min = dec_min = pnt_min = idle_min = lock_min = INT64_MAX; + recv_max = dec_max = pnt_max = idle_max = lock_max = 0; + recv_sum = dec_sum = pnt_sum = idle_sum = lock_sum = 0; + window_samples = 0; } } } diff --git a/scrypted/scrypted-viewport.ts b/scrypted/scrypted-viewport.ts index cace56f..6aa063f 100644 --- a/scrypted/scrypted-viewport.ts +++ b/scrypted/scrypted-viewport.ts @@ -6,7 +6,7 @@ // short git hash of the commit that added this constant — if the // hash in the log doesn't match the HEAD this file came from, the // Scrypted Script editor is still on stale code. -const SCRIPT_VERSION = "d8d9a66"; +const SCRIPT_VERSION = "pending"; // // Architecture // ------------ @@ -123,6 +123,16 @@ class Viewport extends ScryptedDeviceBase implements Settings { const v = this.storage.getItem("brightness"); return v ? Math.max(0, Math.min(100, parseInt(v, 10) || 0)) : DEFAULT_BRIGHTNESS; } + // Preferred camera substream. "auto" walks a low-latency-first + // priority list and takes the first that resolves; the other + // options pin a specific substream so the user can pick a higher- + // fps source when "auto" lands on a slow 5-fps preview stream. + get streamDestination(): string { + const v = this.storage.getItem("stream_destination"); + const allowed = new Set(["auto", "low-resolution", "medium-resolution", + "local", "remote", "remote-recorder"]); + return allowed.has(v as any) ? (v as string) : "auto"; + } // ffmpeg mjpeg encoder -q:v. Valid range 1..31, lower = higher // quality + bigger JPEG (1 ≈ visually lossless, 31 ≈ very lossy). // Default 1 — with HTTP keep-alive + NODELAY we have plenty of @@ -197,6 +207,14 @@ class Viewport extends ScryptedDeviceBase implements Settings { } as any, { group: "Display", + key: "stream_destination", + title: "Camera substream", + description: "Which camera-side stream to pull. 'auto' walks low-latency-first and picks the first that resolves (typically a low-fps preview substream — ~5-8 fps). Pin to medium-resolution or remote-recorder to force a higher-fps stream at the cost of larger frames + latency.", + choices: ["auto", "low-resolution", "medium-resolution", "local", "remote", "remote-recorder"], + value: this.streamDestination, + } as any, + { + group: "Display", key: "jpeg_quality", title: "JPEG quality (1–31, lower = better)", description: "ffmpeg mjpeg encoder -q:v. 1 ≈ visually lossless (~140KB at panel-native), 5 ≈ good (~70KB), 10+ noticeably lossy. Default 1.", @@ -759,7 +777,7 @@ class ScryptedViewportProvider extends ScryptedDeviceBase // firmware still rejects 480x800, the rotation didn't apply // (very rare ffmpeg build issue) and we'd need to look at // installed ffmpeg version. - this.console.log(`stream "${v.name}": orientation=${v.orientation} panel=${panelW}x${panelH} vf="${vf}"`); + // (stream config log emitted after substream selection below) // Pull the camera's video stream, convert to ffmpeg input args, and // pipe through a single ffmpeg child: input → scale(lanczos) → @@ -777,12 +795,17 @@ class ScryptedViewportProvider extends ScryptedDeviceBase // Walk substreams from lowest-latency → highest-latency and // take the first one that resolves. let stream: any; - const destOrder = ["low-resolution", "medium-resolution", "local", "remote", "remote-recorder"]; + let pickedDest = "(default)"; + const userPref = v.streamDestination; + const destOrder = userPref === "auto" + ? ["low-resolution", "medium-resolution", "local", "remote", "remote-recorder"] + : [userPref]; for (const destination of destOrder) { - try { stream = await cam.getVideoStream({ destination }); break; } + try { stream = await cam.getVideoStream({ destination }); pickedDest = destination; break; } catch { /* try next */ } } - if (!stream) stream = await cam.getVideoStream(); + if (!stream) { stream = await cam.getVideoStream(); pickedDest = "(camera-default)"; } + this.console.log(`stream "${v.name}": orientation=${v.orientation} panel=${panelW}x${panelH} vf="${vf}" substream=${pickedDest}`); const ffmpegInputBuf: Buffer = await mediaManager.convertMediaObjectToBuffer( stream, "x-scrypted/x-ffmpeg-input"); let ffmpegInput: any; @@ -1022,11 +1045,17 @@ class ScryptedViewportProvider extends ScryptedDeviceBase } } - // First-paint fast path. takePicture → quick ffmpeg one-shot to - // transpose+scale to panel-native dims → POST /frame. Shares the - // viewport's frameSeq counter so a slow snapshot can't paint over - // a faster stream frame — whichever loses the race gets stale- - // dropped by the firmware. + // First-paint fast path. takePicture → resize/rotate to panel + // native → POST /frame. Tries three transforms in order of cost: + // 1. sharp — libvips bindings, ~5-15ms per image, handles + // resize + rotate in one call. Not always present + // in Scrypted's plugin sandbox. + // 2. mediaManager.convertMediaObjectToBuffer with size hint — + // Scrypted's native converter (often vips-backed). + // Resize-capable; rotation support varies. We only + // use it for landscape (no rotate needed). + // 3. ffmpeg one-shot — old slow path, ~500-700ms cold start. + // Always works; the safety net. private async pushSnapshot(v: Viewport, cam: any) { const t0 = Date.now(); let mo: any; @@ -1036,43 +1065,76 @@ class ScryptedViewportProvider extends ScryptedDeviceBase const srcJpeg: Buffer = await mediaManager.convertMediaObjectToBuffer(mo, "image/jpeg"); if (!srcJpeg || srcJpeg.length < 4) return; + const tDecoded = Date.now(); // Cached dims from prior /state read; falls back to 800x480. const panelW = parseInt(v.storage.getItem("panel_w") || "0", 10) || 800; const panelH = parseInt(v.storage.getItem("panel_h") || "0", 10) || 480; - const vf = v.orientation === "portrait" - ? `transpose=1,scale=${panelW}:${panelH}:flags=lanczos,setsar=1` - : `scale=${panelW}:${panelH}:flags=lanczos,setsar=1`; + const needsRotate = v.orientation === "portrait"; - const { spawn } = require("child_process"); - const ffmpegPath = - (mediaManager.getFFmpegPath ? await mediaManager.getFFmpegPath() : undefined) || - "ffmpeg"; + let transformed: Buffer = Buffer.alloc(0); + let path = ""; - const transformed: Buffer = await new Promise<Buffer>((resolve, reject) => { - const p = spawn(ffmpegPath, [ - "-hide_banner", "-loglevel", "error", - "-f", "image2pipe", "-i", "pipe:0", - "-vf", vf, - "-frames:v", "1", - "-c:v", "mjpeg", "-q:v", String(v.jpegQuality), - "-f", "image2pipe", "pipe:1", - ]); - const chunks: Buffer[] = []; - p.stdout.on("data", (c: Buffer) => chunks.push(c)); - p.on("close", (code: number) => { - if (code !== 0) reject(new Error(`ffmpeg snapshot exit ${code}`)); - else resolve(Buffer.concat(chunks)); - }); - p.on("error", reject); - p.stdin.on("error", () => {}); // suppress EPIPE if ffmpeg fails early - p.stdin.end(srcJpeg); - // Hard cap so a stuck ffmpeg can't delay the main stream - // (which is starting in parallel anyway). - setTimeout(() => { try { p.kill("SIGTERM"); } catch {} }, 2000); - }).catch(() => Buffer.alloc(0)); + // Path 1: sharp. require()-fail caught at the boundary so a + // missing native module just falls through. + if (!transformed.length) { + try { + const sharp = require("sharp"); + let img = sharp(srcJpeg, { failOnError: false }); + if (needsRotate) img = img.rotate(90); + transformed = await img + .resize(panelW, panelH, { fit: "fill", kernel: "lanczos3" }) + .jpeg({ quality: Math.max(50, 100 - v.jpegQuality * 3) }) + .toBuffer(); + path = "sharp"; + } catch { /* fall through */ } + } + + // Path 2: Scrypted's native converter. Only used for landscape + // because the mime-parameter spec has no documented rotation + // and most implementations don't support it. + if (!transformed.length && !needsRotate) { + try { + transformed = await mediaManager.convertMediaObjectToBuffer( + mo, `image/jpeg;width=${panelW};height=${panelH}`); + if (transformed?.length) path = "media-mgr"; + } catch { /* fall through */ } + } + + // Path 3: ffmpeg fallback. The slow ~500ms cold-start path. + if (!transformed.length) { + const vf = needsRotate + ? `transpose=1,scale=${panelW}:${panelH}:flags=lanczos,setsar=1` + : `scale=${panelW}:${panelH}:flags=lanczos,setsar=1`; + const { spawn } = require("child_process"); + const ffmpegPath = + (mediaManager.getFFmpegPath ? await mediaManager.getFFmpegPath() : undefined) || + "ffmpeg"; + transformed = await new Promise<Buffer>((resolve, reject) => { + const p = spawn(ffmpegPath, [ + "-hide_banner", "-loglevel", "error", + "-f", "image2pipe", "-i", "pipe:0", + "-vf", vf, + "-frames:v", "1", + "-c:v", "mjpeg", "-q:v", String(v.jpegQuality), + "-f", "image2pipe", "pipe:1", + ]); + const chunks: Buffer[] = []; + p.stdout.on("data", (c: Buffer) => chunks.push(c)); + p.on("close", (code: number) => { + if (code !== 0) reject(new Error(`ffmpeg snapshot exit ${code}`)); + else resolve(Buffer.concat(chunks)); + }); + p.on("error", reject); + p.stdin.on("error", () => {}); + p.stdin.end(srcJpeg); + setTimeout(() => { try { p.kill("SIGTERM"); } catch {} }, 2000); + }).catch(() => Buffer.alloc(0)); + if (transformed.length) path = "ffmpeg"; + } if (transformed.length < 4) return; + const tTransformed = Date.now(); const seq = (this.frameSeq.get(v.nativeId!) || 0) + 1; this.frameSeq.set(v.nativeId!, seq); @@ -1084,8 +1146,13 @@ class ScryptedViewportProvider extends ScryptedDeviceBase signal: AbortSignal.timeout(2000), }); await res.text().catch(() => ""); - const wasStale = res.headers.get("X-Frame-Drop") === "stale-seq"; - this.console.log(`snapshot "${v.name}": ${Date.now() - t0}ms total (${(transformed.length / 1024).toFixed(0)}KB)${wasStale ? " — beaten by stream frame" : ""}`); + const tPosted = Date.now(); + const wasStale = res.headers.get("X-Frame-Drop") === "stale-seq"; + this.console.log( + `snapshot "${v.name}" via ${path}: ${tPosted - t0}ms total ` + + `(takePicture=${tDecoded - t0}ms transform=${tTransformed - tDecoded}ms ` + + `post=${tPosted - tTransformed}ms, ${(transformed.length / 1024).toFixed(0)}KB)` + + (wasStale ? " — beaten by stream frame" : "")); } catch { /* stream is starting anyway */ } } |