src.nth.io/

summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLuke Hoersten <[email protected]>2026-06-19 19:59:14 -0500
committerLuke Hoersten <[email protected]>2026-06-19 19:59:14 -0500
commitb97c2502b74277f9ea9dace97e8188201d5570fb (patch)
tree5a36c4f3f06ae3e310da5d42e3df84e989e91e3b
parentc89f832eb2c10f007ecf000165625c9953216a82 (diff)
streaming pivot: raw TCP data plane, drop per-frame HTTP entirely
Replaces the per-frame HTTP POST loop with a single long-lived TCP connection on port 81. The HTTP control plane (/state, /config, /frame for snapshot) stays unchanged. Wire protocol (big-endian, repeating until connection close): [4 bytes jpeg_len][4 bytes seq][jpeg_len bytes JPEG] Why --- The HTTP path hit a measured floor of ~37ms p50 with intermittent ~230ms p95 spikes that survived every Nagle/keep-alive fix attempt. Each frame paid: TCP setup (or pool churn), HTTP parsing, body recv, decode, paint, response write, response ACK. Streaming removes everything except recv + decode + paint. Firmware (main/stream_server.[ch], new) --------------------------------------- - TCP listen on configurable port (81) in its own FreeRTOS task, one client at a time (matches the one-stream-per-device model). - Per accepted socket: TCP_NODELAY on, then loop reading 8-byte header → jpeg body → through the existing jpeg_decoder + display paths. Same stale-seq guard as the HTTP /frame handler (reset per connection so each session starts at seq 1). - Frames received while asleep are still drained (to stay framed) but not painted. The HTTP control-plane POST /state {wake} resumes painting on the next frame. - Every 30 painted frames a structured serial log shows per-stage timing + sustained MB/s — replaces the cross-side Server-Timing header (no HTTP response to attach it to anymore). Scrypted side ------------- - net.createConnection({ host, port: 81, noDelay: true }) opened once per stream session. On disconnect/error/close we auto- reconnect after 500ms. - ffmpeg stdout demux writes [header][body] directly to the socket. sock.write() returning false sets a backpressured flag that drops incoming ffmpeg frames until 'drain' fires — natural TCP backpressure handles "firmware can't keep up" without us modeling it manually. - Stripped the entire fetch-based timing infrastructure (pushStreamFrame, fetchSamples, parseServerTiming, depth histogram, Server-Timing parser). Replaced with one stream-shaped log every 10s: fps + MB/s + socket.write p50/p95/max + drop count + backpressured flag. - pushSnapshot (one-shot first-paint) still uses the HTTP /frame endpoint — small and infrequent, not worth reworking. - postJSON still uses node:http for /state and /config. State machine status flags -------------------------- Extended boot-time flags array from 6 → 7 slots so the stream server's bring-up shows up alongside ETH/MDNS/HTTP. Layout is now E M H S D J T (was E M H D J T).
-rw-r--r--main/app_main.c7
-rw-r--r--main/stream_server.c245
-rw-r--r--main/stream_server.h18
-rw-r--r--scrypted/scrypted-viewport.ts310
4 files changed, 358 insertions, 222 deletions
diff --git a/main/app_main.c b/main/app_main.c
index 00a77bb..a6174c8 100644
--- a/main/app_main.c
+++ b/main/app_main.c
@@ -7,6 +7,7 @@
#include "nvs_config.h"
#include "state_client.h"
#include "state_machine.h"
+#include "stream_server.h"
#include "touch.h"
#include "viewport_state.h"
@@ -52,7 +53,7 @@ void app_main(void)
// don't get a DHCP lease. mDNS + HTTP advertise / bind anyway and will
// start serving the moment the link comes up.
// ------------------------------------------------------------------
- char flags[7] = { '-','-','-','-','-','-', 0 }; // E M H D J T
+ char flags[8] = { '-','-','-','-','-','-','-', 0 }; // E M H S D J T
esp_err_t eth_err = net_eth_init();
mark(eth_err, 'E', &flags[0]);
@@ -71,6 +72,10 @@ void app_main(void)
ESP_ERROR_CHECK(state_client_init());
mark(mdns_service_start(), 'M', &flags[1]);
mark(http_api_start(), 'H', &flags[2]);
+ // Data plane: raw TCP on :81 for back-to-back JPEG streaming,
+ // bypassing per-frame HTTP overhead. /frame HTTP stays alive for
+ // the snapshot one-shot and for curl debugging.
+ mark(stream_server_start(81), 'S', &flags[3]);
// ------------------------------------------------------------------
// Display + I²C-bound peripherals run on their own task. ESP-IDF's
diff --git a/main/stream_server.c b/main/stream_server.c
new file mode 100644
index 0000000..cebe3cb
--- /dev/null
+++ b/main/stream_server.c
@@ -0,0 +1,245 @@
+#include "stream_server.h"
+
+#include <errno.h>
+#include <string.h>
+
+#include "esp_log.h"
+#include "esp_timer.h"
+#include "freertos/FreeRTOS.h"
+#include "freertos/task.h"
+#include "lwip/netdb.h"
+#include "lwip/sockets.h"
+
+#include "display.h"
+#include "jpeg_decoder.h"
+#include "state_machine.h"
+#include "viewport_state.h"
+
+static const char *TAG = "stream";
+
+#define HEADER_BYTES 8 // 4 jpeg_len + 4 seq
+
+static uint16_t s_port;
+
+// recv() in a loop until n bytes are read or the connection drops.
+// Returns ESP_OK on full read, ESP_FAIL on EOF or socket error.
+static esp_err_t read_n(int fd, void *buf, size_t n)
+{
+ uint8_t *p = (uint8_t *)buf;
+ size_t got = 0;
+ while (got < n) {
+ ssize_t r = recv(fd, p + got, n - got, 0);
+ if (r <= 0) return ESP_FAIL;
+ got += (size_t)r;
+ }
+ return ESP_OK;
+}
+
+// Read and discard exactly n bytes — used to stay framed when we
+// have to skip a frame (decoder busy, asleep, etc) without dropping
+// the connection.
+static esp_err_t drain_n(int fd, size_t n)
+{
+ uint8_t scratch[256];
+ while (n > 0) {
+ size_t want = n > sizeof(scratch) ? sizeof(scratch) : n;
+ ssize_t r = recv(fd, scratch, want, 0);
+ if (r <= 0) return ESP_FAIL;
+ n -= (size_t)r;
+ }
+ return ESP_OK;
+}
+
+// One client owns the decoder for the lifetime of the connection.
+// Loops: header → body → (decode + paint OR skip) → repeat.
+static void handle_client(int fd, const char *peer)
+{
+ // NODELAY on the accepted socket so our outbound (response-less)
+ // ACKs flow immediately. There's no app-layer "response" in this
+ // protocol so we have to make sure TCP doesn't withhold ACKs
+ // waiting to piggyback on data we'll never send.
+ int one = 1;
+ (void)setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &one, sizeof(one));
+
+ // Sequence counter is per-connection — Scrypted resets to 1 on
+ // each new socket so we follow.
+ uint32_t last_painted_seq = 0;
+ uint64_t frames_decoded = 0;
+ int64_t t_window_start = esp_timer_get_time();
+ uint64_t bytes_in_window = 0;
+
+ while (1) {
+ uint8_t hdr[HEADER_BYTES];
+ if (read_n(fd, hdr, HEADER_BYTES) != ESP_OK) {
+ ESP_LOGI(TAG, "client %s disconnected (header read)", peer);
+ return;
+ }
+ uint32_t jpeg_len = ((uint32_t)hdr[0] << 24) | ((uint32_t)hdr[1] << 16)
+ | ((uint32_t)hdr[2] << 8) | (uint32_t)hdr[3];
+ uint32_t seq = ((uint32_t)hdr[4] << 24) | ((uint32_t)hdr[5] << 16)
+ | ((uint32_t)hdr[6] << 8) | (uint32_t)hdr[7];
+
+ if (jpeg_len == 0 || jpeg_len > JPEG_DECODER_MAX_INPUT_BYTES) {
+ ESP_LOGW(TAG, "bad frame length %u from %s — closing connection",
+ (unsigned)jpeg_len, peer);
+ return;
+ }
+
+ int64_t t_entry = esp_timer_get_time();
+
+ if (!jpeg_decoder_try_lock(2000)) {
+ ESP_LOGW(TAG, "decoder busy 2s — skipping seq %u", (unsigned)seq);
+ if (drain_n(fd, jpeg_len) != ESP_OK) return;
+ continue;
+ }
+
+ uint8_t *in = jpeg_decoder_input_buffer();
+ if (read_n(fd, in, jpeg_len) != ESP_OK) {
+ jpeg_decoder_unlock();
+ ESP_LOGI(TAG, "client %s disconnected (body read)", peer);
+ return;
+ }
+ int64_t t_recv = esp_timer_get_time();
+ bytes_in_window += jpeg_len;
+
+ // While asleep we still drain frames (to stay in sync) but
+ // skip the decode + paint. The state machine will wake on a
+ // POST /state {wake} from the control plane.
+ if (state_machine_current() != VIEWPORT_STATE_AWAKE) {
+ jpeg_decoder_unlock();
+ continue;
+ }
+
+ // Stale-seq guard. Each socket starts at 0 so the first
+ // frame (seq=1) always paints.
+ if (seq != 0 && seq <= last_painted_seq) {
+ jpeg_decoder_unlock();
+ continue;
+ }
+
+ size_t back_size = 0;
+ void *back = display_back_buffer(&back_size);
+ uint16_t w = 0, h = 0;
+ if (jpeg_decoder_decode(jpeg_len, back, back_size, &w, &h) != ESP_OK) {
+ viewport_state_lock();
+ viewport_state_get()->decode_errors++;
+ viewport_state_unlock();
+ jpeg_decoder_unlock();
+ continue;
+ }
+ if (w != VIEWPORT_PANEL_WIDTH || h != VIEWPORT_PANEL_HEIGHT) {
+ viewport_state_lock();
+ viewport_state_get()->decode_errors++;
+ viewport_state_unlock();
+ jpeg_decoder_unlock();
+ ESP_LOGW(TAG, "dim mismatch seq=%u: expected %ux%u got %ux%u",
+ (unsigned)seq, VIEWPORT_PANEL_WIDTH, VIEWPORT_PANEL_HEIGHT, w, h);
+ continue;
+ }
+ int64_t t_decode = esp_timer_get_time();
+ if (display_flip_back_buffer() != ESP_OK) {
+ jpeg_decoder_unlock();
+ continue;
+ }
+ int64_t t_paint = esp_timer_get_time();
+
+ viewport_state_lock();
+ viewport_state_t *st = viewport_state_get();
+ st->frames_received++;
+ st->last_frame_us = esp_timer_get_time();
+ viewport_state_unlock();
+
+ last_painted_seq = seq;
+ jpeg_decoder_unlock();
+ state_machine_frame_painted();
+ frames_decoded++;
+
+ // Every 30 frames log a per-stage breakdown + window throughput
+ // — same shape as the old /frame log so the serial output stays
+ // useful for debugging without the script side.
+ if (frames_decoded % 30 == 0) {
+ int64_t now = esp_timer_get_time();
+ double win_s = (now - t_window_start) / 1.0e6;
+ double mb_per_s = (win_s > 0)
+ ? ((double)bytes_in_window / win_s) / (1024.0 * 1024.0) : 0.0;
+ ESP_LOGI(TAG,
+ "frame %llu seq=%u: recv=%lldus dec=%lldus paint=%lldus "
+ "total=%lldus (jpeg=%uKB) window=%llumiB/%.1fs (%.2fMB/s)",
+ (unsigned long long)frames_decoded, (unsigned)seq,
+ (long long)(t_recv - t_entry),
+ (long long)(t_decode - t_recv),
+ (long long)(t_paint - t_decode),
+ (long long)(t_paint - t_entry),
+ (unsigned)(jpeg_len / 1024),
+ (unsigned long long)(bytes_in_window / (1024 * 1024)),
+ win_s, mb_per_s);
+ t_window_start = now;
+ bytes_in_window = 0;
+ }
+ }
+}
+
+static void accept_task(void *arg)
+{
+ (void)arg;
+ int listen_sock = -1;
+
+ while (1) {
+ if (listen_sock < 0) {
+ listen_sock = socket(AF_INET, SOCK_STREAM, 0);
+ if (listen_sock < 0) {
+ ESP_LOGE(TAG, "socket() failed: errno=%d", errno);
+ vTaskDelay(pdMS_TO_TICKS(1000));
+ continue;
+ }
+ int reuse = 1;
+ setsockopt(listen_sock, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse));
+
+ struct sockaddr_in addr = {
+ .sin_family = AF_INET,
+ .sin_addr.s_addr = htonl(INADDR_ANY),
+ .sin_port = htons(s_port),
+ };
+ if (bind(listen_sock, (struct sockaddr *)&addr, sizeof(addr)) < 0) {
+ ESP_LOGE(TAG, "bind(%d) failed: errno=%d", s_port, errno);
+ close(listen_sock);
+ listen_sock = -1;
+ vTaskDelay(pdMS_TO_TICKS(1000));
+ continue;
+ }
+ if (listen(listen_sock, 1) < 0) {
+ ESP_LOGE(TAG, "listen() failed: errno=%d", errno);
+ close(listen_sock);
+ listen_sock = -1;
+ vTaskDelay(pdMS_TO_TICKS(1000));
+ continue;
+ }
+ ESP_LOGI(TAG, "stream server listening on tcp/:%d", s_port);
+ }
+
+ struct sockaddr_in client;
+ socklen_t client_len = sizeof(client);
+ int fd = accept(listen_sock, (struct sockaddr *)&client, &client_len);
+ if (fd < 0) {
+ ESP_LOGW(TAG, "accept() failed: errno=%d", errno);
+ vTaskDelay(pdMS_TO_TICKS(100));
+ continue;
+ }
+
+ char ip[INET_ADDRSTRLEN];
+ inet_ntoa_r(client.sin_addr, ip, sizeof(ip));
+ ESP_LOGI(TAG, "client connected from %s:%u",
+ ip, (unsigned)ntohs(client.sin_port));
+
+ handle_client(fd, ip);
+ shutdown(fd, SHUT_RDWR);
+ close(fd);
+ }
+}
+
+esp_err_t stream_server_start(uint16_t port)
+{
+ s_port = port;
+ BaseType_t ok = xTaskCreate(accept_task, "stream", 8192, NULL, 5, NULL);
+ return (ok == pdPASS) ? ESP_OK : ESP_FAIL;
+}
diff --git a/main/stream_server.h b/main/stream_server.h
new file mode 100644
index 0000000..ef8a334
--- /dev/null
+++ b/main/stream_server.h
@@ -0,0 +1,18 @@
+#pragma once
+
+#include <stdint.h>
+#include "esp_err.h"
+
+// Raw-TCP frame ingestion server. Replaces the per-frame HTTP /frame
+// POST loop with a single long-lived TCP connection that streams
+// length+seq prefixed JPEGs back-to-back. Eliminates per-frame TCP
+// setup, HTTP parsing, and the 200ms Nagle/delayed-ACK deadlocks
+// that intermittently spiked the HTTP path to ~250ms wall.
+//
+// Wire protocol (one connection, repeating; all integers big-endian):
+// [4 bytes: jpeg_len ][4 bytes: seq][jpeg_len bytes: JPEG body]
+//
+// One client at a time. New client = previous client's seq counter
+// is reset (so each stream session starts fresh, and stale frames
+// from a previous reconnect can't paint over current ones).
+esp_err_t stream_server_start(uint16_t port);
diff --git a/scrypted/scrypted-viewport.ts b/scrypted/scrypted-viewport.ts
index f05ecbd..4059673 100644
--- a/scrypted/scrypted-viewport.ts
+++ b/scrypted/scrypted-viewport.ts
@@ -6,7 +6,7 @@
// short git hash of the commit that added this constant — if the
// hash in the log doesn't match the HEAD this file came from, the
// Scrypted Script editor is still on stale code.
-const SCRIPT_VERSION = "e88aa51";
+const SCRIPT_VERSION = "pending";
//
// Architecture
// ------------
@@ -810,23 +810,62 @@ class ScryptedViewportProvider extends ScryptedDeviceBase
const abort = new AbortController();
- // Single-flight: only one POST /frame in flight at a time. The
- // firmware's JPEG decoder mutex returns 503 otherwise; we just
- // drop the spare frames silently — natural rate-limiter.
- // Allow up to MAX_INFLIGHT concurrent /frame POSTs so that the
- // network upload of frame N+1 overlaps with the firmware's
- // decode+paint of frame N. The firmware's JPEG decoder mutex
- // serialises the decode itself; this only buys us the upload
- // overlap (≈ half of total /frame wall time). Keep this in
- // sync with cfg.max_open_sockets in main/http_api.c — that
- // sits at 4 (2 for streaming + 2 spare for /state, /config).
- const MAX_INFLIGHT = 2;
- let inFlight = 0;
+ // ── DATA PLANE: raw TCP socket to firmware port 81 ────────────
+ // Replaces per-frame HTTP POSTs. One socket per stream session.
+ // Frame format on the wire (big-endian):
+ // [4 bytes jpeg_len][4 bytes seq][jpeg_len bytes JPEG body]
+ // We let TCP flow-control backpressure us naturally: if
+ // socket.write() returns false the kernel buffer is full —
+ // we drop incoming ffmpeg frames until 'drain' fires. No HTTP
+ // headers, no per-frame ACK round-trip, no Nagle/delayed-ACK
+ // dance, no httpd worker churn.
+ const net = require("net");
+ let sock: any = null;
+ let socketReady = false;
+ let socketBackpressured = false;
+ let seq = 0;
let droppedFrames = 0;
let sentFrames = 0;
+ let bytesSent = 0;
let lastLogUs = Date.now();
let workBuf: Buffer = Buffer.alloc(0);
+ // Latency probe — wall-clock from "ffmpeg emitted the JPEG"
+ // to "kernel accepted the socket.write". With the keep-alive
+ // socket + TCP_NODELAY this should be sub-millisecond steady
+ // state; visible-double-digit numbers here mean kernel send
+ // buffer is full (= firmware can't ingest fast enough).
+ const writeLatencies: number[] = [];
+
+ const openStreamSocket = () => {
+ if (abort.signal.aborted) return;
+ socketReady = false;
+ socketBackpressured = false;
+ sock = net.createConnection({
+ host: v.host,
+ port: 81,
+ noDelay: true, // TCP_NODELAY on the outbound socket
+ });
+ sock.on("connect", () => {
+ socketReady = true;
+ this.console.log(`stream "${v.name}": tcp/81 open`);
+ });
+ sock.on("drain", () => { socketBackpressured = false; });
+ sock.on("error", (e: Error) => {
+ this.console.warn(`stream "${v.name}" socket: ${e.message}`);
+ socketReady = false;
+ if (!abort.signal.aborted) setTimeout(openStreamSocket, 500);
+ });
+ sock.on("close", () => {
+ socketReady = false;
+ if (!abort.signal.aborted) setTimeout(openStreamSocket, 500);
+ });
+ };
+ openStreamSocket();
+ abort.signal.addEventListener("abort", () => {
+ try { sock?.destroy(); } catch {}
+ });
+
// Auto-restart accounting: cameras occasionally end their RTSP
// stream mid-event (network blip, source rotation, etc.) and
// ffmpeg exits clean. If the stream-timeout hasn't fired yet
@@ -875,19 +914,29 @@ class ScryptedViewportProvider extends ScryptedDeviceBase
workBuf = workBuf.subarray(eoi + 2);
if (frame.length < 4 || frame[0] !== 0xff || frame[1] !== 0xd8) continue;
- if (inFlight >= MAX_INFLIGHT) { droppedFrames++; continue; }
- const emitMs = Date.now();
- const depthAtQueue = inFlight; // BEFORE the increment
- inFlight++;
+ // Drop if the socket isn't connected yet (initial
+ // open) or if the kernel send buffer is full
+ // (firmware can't ingest as fast as ffmpeg emits).
+ // Frame is gone forever — TCP doesn't queue what
+ // we don't write.
+ if (!socketReady || socketBackpressured) {
+ droppedFrames++;
+ continue;
+ }
+ seq++;
+ const header = Buffer.alloc(8);
+ header.writeUInt32BE(frame.length, 0);
+ header.writeUInt32BE(seq, 4);
+ const t0 = Date.now();
+ // Single combined write avoids splitting header
+ // and body across two TCP packets — the firmware
+ // sees them as one contiguous segment when possible.
+ const ok = sock.write(Buffer.concat([header, frame]));
+ writeLatencies.push(Date.now() - t0);
+ if (writeLatencies.length > 200) writeLatencies.shift();
+ bytesSent += 8 + frame.length;
sentFrames++;
- this.pushStreamFrame(v, frame, abort, emitMs, depthAtQueue)
- .catch(e => {
- if (abort.signal.aborted) return;
- const err = (e as Error);
- const cause = (err as any)?.cause?.code || "";
- this.console.warn(`pushStreamFrame "${v.name}":`, err.message, cause ? `(${cause})` : "");
- })
- .finally(() => { inFlight--; });
+ if (!ok) socketBackpressured = true;
}
});
@@ -935,23 +984,23 @@ class ScryptedViewportProvider extends ScryptedDeviceBase
const now = Date.now();
const window = (now - lastLogUs) / 1000;
if (window > 0 && (sentFrames > 0 || droppedFrames > 0)) {
- const sentRate = sentFrames / window;
- const targetRate = 1000 / v.frameIntervalMs;
- const dropRate = droppedFrames / window;
- // Two distinct symptoms to call out:
- // - drops > 25% of target → backpressure: HTTP POST or
- // panel is the bottleneck, advice is to raise interval.
- // - delivered < 60% AND drops are negligible → ffmpeg's
- // fps filter / camera substream simply isn't producing
- // the requested rate. Raising interval won't help;
- // surface a different message.
- if (dropRate > targetRate * 0.25) {
- this.console.log(`"${v.name}": backpressure — delivered ~${sentRate.toFixed(1)} fps vs target ${targetRate.toFixed(1)} fps, dropped ~${dropRate.toFixed(1)} fps over ${window.toFixed(1)}s (POST stack can't keep up; raise frame_interval_ms slightly to flatten this)`);
- } else if (sentRate < targetRate * 0.6 && sentRate > 0) {
- this.console.log(`"${v.name}": source-limited — delivered ~${sentRate.toFixed(1)} fps vs target ${targetRate.toFixed(1)} fps over ${window.toFixed(1)}s (camera substream / ffmpeg fps filter producing below requested rate; raising frame_interval_ms won't help)`);
- }
+ const sentRate = sentFrames / window;
+ const dropRate = droppedFrames / window;
+ const mbPerSec = (bytesSent / window) / (1024 * 1024);
+ const sortedW = writeLatencies.slice().sort((a, b) => a - b);
+ const p50 = sortedW.length ? sortedW[Math.floor(sortedW.length * 0.5)] : 0;
+ const p95 = sortedW.length ? sortedW[Math.floor(sortedW.length * 0.95)] : 0;
+ const max = sortedW.length ? sortedW[sortedW.length - 1] : 0;
+ this.console.log(
+ `stream "${v.name}": ${sentRate.toFixed(1)} fps, ${mbPerSec.toFixed(2)} MB/s ` +
+ `over ${window.toFixed(1)}s ` +
+ `(drops=${droppedFrames}=${dropRate.toFixed(1)} fps) ` +
+ `socket.write p50=${p50}ms p95=${p95}ms max=${max}ms ` +
+ `backpressured=${socketBackpressured}`);
droppedFrames = 0;
sentFrames = 0;
+ bytesSent = 0;
+ writeLatencies.length = 0;
lastLogUs = now;
}
}, 10_000);
@@ -984,74 +1033,6 @@ class ScryptedViewportProvider extends ScryptedDeviceBase
}
}
- // Scrypted-side per-fetch timing. We log aggregated stats every
- // 10 fetches instead of a single-sample line — single samples were
- // hiding tail latency. Each fetch records four buckets:
- // emit→post : ms from ffmpeg-emitted-the-jpeg to fetch() called.
- // Nonzero means the inFlight queue gated us.
- // req : fetch() called → Response headers arrive. Body
- // upload + server processing + status line round-trip.
- // body-read : Response received → response body drained. Should
- // be near-zero (firmware returns empty 204/200/409).
- // wall : total of all three for sanity.
- // Plus inflight-at-queue (0/1/2) so we can see whether pipelining
- // is actually overlapping anything, and a stale-drop counter for
- // X-Frame-Drop responses from the firmware.
- private fetchCount = new Map<string, number>();
- private fetchSamples = new Map<string, {
- // Script-side wall-clock spans, ms.
- emit: number[]; // ffmpeg emit → fetch() call
- wall: number[]; // emit → fetch resolved (end-to-end)
- req: number[]; // fetch() → Response headers
- bread: number[]; // Response → body drained
- // Firmware-side spans from Server-Timing (already ms).
- fw_recv: number[]; // body bytes off the wire
- fw_dec: number[]; // hardware JPEG decode
- fw_paint:number[]; // back buffer flip
- fw_post: number[]; // state counters + unlock + resp headers
- fw_tot: number[]; // firmware total (recv + dec + paint + post)
- // Derived: time fetch() spent in flight that firmware DIDN'T
- // see. net_up = req - fw_tot ≈ TCP setup + body wire time
- // + httpd dispatch.
- net_up: number[];
- depths: { d0: number; d1: number; d2: number };
- staleDrops: number;
- }>();
-
- private bucketsFor(nid: string) {
- let s = this.fetchSamples.get(nid);
- if (!s) {
- s = { emit: [], wall: [], req: [], bread: [],
- fw_recv: [], fw_dec: [], fw_paint: [], fw_post: [], fw_tot: [],
- net_up: [],
- depths: { d0: 0, d1: 0, d2: 0 }, staleDrops: 0 };
- this.fetchSamples.set(nid, s);
- }
- return s;
- }
-
- // Parse a Server-Timing header like
- // recv;dur=12.3, dec;dur=4.5, paint;dur=0.1, post;dur=0.2, handle;dur=17.1
- // into a record of name→duration_ms. Robust to whitespace and
- // missing entries; returns {} on malformed input.
- private parseServerTiming(h: string | null): Record<string, number> {
- const out: Record<string, number> = {};
- if (!h) return out;
- for (const tok of h.split(",")) {
- const parts = tok.trim().split(";");
- const name = parts[0]?.trim();
- if (!name) continue;
- for (let i = 1; i < parts.length; i++) {
- const [k, v] = parts[i].trim().split("=");
- if (k === "dur") {
- const n = Number(v);
- if (Number.isFinite(n)) out[name] = n;
- }
- }
- }
- return out;
- }
-
// First-paint fast path. takePicture → quick ffmpeg one-shot to
// transpose+scale to panel-native dims → POST /frame. Shares the
// viewport's frameSeq counter so a slow snapshot can't paint over
@@ -1126,119 +1107,6 @@ class ScryptedViewportProvider extends ScryptedDeviceBase
// two stay in step.
private frameSeq = new Map<string, number>();
- private async pushStreamFrame(v: Viewport, jpeg: Buffer, abort: AbortController,
- emitMs: number, depthAtQueue: number) {
- if (abort.signal.aborted) return;
- const seq = (this.frameSeq.get(v.nativeId!) || 0) + 1;
- this.frameSeq.set(v.nativeId!, seq);
- const tFetchStart = Date.now();
- const res = await this.httpRequest({
- host: v.host, port: 80, path: "/frame", method: "POST",
- headers: {
- "Content-Type": "image/jpeg",
- "X-Frame-Seq": String(seq),
- "Content-Length": String(jpeg.length),
- },
- body: jpeg,
- timeoutMs: 10_000,
- abort: abort.signal,
- });
- const tHeaders = res.tHeaders;
- const tDone = res.tDone;
- const status = res.status;
- const hdrGet = (k: string) => {
- const v = res.headers[k.toLowerCase()];
- return Array.isArray(v) ? v[0] : v;
- };
- const wasStale = hdrGet("X-Frame-Drop") === "stale-seq";
- const fwTiming = this.parseServerTiming(hdrGet("Server-Timing") ?? null);
-
- const b = this.bucketsFor(v.nativeId!);
- const req_ms = tHeaders - tFetchStart;
- const fw_tot = fwTiming.handle ?? 0;
- b.emit.push (tFetchStart - emitMs);
- b.req.push (req_ms);
- b.bread.push(tDone - tHeaders);
- b.wall.push (tDone - emitMs);
- if (fwTiming.recv != null) b.fw_recv.push(fwTiming.recv);
- if (fwTiming.dec != null) b.fw_dec.push(fwTiming.dec);
- if (fwTiming.paint != null) b.fw_paint.push(fwTiming.paint);
- if (fwTiming.post != null) b.fw_post.push(fwTiming.post);
- if (fw_tot > 0) b.fw_tot.push(fw_tot);
- // net_up = the slice of req that the firmware DIDN'T see —
- // TCP handshake (when no keep-alive), body bytes on the wire,
- // and httpd dispatch from socket-readable to handler entry.
- // Can go slightly negative under clock skew; clamp at 0.
- if (fw_tot > 0) b.net_up.push(Math.max(0, req_ms - fw_tot));
- if (depthAtQueue === 0) b.depths.d0++;
- else if (depthAtQueue === 1) b.depths.d1++;
- else b.depths.d2++;
- if (wasStale) b.staleDrops++;
-
- const n = (this.fetchCount.get(v.nativeId!) || 0) + 1;
- this.fetchCount.set(v.nativeId!, n);
- if (n % 10 === 0) {
- const p = (arr: number[], q: number) => {
- if (!arr.length) return 0;
- const sorted = arr.slice().sort((a, b) => a - b);
- return sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * q))];
- };
- const stats = (arr: number[]) => {
- if (!arr.length) return null;
- let mn = arr[0], mx = arr[0];
- for (const v of arr) { if (v < mn) mn = v; if (v > mx) mx = v; }
- return { min: mn, p50: p(arr, 0.5), p95: p(arr, 0.95), max: mx };
- };
- const fmt = (arr: number[]) => {
- const s = stats(arr);
- if (!s) return `(no data)`;
- return `min=${s.min.toFixed(1)} p50=${s.p50.toFixed(1)} p95=${s.p95.toFixed(1)} max=${s.max.toFixed(1)}ms`;
- };
- // Identify the worst-wall sample in the window and decompose
- // it into its own stages — answers "what made the slowest
- // frame slow?" without us having to guess from percentiles.
- let worstIdx = 0;
- for (let i = 1; i < b.wall.length; i++) if (b.wall[i] > b.wall[worstIdx]) worstIdx = i;
- const at = (arr: number[], i: number) =>
- (i < arr.length && arr[i] != null) ? arr[i].toFixed(1) : "n/a";
- this.console.log(
- `fetch "${v.name}" #${n} (jpeg=${(jpeg.length / 1024).toFixed(0)}KB)\n` +
- ` wall ${fmt(b.wall)}\n` +
- ` emit→post ${fmt(b.emit)} (queue wait before fetch() called)\n` +
- ` req ${fmt(b.req)} (fetch start → Response headers)\n` +
- ` net_up ${fmt(b.net_up)} (req − fw_total: TCP setup + body wire + dispatch)\n` +
- ` fw_recv ${fmt(b.fw_recv)} (firmware body read off the wire)\n` +
- ` fw_dec ${fmt(b.fw_dec)} (hardware JPEG → BGR888)\n` +
- ` fw_paint ${fmt(b.fw_paint)} (backbuffer flip)\n` +
- ` fw_post ${fmt(b.fw_post)} (state counters + unlock)\n` +
- ` body-read ${fmt(b.bread)} (Response → drained)\n` +
- ` inflight d0=${b.depths.d0} d1=${b.depths.d1} d2=${b.depths.d2} (queue depth at fetch start)\n` +
- ` worst (#${worstIdx + 1}/${b.wall.length} in window): wall=${at(b.wall, worstIdx)}ms = ` +
- `emit→post ${at(b.emit, worstIdx)} + net_up ${at(b.net_up, worstIdx)} + ` +
- `fw_recv ${at(b.fw_recv, worstIdx)} + fw_dec ${at(b.fw_dec, worstIdx)} + ` +
- `fw_paint ${at(b.fw_paint, worstIdx)} + fw_post ${at(b.fw_post, worstIdx)} + ` +
- `body-read ${at(b.bread, worstIdx)}\n` +
- ` stale-drops=${b.staleDrops}`);
- this.fetchSamples.delete(v.nativeId!); // reset window
- }
- if (status === 409) {
- this.console.log(`"${v.name}" returned 409 — device went to sleep, stopping stream`);
- this.stopStream(v.name, /*sendSleep=*/ false);
- } else if (status === 400) {
- // Body already drained; we lost the reason text. Log status
- // alone — repeat 400s in steady state should be rare.
- this.console.warn(`"${v.name}" /frame -> 400`);
- } else if ((status < 200 || status >= 300) && !wasStale) {
- this.console.warn(`"${v.name}" /frame -> ${status}`);
- }
- // Do NOT reset the idle timer on successful paint. The timer is
- // anchored to the triggering CAMERA EVENT, not to "frames are
- // flowing" — otherwise a continuously-streaming source means the
- // stream never times out. Repeated events naturally cancel-and-
- // restart the stream via startStream → stopStream(false), which
- // covers the "still active" case.
- }
-
private findByName(name: string): Viewport | undefined {
for (const v of this.viewports.values()) if (v.name === name) return v;
return undefined;