LAN Realtime Detection Overlay

{"action":"init","width":1920,"height":1080,"codec":"h264","framerate":15}

[1 byte type] [3 bytes length, big-endian] [N bytes value]

┌──────────────────────────────────────────────────┐
│ TIMESTAMP (0x02): 4 + 8 = 12 bytes               │  always present
├──────────────────────────────────────────────────┤
│ PTS_US (0x03): 4 + 8 = 12 bytes                  │  optional
├──────────────────────────────────────────────────┤
│ AI_DETECTIONS (0x04): 4 + N bytes                │  only when a new
│                                                  │  detection is available
├──────────────────────────────────────────────────┤
│ frame-data (0x00 or 0x01): 4 + N bytes           │  always last;
│                                                  │  value is raw H.264
└──────────────────────────────────────────────────┘

type ParsedFrame = {
  timestampMs: number | null;
  ptsUs: number | null;
  detectionJson: string | null;
  isKeyframe: boolean;
  h264Data: Uint8Array;
};

export function parseEncapHeader(buffer: ArrayBuffer): ParsedFrame {
  const view = new DataView(buffer);
  const bytes = new Uint8Array(buffer);
  let offset = 0;
  let timestampMs: number | null = null;
  let ptsUs: number | null = null;
  let detectionJson: string | null = null;

  while (offset + 4 <= buffer.byteLength) {
    const type = view.getUint8(offset);
    const len =
      (view.getUint8(offset + 1) << 16) |
      (view.getUint8(offset + 2) << 8) |
      view.getUint8(offset + 3);
    const valueStart = offset + 4;

    if (type === 0x00 || type === 0x01) {
      return {
        timestampMs,
        ptsUs,
        detectionJson,
        isKeyframe: type === 0x00,
        h264Data: bytes.subarray(valueStart, valueStart + len),
      };
    }

    if (type === 0x02) {
      const hi = view.getUint32(valueStart);
      const lo = view.getUint32(valueStart + 4);
      timestampMs = hi * 0x100000000 + lo;
    } else if (type === 0x03) {
      const hi = view.getUint32(valueStart);
      const lo = view.getUint32(valueStart + 4);
      ptsUs = hi * 0x100000000 + lo;
    } else if (type === 0x04) {
      detectionJson = new TextDecoder().decode(
        bytes.subarray(valueStart, valueStart + len)
      );
    }
    // Unknown types are skipped silently.

    offset = valueStart + len;
  }

  throw new Error("Encapsulation header missing frame-data TLV");
}

[
  {
    "t": 0,
    "c": 8500,
    "id": 3,
    "b": [1200, 3400, 4500, 8900],
    "rs": 2.5,
    "clr": {"red": 4000, "blue": 6000},
    "tight_crop_xxyy": [1500, 4200, 3700, 8500]
  }
]

const TYPE_COLORS = {
  0: "#00ff00", // Human    — green
  1: "#0088ff", // Vehicle  — blue
  2: "#ff00ff", // Face     — magenta
  3: "#ffff00", // LPR      — yellow
  4: "#00ffff", // Pose     — cyan
  5: "#ff8800", // CLIP     — orange
};

const TYPE_LABELS = ["Human", "Vehicle", "Face", "LPR", "Pose", "CLIP"];

export function drawDetections(ctx, canvasWidth, canvasHeight, detections) {
  ctx.clearRect(0, 0, canvasWidth, canvasHeight);

  for (const det of detections) {
    const [left, top, right, bottom] = det.b;
    const x = (left / 10000) * canvasWidth;
    const y = (top / 10000) * canvasHeight;
    const w = ((right - left) / 10000) * canvasWidth;
    const h = ((bottom - top) / 10000) * canvasHeight;

    ctx.strokeStyle = TYPE_COLORS[det.t] ?? "#ffffff";
    ctx.lineWidth = 2;
    ctx.strokeRect(x, y, w, h);

    const conf = Math.round(det.c / 100);
    const label = `${TYPE_LABELS[det.t] ?? "Unknown"} ${conf}% #${det.id}`;
    ctx.fillStyle = ctx.strokeStyle;
    ctx.font = "12px monospace";
    ctx.fillText(label, x, Math.max(10, y - 4));
  }
}

import { RhombusRealtimePlayer } from "@rhombussystems/react";
import { useEffect, useRef, useState } from "react";
import { parseEncapHeader } from "./parseEncapHeader"; // from earlier in this guide
import { drawDetections } from "./drawDetections";     // from earlier in this guide

type Props = {
  cameraUuid: string;
  /** WebSocket URL for this camera (from getMediaUris + auth append) */
  detectionWsUrl: string;
  /** Optional: how long to keep boxes after the last update */
  detectionTtlMs?: number;
};

export function RhombusRealtimePlayerWithDetections({
  cameraUuid,
  detectionWsUrl,
  detectionTtlMs = 2000,
}: Props) {
  const canvasRef = useRef<HTMLCanvasElement>(null);
  const detectionsRef = useRef<any[]>([]);
  const lastTsRef = useRef(0);
  const [size, setSize] = useState({ width: 1920, height: 1080 });

  useEffect(() => {
    const ws = new WebSocket(detectionWsUrl);
    ws.binaryType = "arraybuffer";

    ws.onmessage = (event) => {
      // The server sends one text init message before any binary frames.
      if (typeof event.data === "string") {
        try {
          const init = JSON.parse(event.data);
          if (init.action === "init" && init.width && init.height) {
            setSize({ width: init.width, height: init.height });
          }
        } catch {
          // Ignore non-JSON text messages.
        }
        return;
      }

      try {
        const { detectionJson } = parseEncapHeader(event.data);
        if (!detectionJson) return;
        const dets = JSON.parse(detectionJson);
        detectionsRef.current = dets;
        lastTsRef.current = Date.now();
      } catch (err) {
        console.warn("Failed to parse encap header", err);
      }
    };

    return () => ws.close();
  }, [detectionWsUrl]);

  useEffect(() => {
    let raf = 0;

    const tick = () => {
      const canvas = canvasRef.current;
      if (canvas) {
        const ctx = canvas.getContext("2d");
        if (ctx) {
          const fresh = Date.now() - lastTsRef.current < detectionTtlMs;
          drawDetections(
            ctx,
            canvas.width,
            canvas.height,
            fresh ? detectionsRef.current : []
          );
        }
      }
      raf = requestAnimationFrame(tick);
    };

    raf = requestAnimationFrame(tick);
    return () => cancelAnimationFrame(raf);
  }, [detectionTtlMs]);

  return (
    <div style={{ position: "relative", width: size.width, height: size.height }}>
      <RhombusRealtimePlayer
        cameraUuid={cameraUuid}
        connectionMode="wan"
      />
      <canvas
        ref={canvasRef}
        width={size.width}
        height={size.height}
        style={{
          position: "absolute",
          inset: 0,
          width: "100%",
          height: "100%",
          pointerEvents: "none",
        }}
      />
    </div>
  );
}

<canvas id="overlay" width="1920" height="1080"
  style="position:absolute; inset:0; pointer-events:none;"></canvas>

<script type="module">
  import { parseEncapHeader } from "./parseEncapHeader.js";
  import { drawDetections } from "./drawDetections.js";

  // Resolve detectionWsUrl from your backend after calling
  // /api/camera/getMediaUris and appending the federated token.
  const ws = new WebSocket(detectionWsUrl);
  ws.binaryType = "arraybuffer";

  const canvas = document.getElementById("overlay");
  const ctx = canvas.getContext("2d");
  let lastDetections = [];
  let lastTs = 0;
  const TTL_MS = 2000;

  ws.onmessage = (event) => {
    if (typeof event.data === "string") return; // skip init handshake
    const { detectionJson } = parseEncapHeader(event.data);
    if (!detectionJson) return;
    lastDetections = JSON.parse(detectionJson);
    lastTs = Date.now();
  };

  function tick() {
    const fresh = Date.now() - lastTs < TTL_MS;
    drawDetections(ctx, canvas.width, canvas.height, fresh ? lastDetections : []);
    requestAnimationFrame(tick);
  }
  requestAnimationFrame(tick);
</script>