H264-specific issue when subscribing to a WHIP ingress video track from Python. #592

{
  "input_type": 1,
  "name": "gst-whip-test",
  "room_name": "test-room",
  "participant_identity": "camera-1",
  "participant_name": "Camera 1",
  "enable_transcoding": false
}

async def monitor_video_frames(video_input, participant_identity: str) -> None:
    frame_count = 0
    started_at = time.monotonic()
    last_report_at = started_at

    logger.info(
        "starting video frame monitor",
        extra={"participant": participant_identity},
    )

    async for _frame in video_input:
        frame_count += 1
        now = time.monotonic()

        if frame_count == 1:
            logger.info(
                "first video frame received",
                extra={"participant": participant_identity},
            )

        if now - last_report_at >= 5.0:
            elapsed = now - started_at
            fps = frame_count / elapsed if elapsed > 0 else 0.0
            logger.info(
                "video frames received",
                extra={
                    "participant": participant_identity,
                    "frames": frame_count,
                    "elapsed_s": round(elapsed, 2),
                    "avg_fps": round(fps, 2),
                },
            )
            last_report_at = now

        frame_monitor_task = asyncio.create_task(
            monitor_video_frames(session._room_io.video_input, ingress.identity)
        )

https://docs.livekit.io/recipes/gemini_live_vision/
uv add "livekit-agents[silero,google,images]" python-dotenv

gst-launch-1.0 \
      whipclientsink name=whip \
        signaller::whip-endpoint="https://.....whip.livekit.cloud/w/....." \
        video-caps="video/x-h264" \
      videotestsrc is-live=true pattern=ball ! \
      video/x-raw,width=1280,height=720,framerate=30/1 ! \
      videoconvert ! \
      x264enc tune=zerolatency speed-preset=veryfast bitrate=2000 key-int-max=30 ! \
      h264parse ! \
      queue ! \
      whip.