From c39ec4e954f275000a50b570a0a3362b6baed0f0 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 5 Mar 2026 12:27:52 -0600 Subject: [PATCH] Add Prometheus exporter for go2rtc and Frigate stream health Standalone Python exporter that polls go2rtc /api/streams and Frigate /api/stats every 15s, tracking bytes_recv deltas to detect stalled streams. Exposes metrics on port 9199 including stream_up, bytes_per_second, consumers, and camera_fps. --- monitoring/config.yaml | 20 ++++ monitoring/exporter.py | 184 ++++++++++++++++++++++++++++++++++++ monitoring/requirements.txt | 3 + 3 files changed, 207 insertions(+) create mode 100644 monitoring/config.yaml create mode 100644 monitoring/exporter.py create mode 100644 monitoring/requirements.txt diff --git a/monitoring/config.yaml b/monitoring/config.yaml new file mode 100644 index 0000000..90d9ae0 --- /dev/null +++ b/monitoring/config.yaml @@ -0,0 +1,20 @@ +go2rtc_url: "http://192.168.1.241:1985" +frigate_url: "http://192.168.1.241:5000" + +poll_interval: 15 +request_timeout: 5 +metrics_port: 9199 + +cameras: + - FPE + - Porch_Downstairs + - Front_Porch + - Driveway_door + - Street_side + - Backyard + - House_side + - Driveway + - BackDoor + - Parlor + - Livingroom + - WyzePanV3 diff --git a/monitoring/exporter.py b/monitoring/exporter.py new file mode 100644 index 0000000..19a0a95 --- /dev/null +++ b/monitoring/exporter.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +"""Prometheus exporter for go2rtc and Frigate camera stream health.""" + +import logging +import signal +import sys +import time +from pathlib import Path + +import requests +import yaml +from prometheus_client import Gauge, start_http_server + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", +) +log = logging.getLogger("go2rtc-exporter") + +# --- Metrics --- + +go2rtc_stream_up = Gauge( + "go2rtc_stream_up", + "Whether a go2rtc stream has active producers (1=up, 0=down)", + ["camera"], +) +go2rtc_stream_bytes_per_second = Gauge( + "go2rtc_stream_bytes_per_second", + "Bytes per second received by go2rtc producer", + ["camera"], +) +go2rtc_stream_consumers = Gauge( + "go2rtc_stream_consumers", + "Number of consumers connected to the stream", + ["camera"], +) +frigate_camera_fps = Gauge( + "frigate_camera_fps", + "Camera FPS reported by Frigate", + ["camera"], +) +go2rtc_up = Gauge( + "go2rtc_up", + "Whether go2rtc API is reachable (1=up, 0=down)", +) +frigate_up = Gauge( + "frigate_up", + "Whether Frigate API is reachable (1=up, 0=down)", +) + +# --- State --- + +prev_bytes: dict[str, int] = {} +prev_time: dict[str, float] = {} + + +def load_config(path: str) -> dict: + with open(path) as f: + return yaml.safe_load(f) + + +def poll_go2rtc(config: dict) -> None: + url = f"{config['go2rtc_url']}/api/streams" + timeout = config.get("request_timeout", 5) + cameras = config["cameras"] + + try: + resp = requests.get(url, timeout=timeout) + resp.raise_for_status() + streams = resp.json() + go2rtc_up.set(1) + except Exception as e: + log.warning("go2rtc unreachable: %s", e) + go2rtc_up.set(0) + for cam in cameras: + go2rtc_stream_up.labels(camera=cam).set(0) + go2rtc_stream_bytes_per_second.labels(camera=cam).set(0) + go2rtc_stream_consumers.labels(camera=cam).set(0) + return + + now = time.monotonic() + + for cam in cameras: + stream = streams.get(cam) + if stream is None: + go2rtc_stream_up.labels(camera=cam).set(0) + go2rtc_stream_bytes_per_second.labels(camera=cam).set(0) + go2rtc_stream_consumers.labels(camera=cam).set(0) + continue + + producers = stream.get("producers") or [] + consumers = stream.get("consumers") or [] + + # A stream is "up" if it has at least one producer + has_producer = len(producers) > 0 + go2rtc_stream_up.labels(camera=cam).set(1 if has_producer else 0) + go2rtc_stream_consumers.labels(camera=cam).set(len(consumers)) + + # Calculate bytes/sec from delta + total_bytes = sum(p.get("bytes_recv", 0) for p in producers) + + if cam in prev_bytes and cam in prev_time: + dt = now - prev_time[cam] + if dt > 0: + dbytes = total_bytes - prev_bytes[cam] + # Handle counter reset (go2rtc restart) + if dbytes < 0: + dbytes = total_bytes + bps = dbytes / dt + go2rtc_stream_bytes_per_second.labels(camera=cam).set(round(bps, 1)) + else: + go2rtc_stream_bytes_per_second.labels(camera=cam).set(0) + else: + # First poll — no delta yet + go2rtc_stream_bytes_per_second.labels(camera=cam).set(0) + + prev_bytes[cam] = total_bytes + prev_time[cam] = now + + +def poll_frigate(config: dict) -> None: + url = f"{config['frigate_url']}/api/stats" + timeout = config.get("request_timeout", 5) + cameras = config["cameras"] + + try: + resp = requests.get(url, timeout=timeout) + resp.raise_for_status() + stats = resp.json() + frigate_up.set(1) + except Exception as e: + log.warning("Frigate unreachable: %s", e) + frigate_up.set(0) + for cam in cameras: + frigate_camera_fps.labels(camera=cam).set(0) + return + + cam_stats = stats.get("cameras", {}) + for cam in cameras: + cs = cam_stats.get(cam) + if cs and isinstance(cs, dict): + frigate_camera_fps.labels(camera=cam).set(cs.get("camera_fps", 0)) + else: + frigate_camera_fps.labels(camera=cam).set(0) + + +def main() -> None: + config_path = Path(__file__).parent / "config.yaml" + if len(sys.argv) > 1: + config_path = Path(sys.argv[1]) + + config = load_config(str(config_path)) + port = config.get("metrics_port", 9199) + interval = config.get("poll_interval", 15) + + log.info("Starting go2rtc exporter on port %d (poll every %ds)", port, interval) + log.info("Monitoring cameras: %s", ", ".join(config["cameras"])) + + start_http_server(port) + + shutdown = False + + def handle_signal(signum, frame): + nonlocal shutdown + log.info("Received signal %d, shutting down", signum) + shutdown = True + + signal.signal(signal.SIGTERM, handle_signal) + signal.signal(signal.SIGINT, handle_signal) + + while not shutdown: + try: + poll_go2rtc(config) + poll_frigate(config) + except Exception: + log.exception("Unexpected error during poll") + time.sleep(interval) + + log.info("Exporter stopped") + + +if __name__ == "__main__": + main() diff --git a/monitoring/requirements.txt b/monitoring/requirements.txt new file mode 100644 index 0000000..459352e --- /dev/null +++ b/monitoring/requirements.txt @@ -0,0 +1,3 @@ +prometheus-client>=0.20.0 +requests>=2.31.0 +pyyaml>=6.0