import { useEffect, useRef, useState, useCallback } from 'react'; import { useSettingsStore } from '@/stores/settingsStore'; interface UseLocalPresenceOptions { enabled?: boolean; confidenceThreshold?: number; checkIntervalMs?: number; onPersonDetected?: () => void; onPersonCleared?: () => void; } /** * Hook for local presence detection using TensorFlow.js COCO-SSD model. * Uses the Kitchen_Panel go2rtc stream instead of direct webcam access * (since mediamtx uses the webcam for streaming). */ export function useLocalPresence({ enabled = true, confidenceThreshold = 0.5, checkIntervalMs = 2000, onPersonDetected, onPersonCleared, }: UseLocalPresenceOptions = {}) { const [isDetecting, setIsDetecting] = useState(false); const [hasPersonPresent, setHasPersonPresent] = useState(false); const [error, setError] = useState(null); const canvasRef = useRef(null); const imgRef = useRef(null); const modelRef = useRef(null); const intervalRef = useRef | null>(null); const noPersonCountRef = useRef(0); const wasPersonPresentRef = useRef(false); const go2rtcUrl = useSettingsStore((state) => state.config.go2rtcUrl); const NO_PERSON_THRESHOLD = 3; // Frames without person before clearing const detectPerson = useCallback(async () => { if (!modelRef.current || !canvasRef.current || !imgRef.current) return; const ctx = canvasRef.current.getContext('2d'); if (!ctx) return; try { // Fetch a frame from go2rtc MJPEG snapshot const snapshotUrl = `${go2rtcUrl}/api/frame.jpeg?src=Kitchen_Panel&t=${Date.now()}`; // Load image await new Promise((resolve, reject) => { if (!imgRef.current) return reject('No image element'); imgRef.current.crossOrigin = 'anonymous'; imgRef.current.onload = () => resolve(); imgRef.current.onerror = () => reject('Failed to load frame'); imgRef.current.src = snapshotUrl; }); // Draw to canvas ctx.drawImage(imgRef.current, 0, 0, 320, 240); // Run detection const predictions = await modelRef.current.detect(canvasRef.current); // Check for person with sufficient confidence const personDetection = predictions.find( (p: any) => p.class === 'person' && p.score >= confidenceThreshold ); if (personDetection) { noPersonCountRef.current = 0; if (!wasPersonPresentRef.current) { wasPersonPresentRef.current = true; setHasPersonPresent(true); console.log('Local presence: Person detected via go2rtc stream'); onPersonDetected?.(); } } else { noPersonCountRef.current++; if (wasPersonPresentRef.current && noPersonCountRef.current >= NO_PERSON_THRESHOLD) { wasPersonPresentRef.current = false; setHasPersonPresent(false); console.log('Local presence: Person cleared'); onPersonCleared?.(); } } } catch (err) { // Silently fail on individual frame errors - stream might be briefly unavailable console.debug('Detection frame error:', err); } }, [go2rtcUrl, confidenceThreshold, onPersonDetected, onPersonCleared]); const startDetection = useCallback(async () => { if (isDetecting) return; try { setError(null); // Dynamically import TensorFlow.js and COCO-SSD const tf = await import('@tensorflow/tfjs'); const cocoSsd = await import('@tensorflow-models/coco-ssd'); // Set backend await tf.setBackend('webgl'); await tf.ready(); // Load model (lite version for speed) console.log('Loading COCO-SSD model for presence detection...'); modelRef.current = await cocoSsd.load({ base: 'lite_mobilenet_v2', }); console.log('Model loaded'); // Create hidden image element for loading frames imgRef.current = document.createElement('img'); // Create canvas for processing canvasRef.current = document.createElement('canvas'); canvasRef.current.width = 320; canvasRef.current.height = 240; // Start detection loop setIsDetecting(true); intervalRef.current = setInterval(detectPerson, checkIntervalMs); console.log('Local presence detection started (using go2rtc Kitchen_Panel stream)'); } catch (err) { const message = err instanceof Error ? err.message : 'Failed to start presence detection'; console.error('Presence detection error:', message); setError(message); } }, [isDetecting, checkIntervalMs, detectPerson]); const stopDetection = useCallback(() => { if (intervalRef.current) { clearInterval(intervalRef.current); intervalRef.current = null; } canvasRef.current = null; imgRef.current = null; modelRef.current = null; setIsDetecting(false); setHasPersonPresent(false); wasPersonPresentRef.current = false; noPersonCountRef.current = 0; console.log('Local presence detection stopped'); }, []); // Start/stop based on enabled prop useEffect(() => { if (enabled) { startDetection(); } else { stopDetection(); } return () => { stopDetection(); }; }, [enabled, startDetection, stopDetection]); return { isDetecting, hasPersonPresent, error, startDetection, stopDetection, }; }