From 2eb5fbf1126f5f92b121dcd89241ad798b7c3c9b Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Wed, 6 Nov 2024 06:59:33 -0700 Subject: [PATCH 1/9] Add more debug logs for preview and output (#14833) --- frigate/output/output.py | 10 ++++++++++ frigate/output/preview.py | 9 +++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/frigate/output/output.py b/frigate/output/output.py index 7d5b6d39a8..1859ebd69f 100644 --- a/frigate/output/output.py +++ b/frigate/output/output.py @@ -63,6 +63,7 @@ def receiveSignal(signalNumber, frame): birdseye: Optional[Birdseye] = None preview_recorders: dict[str, PreviewRecorder] = {} preview_write_times: dict[str, float] = {} + failed_frame_requests: dict[str, int] = {} move_preview_frames("cache") @@ -99,7 +100,16 @@ def receiveSignal(signalNumber, frame): if frame is None: logger.debug(f"Failed to get frame {frame_id} from SHM") + failed_frame_requests[camera] = failed_frame_requests.get(camera, 0) + 1 + + if failed_frame_requests[camera] > config.cameras[camera].detect.fps: + logger.warning( + f"Failed to retrieve many frames for {camera} from SHM, consider increasing SHM size if this continues." + ) + continue + else: + failed_frame_requests[camera] = 0 # send camera frame to ffmpeg process if websockets are connected if any( diff --git a/frigate/output/preview.py b/frigate/output/preview.py index a8915f688a..9eae6b7dea 100644 --- a/frigate/output/preview.py +++ b/frigate/output/preview.py @@ -154,6 +154,7 @@ def __init__(self, config: CameraConfig) -> None: self.start_time = 0 self.last_output_time = 0 self.output_frames = [] + if config.detect.width > config.detect.height: self.out_height = PREVIEW_HEIGHT self.out_width = ( @@ -274,7 +275,7 @@ def should_write_frame( return False - def write_frame_to_cache(self, frame_time: float, frame) -> None: + def write_frame_to_cache(self, frame_time: float, frame: np.ndarray) -> None: # resize yuv frame small_frame = np.zeros((self.out_height * 3 // 2, self.out_width), np.uint8) copy_yuv_to_position( @@ -303,7 +304,7 @@ def write_data( current_tracked_objects: list[dict[str, any]], motion_boxes: list[list[int]], frame_time: float, - frame, + frame: np.ndarray, ) -> bool: # check for updated record config _, updated_record_config = self.config_subscriber.check_for_update() @@ -332,6 +333,10 @@ def write_data( self.output_frames, self.requestor, ).start() + else: + logger.debug( + f"Not saving preview for {self.config.name} because there are no saved frames." + ) # reset frame cache self.segment_end = ( From bc371acb3effe1c51b938161e3430c1ca36e476a Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Wed, 6 Nov 2024 11:05:44 -0600 Subject: [PATCH 2/9] Cleanup batching (#14836) * Implement batching for event cleanup * remove import * add debug logging --- frigate/events/cleanup.py | 42 +++++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/frigate/events/cleanup.py b/frigate/events/cleanup.py index 7d3e7c4561..8ae38b5340 100644 --- a/frigate/events/cleanup.py +++ b/frigate/events/cleanup.py @@ -21,6 +21,9 @@ class EventCleanupType(str, Enum): snapshots = "snapshots" +CHUNK_SIZE = 50 + + class EventCleanup(threading.Thread): def __init__( self, config: FrigateConfig, stop_event: MpEvent, db: SqliteVecQueueDatabase @@ -107,6 +110,7 @@ def expire(self, media_type: EventCleanupType) -> list[str]: .namedtuples() .iterator() ) + logger.debug(f"{len(expired_events)} events can be expired") # delete the media from disk for expired in expired_events: media_name = f"{expired.camera}-{expired.id}" @@ -125,13 +129,34 @@ def expire(self, media_type: EventCleanupType) -> list[str]: logger.warning(f"Unable to delete event images: {e}") # update the clips attribute for the db entry - update_query = Event.update(update_params).where( + query = Event.select(Event.id).where( Event.camera.not_in(self.camera_keys), Event.start_time < expire_after, Event.label == event.label, Event.retain_indefinitely == False, ) - update_query.execute() + + events_to_update = [] + + for batch in query.iterator(): + events_to_update.extend([event.id for event in batch]) + if len(events_to_update) >= CHUNK_SIZE: + logger.debug( + f"Updating {update_params} for {len(events_to_update)} events" + ) + Event.update(update_params).where( + Event.id << events_to_update + ).execute() + events_to_update = [] + + # Update any remaining events + if events_to_update: + logger.debug( + f"Updating clips/snapshots attribute for {len(events_to_update)} events" + ) + Event.update(update_params).where( + Event.id << events_to_update + ).execute() events_to_update = [] @@ -196,7 +221,11 @@ def expire(self, media_type: EventCleanupType) -> list[str]: logger.warning(f"Unable to delete event images: {e}") # update the clips attribute for the db entry - Event.update(update_params).where(Event.id << events_to_update).execute() + for i in range(0, len(events_to_update), CHUNK_SIZE): + batch = events_to_update[i : i + CHUNK_SIZE] + logger.debug(f"Updating {update_params} for {len(batch)} events") + Event.update(update_params).where(Event.id << batch).execute() + return events_to_update def run(self) -> None: @@ -222,10 +251,11 @@ def run(self) -> None: .iterator() ) events_to_delete = [e.id for e in events] + logger.debug(f"Found {len(events_to_delete)} events that can be expired") if len(events_to_delete) > 0: - chunk_size = 50 - for i in range(0, len(events_to_delete), chunk_size): - chunk = events_to_delete[i : i + chunk_size] + for i in range(0, len(events_to_delete), CHUNK_SIZE): + chunk = events_to_delete[i : i + CHUNK_SIZE] + logger.debug(f"Deleting {len(chunk)} events from the database") Event.delete().where(Event.id << chunk).execute() if self.config.semantic_search.enabled: From 15bd26c9b19008eaa7b6202c793cd1f2f7229720 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Thu, 7 Nov 2024 08:25:13 -0600 Subject: [PATCH 3/9] Re-send camera states after websocket disconnects and reconnects (#14847) --- web/src/api/ws.tsx | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/web/src/api/ws.tsx b/web/src/api/ws.tsx index b0c89d5ddf..c7bb74095b 100644 --- a/web/src/api/ws.tsx +++ b/web/src/api/ws.tsx @@ -69,7 +69,10 @@ function useValue(): useValueReturn { ...prevState, ...cameraStates, })); - setHasCameraState(true); + + if (Object.keys(cameraStates).length > 0) { + setHasCameraState(true); + } // we only want this to run initially when the config is loaded // eslint-disable-next-line react-hooks/exhaustive-deps }, [wsState]); @@ -93,6 +96,9 @@ function useValue(): useValueReturn { retain: false, }); }, + onClose: () => { + setHasCameraState(false); + }, shouldReconnect: () => true, retryOnError: true, }); From 0d59754be29df9bb60cf589e08a4b6385ce517e0 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Thu, 7 Nov 2024 14:27:55 -0600 Subject: [PATCH 4/9] Small genai fix (#14850) * Ensure the regenerate button shows when genai is only enabled at the camera level * update docs --- docs/docs/configuration/genai.md | 8 ++++++-- frigate/api/event.py | 4 +++- frigate/genai/__init__.py | 9 ++++----- web/src/components/overlay/detail/SearchDetailDialog.tsx | 2 +- web/src/types/frigateConfig.ts | 7 +++++++ 5 files changed, 21 insertions(+), 9 deletions(-) diff --git a/docs/docs/configuration/genai.md b/docs/docs/configuration/genai.md index 2ee27f7242..2ec9a62763 100644 --- a/docs/docs/configuration/genai.md +++ b/docs/docs/configuration/genai.md @@ -3,9 +3,13 @@ id: genai title: Generative AI --- -Generative AI can be used to automatically generate descriptive text based on the thumbnails of your tracked objects. This helps with [Semantic Search](/configuration/semantic_search) in Frigate to provide more context about your tracked objects. +Generative AI can be used to automatically generate descriptive text based on the thumbnails of your tracked objects. This helps with [Semantic Search](/configuration/semantic_search) in Frigate to provide more context about your tracked objects. Descriptions are accessed via the _Explore_ view in the Frigate UI by clicking on a tracked object's thumbnail. -Semantic Search must be enabled to use Generative AI. Descriptions are accessed via the _Explore_ view in the Frigate UI by clicking on a tracked object's thumbnail. +:::info + +Semantic Search must be enabled to use Generative AI. + +::: ## Configuration diff --git a/frigate/api/event.py b/frigate/api/event.py index ac414cdde6..cf0ac26cc9 100644 --- a/frigate/api/event.py +++ b/frigate/api/event.py @@ -996,9 +996,11 @@ def regenerate_description( status_code=404, ) + camera_config = request.app.frigate_config.cameras[event.camera] + if ( request.app.frigate_config.semantic_search.enabled - and request.app.frigate_config.genai.enabled + and camera_config.genai.enabled ): request.app.event_metadata_updater.publish((event.id, params.source)) diff --git a/frigate/genai/__init__.py b/frigate/genai/__init__.py index e2d509383c..74fae9fea5 100644 --- a/frigate/genai/__init__.py +++ b/frigate/genai/__init__.py @@ -54,11 +54,10 @@ def _send(self, prompt: str, images: list[bytes]) -> Optional[str]: def get_genai_client(genai_config: GenAIConfig) -> Optional[GenAIClient]: """Get the GenAI client.""" - if genai_config.enabled: - load_providers() - provider = PROVIDERS.get(genai_config.provider) - if provider: - return provider(genai_config) + load_providers() + provider = PROVIDERS.get(genai_config.provider) + if provider: + return provider(genai_config) return None diff --git a/web/src/components/overlay/detail/SearchDetailDialog.tsx b/web/src/components/overlay/detail/SearchDetailDialog.tsx index f158df3291..f56074a52a 100644 --- a/web/src/components/overlay/detail/SearchDetailDialog.tsx +++ b/web/src/components/overlay/detail/SearchDetailDialog.tsx @@ -477,7 +477,7 @@ function ObjectDetailsTab({ onChange={(e) => setDesc(e.target.value)} />
- {config?.genai.enabled && ( + {config?.cameras[search.camera].genai.enabled && (
-
diff --git a/web/src/components/overlay/MobileReviewSettingsDrawer.tsx b/web/src/components/overlay/MobileReviewSettingsDrawer.tsx index d58d485b93..3eeb639cd1 100644 --- a/web/src/components/overlay/MobileReviewSettingsDrawer.tsx +++ b/web/src/components/overlay/MobileReviewSettingsDrawer.tsx @@ -4,7 +4,7 @@ import { Button } from "../ui/button"; import { FaArrowDown, FaCalendarAlt, FaCog, FaFilter } from "react-icons/fa"; import { TimeRange } from "@/types/timeline"; import { ExportContent, ExportPreviewDialog } from "./ExportDialog"; -import { ExportMode } from "@/types/filter"; +import { ExportMode, GeneralFilter } from "@/types/filter"; import ReviewActivityCalendar from "./ReviewActivityCalendar"; import { SelectSeparator } from "../ui/select"; import { ReviewFilter, ReviewSeverity, ReviewSummary } from "@/types/review"; @@ -114,12 +114,12 @@ export default function MobileReviewSettingsDrawer({ // filters - const [currentLabels, setCurrentLabels] = useState( - filter?.labels, - ); - const [currentZones, setCurrentZones] = useState( - filter?.zones, - ); + const [currentFilter, setCurrentFilter] = useState({ + labels: filter?.labels, + zones: filter?.zones, + showAll: filter?.showAll, + ...filter, + }); if (!isMobile) { return; @@ -260,23 +260,21 @@ export default function MobileReviewSettingsDrawer({ - onUpdateFilter({ ...filter, zones: newZones }) - } - setShowAll={(showAll) => { - onUpdateFilter({ ...filter, showAll }); + onUpdateFilter={setCurrentFilter} + onApply={() => { + if (currentFilter !== filter) { + onUpdateFilter(currentFilter); + } + }} + onReset={() => { + const resetFilter: GeneralFilter = {}; + setCurrentFilter(resetFilter); + onUpdateFilter(resetFilter); }} - setCurrentLabels={setCurrentLabels} - updateLabelFilter={(newLabels) => - onUpdateFilter({ ...filter, labels: newLabels }) - } onClose={() => setDrawerMode("select")} />
diff --git a/web/src/types/filter.ts b/web/src/types/filter.ts index 09ff5b99a6..b7d2223c01 100644 --- a/web/src/types/filter.ts +++ b/web/src/types/filter.ts @@ -10,3 +10,9 @@ export type FilterList = { }; export const LAST_24_HOURS_KEY = "last24Hours"; + +export type GeneralFilter = { + showAll?: boolean; + labels?: string[]; + zones?: string[]; +}; From 7bae9463b25201954f1e57afc43161b51bab1b73 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Fri, 8 Nov 2024 08:49:05 -0600 Subject: [PATCH 8/9] Small general filter bugfix (#14870) --- web/src/components/filter/ReviewFilterGroup.tsx | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/web/src/components/filter/ReviewFilterGroup.tsx b/web/src/components/filter/ReviewFilterGroup.tsx index bea816203c..d315965611 100644 --- a/web/src/components/filter/ReviewFilterGroup.tsx +++ b/web/src/components/filter/ReviewFilterGroup.tsx @@ -214,7 +214,9 @@ export default function ReviewFilterGroup({ showAll={filter?.showAll == true} allZones={filterValues.zones} selectedZones={filter?.zones} - onUpdateFilter={onUpdateFilter} + onUpdateFilter={(general) => { + onUpdateFilter({ ...filter, ...general }); + }} /> )} {isMobile && mobileSettingsFeatures.length > 0 && ( @@ -293,7 +295,7 @@ type GeneralFilterButtonProps = { allZones: string[]; selectedZones?: string[]; filter?: GeneralFilter; - onUpdateFilter: (filter: ReviewFilter) => void; + onUpdateFilter: (filter: GeneralFilter) => void; }; function GeneralFilterButton({ @@ -370,7 +372,11 @@ function GeneralFilterButton({ setOpen(false); }} onReset={() => { - const resetFilter: GeneralFilter = {}; + const resetFilter: GeneralFilter = { + labels: undefined, + zones: undefined, + showAll: false, + }; setCurrentFilter(resetFilter); onUpdateFilter(resetFilter); }} From 3249ffb273cd17e1cd50247c90762a4c63150b4a Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Fri, 8 Nov 2024 09:19:49 -0600 Subject: [PATCH 9/9] Auto-unmute inbound audio when enabling two way audio (#14871) * Automatically enable audio when initiating two way talk with mic * remove check --- web/src/views/live/LiveCameraView.tsx | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/web/src/views/live/LiveCameraView.tsx b/web/src/views/live/LiveCameraView.tsx index 13ed85c4c8..a3bbeea065 100644 --- a/web/src/views/live/LiveCameraView.tsx +++ b/web/src/views/live/LiveCameraView.tsx @@ -434,7 +434,13 @@ export default function LiveCameraView({ Icon={mic ? FaMicrophone : FaMicrophoneSlash} isActive={mic} title={`${mic ? "Disable" : "Enable"} Two Way Talk`} - onClick={() => setMic(!mic)} + onClick={() => { + setMic(!mic); + // Turn on audio when enabling the mic if audio is currently off + if (!mic && !audio) { + setAudio(true); + } + }} /> )} {supportsAudioOutput && preferredLiveMode != "jsmpeg" && (