diff --git a/global_popular_videos_l7d/ds__global_popular_videos_l7d.py b/global_popular_videos_l7d/ds__global_popular_videos_l7d.py index b97741b..3cbc1b9 100644 --- a/global_popular_videos_l7d/ds__global_popular_videos_l7d.py +++ b/global_popular_videos_l7d/ds__global_popular_videos_l7d.py @@ -21,6 +21,7 @@ def send_alert_to_google_chat(): query = """ + CREATE OR REPLACE TABLE `hot-or-not-feed-intelligence.yral_ds.global_popular_videos_l7d` AS WITH stats AS ( SELECT @@ -62,16 +63,25 @@ def send_alert_to_google_chat(): LEAST(normalized_like_perc, normalized_watch_perc) AS min_normalized_perc FROM normalized_stats +), +popular_videos AS ( + SELECT + video_id, + normalized_like_perc - min_normalized_perc + 1 as normalized_like_perc_p, + normalized_watch_perc - min_normalized_perc + 1 as normalized_watch_perc_p, + 2 / (1 / (normalized_like_perc - min_normalized_perc + 1 + 1e-9) + 1 / (normalized_watch_perc - min_normalized_perc + 1 + 1e-9)) AS global_popularity_score + FROM + offset_stats + ORDER BY + global_popularity_score DESC ) -SELECT - video_id, - normalized_like_perc - min_normalized_perc + 1 as normalized_like_perc_p, - normalized_watch_perc - min_normalized_perc + 1 as normalized_watch_perc_p, - 2 / (1 / (normalized_like_perc - min_normalized_perc + 1 + 1e-9) + 1 / (normalized_watch_perc - min_normalized_perc + 1 + 1e-9)) AS global_popularity_score -FROM - offset_stats -ORDER BY - global_popularity_score DESC + +select popular_videos.*, is_nsfw, nsfw_ec, nsfw_gore +from popular_videos +inner join `hot-or-not-feed-intelligence.yral_ds.video_nsfw` as video_nsfw +on popular_videos.video_id = video_nsfw.video_id +order by global_popularity_score DESC + """ def create_global_popular_videos_l7d(): diff --git a/video_embedding/video_embedding_dag.py b/video_embedding/video_embedding_dag.py index 9c4169f..36a9c52 100644 --- a/video_embedding/video_embedding_dag.py +++ b/video_embedding/video_embedding_dag.py @@ -76,6 +76,26 @@ def send_alert_to_google_chat(): ``` +Video Index recreation query + +``` + +CREATE OR REPLACE VECTOR INDEX vector_video_index +ON `hot-or-not-feed-intelligence.yral_ds.video_index`(embedding) +STORING (uri, post_id, timestamp, canister_id, is_nsfw, nsfw_ec, nsfw_gore) +OPTIONS ( + index_type = 'IVF', + distance_type = 'cosine' +); + +``` + +``` + +SELECT table_name, index_name, ddl, coverage_percentage +FROM `hot-or-not-feed-intelligence.yral_ds.INFORMATION_SCHEMA.VECTOR_INDEXES`; + +``` """