Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: added connection logging retrying and backoff - mongodb #45

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
+68 −102
Open

feat: added connection logging retrying and backoff - mongodb #45

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
432707c
feat: added connection logging retrying and backoff - mongodb
ARYPROGRAMMER Jan 11, 2025
a0fc8a7
added indexes
ARYPROGRAMMER Jan 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
170 changes: 68 additions & 102 deletions database_setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,28 +1,69 @@
from pymongo import MongoClient, ASCENDING
from pymongo.errors import ConnectionFailure, ServerSelectionTimeoutError
from pymongo.server_api import ServerApi
import logging
import os
import time
from dotenv import load_dotenv

load_dotenv()

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

# Configuration for retry and timeouts
MAX_RETRIES = 5
INITIAL_RETRY_DELAY = 1
MAX_RETRY_DELAY = 30
TIMEOUT_CONFIG = {
"connectTimeoutMS": 5000,
"socketTimeoutMS": 10000
}

def exponential_backoff(attempt):
"""Calculate exponential backoff delay."""
return min(INITIAL_RETRY_DELAY * (2 ** attempt), MAX_RETRY_DELAY)

def connect_to_mongodb():
"""Establish a MongoDB connection with retries and timeouts."""
uri = os.getenv("MONGODB_URL")
if not uri:
logger.error("MONGODB_URL is not set in the environment.")
raise ValueError("MONGODB_URL is missing")

for attempt in range(MAX_RETRIES):
try:
logger.info(f"Attempting to connect to MongoDB (Attempt {attempt + 1}/{MAX_RETRIES})")
client = MongoClient(uri, server_api=ServerApi('1'), **TIMEOUT_CONFIG)
# Verify connection
client.admin.command("ping")
logger.info("Connected to MongoDB successfully")
return client
except (ConnectionFailure, ServerSelectionTimeoutError) as e:
delay = exponential_backoff(attempt)
logger.warning(f"Connection failed (Attempt {attempt + 1}/{MAX_RETRIES}): {e}")
if attempt < MAX_RETRIES - 1:
logger.info(f"Retrying in {delay} seconds...")
time.sleep(delay)
else:
logger.error("Maximum retry attempts reached. Exiting...")
raise
except Exception as e:
logger.error(f"Unexpected error while connecting to MongoDB: {e}")
raise

def setup_product_database():
"""Setup product reference database with sample data"""
uri = os.getenv("MONGODB_URL")

client = None
try:
client = MongoClient(uri, server_api=ServerApi('1'))
client = connect_to_mongodb()
db = client.social_media_products

# Create Image Collections
# Create Collections
product_collection = db["products"]
listing_collection = db["listings"]
analytics_collection = db["analytics"]
review_collection = db["reviews"]

# Video Collections
video_collection = db["videos"]
video_listings_collection = db["video_listings"]
video_analytics_collection = db["video_analytics"]
Expand All @@ -36,6 +77,7 @@ def setup_product_database():
product_collection.create_index([("price_range", ASCENDING)])
product_collection.create_index([("created_at", ASCENDING)])
product_collection.create_index([("updated_at", ASCENDING)])

# Create indexes for listings
listing_collection.create_index([("id", ASCENDING), ("product_id", ASCENDING)])
listing_collection.create_index([("product_id", ASCENDING), ("created_at", ASCENDING)])
Expand All @@ -45,6 +87,7 @@ def setup_product_database():
listing_collection.create_index([("title", ASCENDING)])
listing_collection.create_index([("price", ASCENDING)])
listing_collection.create_index([("features", ASCENDING)], name="features_index")

# Create indexes for analytics
analytics_collection.create_index([("id", ASCENDING), ("product_id", ASCENDING)])
analytics_collection.create_index([("product_id", ASCENDING), ("created_at", ASCENDING)])
Expand All @@ -60,6 +103,7 @@ def setup_product_database():
analytics_collection.create_index([("customer_behavior.average_rating", ASCENDING)])
analytics_collection.create_index([("marketing_metrics.click_through_rate", ASCENDING)])
analytics_collection.create_index([("marketing_metrics.social_media_engagement", ASCENDING)])

# Create indexes for review
review_collection.create_index([("product_id", ASCENDING), ("rating", ASCENDING)])
review_collection.create_index([("user_id", ASCENDING), ("product_id", ASCENDING)])
Expand All @@ -86,7 +130,8 @@ def setup_product_database():
video_collection.create_index([("created_at", ASCENDING)])
video_collection.create_index([("updated_at", ASCENDING)])
video_collection.create_index([("key_features", ASCENDING)])
video_collection.create_index([("highlights", ASCENDING)])
video_collection.create_index([("highlights", ASCENDING)])

# Create indexes for video listing
video_listings_collection.create_index([("product_id", ASCENDING), ("id", ASCENDING)])
video_listings_collection.create_index([("id", ASCENDING)], unique=True)
Expand All @@ -97,7 +142,8 @@ def setup_product_database():
video_listings_collection.create_index([("rating", ASCENDING)])
video_listings_collection.create_index([("created_at", ASCENDING)])
video_listings_collection.create_index([("updated_at", ASCENDING)])
video_listings_collection.create_index([("product_links.price", ASCENDING)])
video_listings_collection.create_index([("product_links.price", ASCENDING)])

# Create indexes for video analytics
video_analytics_collection.create_index([("id", ASCENDING), ("product_id", ASCENDING)])
video_analytics_collection.create_index([("engagement.views", ASCENDING), ("engagement.likes", ASCENDING)])
Expand All @@ -113,102 +159,22 @@ def setup_product_database():
video_analytics_collection.create_index([("audience.demographics", ASCENDING)])
video_analytics_collection.create_index([("audience.top_regions", ASCENDING)])
video_analytics_collection.create_index([("performance.retention_rate", ASCENDING)])
video_analytics_collection.create_index([("performance.click_through_rate", ASCENDING)])

# Sample product reference data
sample_products = [
{
"category": "Electronics",
"subcategory": "Smartphones",
"brand_options": ["Samsung Galaxy S24", "iPhone 15", "Google Pixel 8"],
"price_ranges": {
"budget": {"min": 299, "max": 499},
"mid_range": {"min": 500, "max": 799},
"premium": {"min": 800, "max": 1299}
},
"common_features": [
"5G Connectivity",
"AI-Enhanced Camera",
"AMOLED Display",
"Fast Charging",
"Wireless Charging"
],
"keywords": ["smartphone", "mobile phone", "cell phone", "android", "ios"]
},
{
"category": "Electronics",
"subcategory": "Wireless Earbuds",
"brand_options": ["Apple AirPods Pro", "Samsung Galaxy Buds", "Google Pixel Buds"],
"price_ranges": {
"budget": {"min": 49, "max": 99},
"mid_range": {"min": 100, "max": 199},
"premium": {"min": 200, "max": 299}
},
"common_features": [
"Active Noise Cancellation",
"Touch Controls",
"Wireless Charging Case",
"Water Resistance",
"Voice Assistant Support"
],
"keywords": ["earbuds", "wireless earphones", "tws", "headphones"]
},
{
"category": "Electronics",
"subcategory": "Smartwatches",
"brand_options": ["Apple Watch Series 9", "Samsung Galaxy Watch 6", "Google Pixel Watch"],
"price_ranges": {
"budget": {"min": 149, "max": 249},
"mid_range": {"min": 250, "max": 399},
"premium": {"min": 400, "max": 799}
},
"common_features": [
"Health Monitoring",
"Fitness Tracking",
"GPS",
"Always-On Display",
"Water Resistance"
],
"keywords": ["smartwatch", "fitness tracker", "smart watch", "wearable"]
}
]

# Clear existing data
# product_references.delete_many({})
# listings.delete_many({})

# Insert sample data
# product_references.insert_many(sample_products)
logger.info("Sample product references inserted successfully")

# Insert sample listings
# sample_listings = [
# {
# "product_id": str(product_references.find_one({"brand_options": "iPhone 15"})["_id"]),
# "title": "iPhone 15 Pro Max",
# "category": "Electronics",
# "subcategory": "Smartphones",
# "description": "Latest iPhone with A17 Pro chip and titanium design",
# "price": "$999",
# "features": [
# "48MP Main Camera",
# "Titanium Design",
# "Action Button"
# ],
# "keywords": ["iphone", "smartphone", "apple"],
# "original_caption": "Just got the new iPhone 15 Pro! Amazing camera system!",
# "created_at": datetime.utcnow(),
# "status": "active"
# }
# ]

# listings.insert_many(sample_listings)
# logger.info("Sample listings inserted successfully")

video_analytics_collection.create_index([("performance.click_through_rate", ASCENDING)])

logger.info("All indexes created successfully")

# Sample data insertion (commented out by default)
# Uncomment and modify as needed
# product_collection.insert_many(sample_products)
# logger.info("Sample product references inserted successfully")

except Exception as e:
logger.error(f"Error setting up database: {e}")
raise
finally:
client.close()
if client:
client.close()
logger.info("MongoDB connection closed")

if __name__ == "__main__":
setup_product_database()
setup_product_database()