From 7c05a60c88afb115d9af681975f624f384375198 Mon Sep 17 00:00:00 2001 From: Samriti Sadhu Date: Tue, 17 Dec 2024 15:33:50 +0000 Subject: [PATCH 1/9] Adding pool size and max overflow for timeout issue --- application/settings.py | 2 ++ migrations/env.py | 7 +++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/application/settings.py b/application/settings.py index aa407dc1..748b9a24 100644 --- a/application/settings.py +++ b/application/settings.py @@ -21,6 +21,8 @@ class Settings(BaseSettings): GA_MEASUREMENT_ID: Optional[str] = None OS_CLIENT_KEY: Optional[str] = None OS_CLIENT_SECRET: Optional[str] = None + POOL_SIZE: Optional[int] = 10 + MAX_OVERFLOW: Optional[int] = 20 @lru_cache() diff --git a/migrations/env.py b/migrations/env.py index ccbb7abd..74ade3f1 100644 --- a/migrations/env.py +++ b/migrations/env.py @@ -65,8 +65,11 @@ def run_migrations_online(): url = config.get_main_option("sqlalchemy.url") else: url = get_settings().WRITE_DATABASE_URL - - engine = create_engine(url) + engine = create_engine( + url, + pool_size=get_settings().POOL_SIZE, + max_overflow=get_settings().MAX_OVERFLOW, + ) connectable = engine with connectable.connect() as connection: From c725bd8f03bc27353a4d5e429f77e43530a15286 Mon Sep 17 00:00:00 2001 From: Samriti Sadhu Date: Tue, 17 Dec 2024 16:02:12 +0000 Subject: [PATCH 2/9] change default to orginals and use envt variable to increase the limit --- application/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/application/settings.py b/application/settings.py index 748b9a24..87c4af4e 100644 --- a/application/settings.py +++ b/application/settings.py @@ -21,8 +21,8 @@ class Settings(BaseSettings): GA_MEASUREMENT_ID: Optional[str] = None OS_CLIENT_KEY: Optional[str] = None OS_CLIENT_SECRET: Optional[str] = None - POOL_SIZE: Optional[int] = 10 - MAX_OVERFLOW: Optional[int] = 20 + POOL_SIZE: Optional[int] = 5 + MAX_OVERFLOW: Optional[int] = 10 @lru_cache() From 96f47c8c7164a836ec688d4731da2b3a15c4f631 Mon Sep 17 00:00:00 2001 From: Samriti Sadhu Date: Thu, 19 Dec 2024 10:02:27 +0000 Subject: [PATCH 3/9] Improving envt variable names and logging connection pool info --- application/settings.py | 4 ++-- migrations/env.py | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/application/settings.py b/application/settings.py index 87c4af4e..6be30d12 100644 --- a/application/settings.py +++ b/application/settings.py @@ -21,8 +21,8 @@ class Settings(BaseSettings): GA_MEASUREMENT_ID: Optional[str] = None OS_CLIENT_KEY: Optional[str] = None OS_CLIENT_SECRET: Optional[str] = None - POOL_SIZE: Optional[int] = 5 - MAX_OVERFLOW: Optional[int] = 10 + DB_POOL_SIZE: Optional[int] = 5 + DB_POOL_MAX_OVERFLOW: Optional[int] = 10 @lru_cache() diff --git a/migrations/env.py b/migrations/env.py index 74ade3f1..a9406fc9 100644 --- a/migrations/env.py +++ b/migrations/env.py @@ -67,8 +67,11 @@ def run_migrations_online(): url = get_settings().WRITE_DATABASE_URL engine = create_engine( url, - pool_size=get_settings().POOL_SIZE, - max_overflow=get_settings().MAX_OVERFLOW, + pool_size=get_settings().DB_POOL_SIZE, + max_overflow=get_settings().DB_POOL_MAX_OVERFLOW, + ) + logging.info( + f"Connection Pool: pool_size={engine.pool.size()}, max_overflow={engine.pool._max_overflow}" ) connectable = engine From e0b33e96ba6bf6e869c4dea68fc14ec1b341d7c1 Mon Sep 17 00:00:00 2001 From: Samriti Sadhu Date: Thu, 19 Dec 2024 10:04:19 +0000 Subject: [PATCH 4/9] changing the ubuntu version --- .github/workflows/deploy.yml | 2 +- .github/workflows/deply-to-dev.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 72cad991..dc7691de 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -22,7 +22,7 @@ jobs: - run: make lint test: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 services: postgres: image: postgis/postgis:14-master diff --git a/.github/workflows/deply-to-dev.yml b/.github/workflows/deply-to-dev.yml index 20c0a71f..c568b052 100644 --- a/.github/workflows/deply-to-dev.yml +++ b/.github/workflows/deply-to-dev.yml @@ -14,7 +14,7 @@ jobs: - run: make lint test: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 services: postgres: image: postgis/postgis:14-master From 752f5115a2eab023f073dc88e41643c8bb1f0f51 Mon Sep 17 00:00:00 2001 From: Samriti Sadhu Date: Fri, 3 Jan 2025 10:42:48 +0000 Subject: [PATCH 5/9] customize pool_size and max_overflow --- application/db/session.py | 7 +++++++ migrations/env.py | 10 ++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/application/db/session.py b/application/db/session.py index 8849533f..157284a4 100644 --- a/application/db/session.py +++ b/application/db/session.py @@ -5,10 +5,17 @@ from sqlalchemy.orm import Session from application.settings import get_settings +import logging + +logger = logging.getLogger(__name__) # this can be used in fast api path functions using Depends to inject a db session def get_session() -> Iterator[Session]: + logger.info( + f"Database engine created with pool_size={get_settings().DB_POOL_SIZE}, " + f"max_overflow={get_settings().DB_POOL_MAX_OVERFLOW}" + ) yield from _get_fastapi_sessionmaker().get_db() diff --git a/migrations/env.py b/migrations/env.py index a9406fc9..ccbb7abd 100644 --- a/migrations/env.py +++ b/migrations/env.py @@ -65,14 +65,8 @@ def run_migrations_online(): url = config.get_main_option("sqlalchemy.url") else: url = get_settings().WRITE_DATABASE_URL - engine = create_engine( - url, - pool_size=get_settings().DB_POOL_SIZE, - max_overflow=get_settings().DB_POOL_MAX_OVERFLOW, - ) - logging.info( - f"Connection Pool: pool_size={engine.pool.size()}, max_overflow={engine.pool._max_overflow}" - ) + + engine = create_engine(url) connectable = engine with connectable.connect() as connection: From 8e81744e047ecd62e8a0f3e628882083d6768c1a Mon Sep 17 00:00:00 2001 From: Samriti Sadhu Date: Fri, 3 Jan 2025 11:46:29 +0000 Subject: [PATCH 6/9] Changed to using create_engine --- application/db/session.py | 49 ++++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/application/db/session.py b/application/db/session.py index 157284a4..8b195a0d 100644 --- a/application/db/session.py +++ b/application/db/session.py @@ -1,31 +1,42 @@ -from functools import lru_cache +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker, Session from typing import Iterator - -from fastapi_utils.session import FastAPISessionMaker -from sqlalchemy.orm import Session - -from application.settings import get_settings import logging +from application.settings import get_settings +# Set up logging logger = logging.getLogger(__name__) -# this can be used in fast api path functions using Depends to inject a db session -def get_session() -> Iterator[Session]: +def _create_engine(): + settings = get_settings() + engine = create_engine( + settings.READ_DATABASE_URL, + pool_size=settings.DB_POOL_SIZE, + max_overflow=settings.DB_POOL_MAX_OVERFLOW, + ) + logger.info( - f"Database engine created with pool_size={get_settings().DB_POOL_SIZE}, " - f"max_overflow={get_settings().DB_POOL_MAX_OVERFLOW}" + f"Engine created with pool_size={engine.pool.size()}, " + f"max_overflow={engine.pool._max_overflow}, " ) - yield from _get_fastapi_sessionmaker().get_db() + return engine + + +# Create session factory +engine = _create_engine() +SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) -# this can be used in non path functions to create a context manager for a db session -# see https://github.com/dmontagu/fastapi-utils/blob/master/fastapi_utils/session.py#L77:L91 -def get_context_session() -> Iterator[Session]: - return _get_fastapi_sessionmaker().context_session() + +def get_session() -> Iterator[Session]: + db = SessionLocal() + try: + yield db + finally: + db.close() -@lru_cache() -def _get_fastapi_sessionmaker() -> FastAPISessionMaker: - database_uri = get_settings().READ_DATABASE_URL - return FastAPISessionMaker(database_uri) +def get_context_session() -> Iterator[Session]: + with SessionLocal() as session: + yield session From eb410b484efddba322de4b1216bd8bd06a6a4509 Mon Sep 17 00:00:00 2001 From: Samriti Sadhu Date: Fri, 3 Jan 2025 13:35:40 +0000 Subject: [PATCH 7/9] Adding get_context_session --- application/db/session.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/application/db/session.py b/application/db/session.py index 8b195a0d..a03ce36a 100644 --- a/application/db/session.py +++ b/application/db/session.py @@ -3,6 +3,7 @@ from typing import Iterator import logging from application.settings import get_settings +from contextlib import contextmanager # Set up logging logger = logging.getLogger(__name__) @@ -37,6 +38,10 @@ def get_session() -> Iterator[Session]: db.close() +@contextmanager def get_context_session() -> Iterator[Session]: - with SessionLocal() as session: + session = SessionLocal() + try: yield session + finally: + session.close() From 4378fbe9a0beedc3a569bdcb987ff6c9f2870033 Mon Sep 17 00:00:00 2001 From: Samriti Sadhu Date: Fri, 3 Jan 2025 14:58:06 +0000 Subject: [PATCH 8/9] adding basicConfig for logging --- application/db/session.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/application/db/session.py b/application/db/session.py index a03ce36a..d57d7a19 100644 --- a/application/db/session.py +++ b/application/db/session.py @@ -5,6 +5,9 @@ from application.settings import get_settings from contextlib import contextmanager +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s" +) # Set up logging logger = logging.getLogger(__name__) @@ -31,6 +34,7 @@ def _create_engine(): def get_session() -> Iterator[Session]: + logger.info("hello") db = SessionLocal() try: yield db From d1b5b5c10f2d10068cadc6978980d8f8c287b4e4 Mon Sep 17 00:00:00 2001 From: Samriti Sadhu Date: Fri, 3 Jan 2025 15:14:03 +0000 Subject: [PATCH 9/9] typo fixes --- application/db/session.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/application/db/session.py b/application/db/session.py index d57d7a19..72423c64 100644 --- a/application/db/session.py +++ b/application/db/session.py @@ -22,7 +22,7 @@ def _create_engine(): logger.info( f"Engine created with pool_size={engine.pool.size()}, " - f"max_overflow={engine.pool._max_overflow}, " + f"max_overflow={engine.pool._max_overflow} " ) return engine @@ -34,7 +34,6 @@ def _create_engine(): def get_session() -> Iterator[Session]: - logger.info("hello") db = SessionLocal() try: yield db