From b14f840f93fbcafe1aa08a4c7527020314b46879 Mon Sep 17 00:00:00 2001 From: Subhash Bhushan Date: Tue, 9 Jul 2024 09:51:28 -0700 Subject: [PATCH] Check for database provider access on init (#443) If a provider's configuration in invalid, we want to throw an exception as early as possible. This commit validates that a connection is indeed possible during `Domain.init` and throws a `ConfigurationError` exception on error. --- docs/adapters/database/elasticsearch.md | 8 +- docs/adapters/database/postgresql.md | 38 ++++++ docs/adapters/internals.md | 126 ++++++++++++++++++ ...-up-and-tearing-down-database-for-tests.md | 0 docs/start-here.md | 2 +- docs_src/adapters/001.py | 16 +++ docs_src/adapters/database/postgresql/001.py | 29 ++++ mkdocs.yml | 1 + src/protean/adapters/repository/__init__.py | 7 + .../adapters/repository/elasticsearch.py | 5 + src/protean/adapters/repository/memory.py | 4 + src/protean/adapters/repository/sqlalchemy.py | 9 ++ src/protean/domain/__init__.py | 81 ++++++----- src/protean/port/provider.py | 4 + .../elasticsearch_repo/test_provider.py | 21 ++- .../repository/memory/test_provider.py | 3 + .../postgresql/test_provider.py | 19 +++ .../sqlalchemy_repo/sqlite/test_provider.py | 19 +++ tests/config/test_load.py | 5 +- tests/domain/test_config_immutability.py | 7 - 20 files changed, 352 insertions(+), 52 deletions(-) create mode 100644 docs/adapters/internals.md create mode 100644 docs/patterns/setting-up-and-tearing-down-database-for-tests.md create mode 100644 docs_src/adapters/001.py create mode 100644 docs_src/adapters/database/postgresql/001.py create mode 100644 tests/adapters/repository/memory/test_provider.py diff --git a/docs/adapters/database/elasticsearch.md b/docs/adapters/database/elasticsearch.md index 52ea1ca4..2b99d327 100644 --- a/docs/adapters/database/elasticsearch.md +++ b/docs/adapters/database/elasticsearch.md @@ -1,5 +1,7 @@ # Elasticsearch +## Configuration + To use Elasticsearch as a database provider, use the below configuration setting: ```toml @@ -14,18 +16,18 @@ settings = "{'number_of_shards': 3}" Additional options for finer control: -### NAMESPACE_PREFIX +### namespace_prefix Elasticsearch instance are prefixed with the specified string. For example, if the namespace prefix is `prod`, the index for aggregate `Person` will be `prod-person`. -### NAMESPACE_SEPARATOR +### namespace_separator Custom character to join namespace_prefix =n ${Default} yphen(`-`). For example, with `NAMESPACE_SEPARATOR` as `_` and namespace prefix as `prod`, the index of aggregate `Person` will be `prod_person`. -### SETTINGS +### settings Index settings passed as-is to Elasticsearch instance. diff --git a/docs/adapters/database/postgresql.md b/docs/adapters/database/postgresql.md index 9f2fb04d..3a8d674e 100644 --- a/docs/adapters/database/postgresql.md +++ b/docs/adapters/database/postgresql.md @@ -1,2 +1,40 @@ # PostgreSQL +The PostgreSQL adapter uses (`SQLAlchemy`)[https://www.sqlalchemy.org/] under +the covers as the ORM to communicate with the database. + +## Configuration + +```toml +[databases.default] +provider = "postgresql" +database_uri = "postgresql://postgres:postgres@localhost:5432/postgres" +``` + +## Options + +### provider + +`postgresql` is the provider for PostgreSQL. + +### database_uri + +Connection string that specifies how to connect to a PostgreSQL database. + +### schema + +Specifies the database schema to use in the database. + +## SQLAlchemy Model + +You can supply a custom SQLAlchemy Model in place of the one that Protean +generates internally, allowing you full customization. + +```python hl_lines="8-11 20-23" +{! docs_src/adapters/database/postgresql/001.py !} +``` + +!!!note + The column names specified in the model should exactly match the attribute + names of the Aggregate or Entity it represents. + diff --git a/docs/adapters/internals.md b/docs/adapters/internals.md new file mode 100644 index 00000000..29017132 --- /dev/null +++ b/docs/adapters/internals.md @@ -0,0 +1,126 @@ +# Internals + +## Database + +A database adapter has four components that work together to make database +communication possible. Each adapter will override the corresponding base +classes for each of these components with its own implementation. + +!!!note + An exception to this rule is the `Session` class. It may be preferable to + use the `Session` structures provided by the database technology as-is. For + example, `PostgreSQL` adapter that is powered by `SQLAlchemy` simply uses + (and returns) the [sqlalchemy.orm.Session](https://docs.sqlalchemy.org/en/20/orm/session_api.html#sqlalchemy.orm.Session) + object provided by `SQLAlchemy`. + +### Provider + +**Base Class**: `protean.port.database.BaseProvider` + +The Provider is the main component responsible for interfacing with the +specific database technology. It contains the configuration and setup needed +to connect to the database, and it provides methods for interacting with the +database at a high level. + +The Provider acts as the bridge between the application and the database, +ensuring that the necessary connections are established and maintained. +It also handles any database-specific nuances, such as connection pooling, +transaction management, and query optimization. + +To add a new provider, subclass from the Provider Base class and implement +methods for your database technology. + +### Data Access Object (DAO) + +**Base Class**: `protean.port.database.BaseDAO` + +The Data Access Object (DAO) is responsible for encapsulating the details of +the database interactions. It provides an abstract interface to the database, +hiding the complexity of the underlying database operations from the rest of +the application. + +The DAO defines methods for CRUD (Create, Read, Update, Delete) operations +and other database queries, translating them into the appropriate SQL or +database-specific commands. + +By using DAOs, the application code remains clean and decoupled from the +database logic. DAOs also work in conjunction in [lookups](#lookups) to +establish a query API that works across various adapters. + +To add a new DAO for your provider, subclass from the DAO Base class and +implement methods for your database technology. + +### Session + +The Session component manages the lifecycle of database transactions. It is +responsible for opening and closing connections, beginning and committing +transactions, and ensuring data consistency. + +The Session provides a context for performing database operations, +ensuring that all database interactions within a transaction are properly +managed and that resources are released appropriately. This component is +crucial for maintaining transactional integrity and for handling rollback +scenarios in case of errors. + +The Session object is usually constructed and provided by the database orm +or technology package. For example, `PostgreSQL` adapter depends on the +[sqlalchemy.orm.Session](https://docs.sqlalchemy.org/en/20/orm/session_api.html#sqlalchemy.orm.Session) +object provided by `SQLAlchemy`. + +### Model + +**Base Class**: `protean.core.model.BaseModel` + +The Model represents the domain entities that are persisted in the database. +It defines the structure of the data, including fields, relationships, and +constraints. It is also a high-level abstraction for working with database +records, allowing you to interact with the data using Python objects rather +than raw queries. + +Models are typically defined using a schema or an ORM +(Object-Relational Mapping) framework that maps the database tables to +Python objects. + +Implementing a model for your database technology can be slightly involved, +as ORM packages can heavily depend upon interal structures. Every +database package is structured differently. Consult existing models for +PostgreSQL +([SQLAlchemy](https://github.com/proteanhq/protean/blob/main/src/protean/adapters/repository/sqlalchemy.py#L139)) +and Elasticsearch +([Elasticsearch](https://github.com/proteanhq/protean/blob/main/src/protean/adapters/repository/elasticsearch.py#L43)) +for examples on constructing model classes. + +Once you define a base model for your provider, Protean auto-generates the model +class for aggregates or entities when needed. You can control this behavior +by supplying an explicit hand-crafted model class for your entity. + + + +### Lookups + +Lookups in Protean are mechanisms used to query the database based on certain +criteria. They are customized implementations of different types of filters +that help filter and retrieve data from the database, making it easier to +perform complex queries without writing raw SQL. Lookups are typically used in +the DAO layer to fetch records that match specific conditions. + +Refer to the section on +[querying](../guides/change-state/retrieve-aggregates.md#advanced-filtering-criteria) +aggregates for examples of lookups. Consult the documentation on your specific +chosen adapter for more information. The adapter may support specialized +lookups for efficient or technology-specific queries. + +## Initialization + +Adapters are initialized as part of Domain initialization. Protean creates +the provider objects in the adapter and establishes connections with the +underlying infrastructure. + +If a connection cannot be established for whatever reason, the Protean +initialization procedure immediately halts and exits with an error message: + +```python hl_lines="8" +{! docs_src/adapters/001.py !} +``` + +1. `foobar` database does not exist on port 5444 \ No newline at end of file diff --git a/docs/patterns/setting-up-and-tearing-down-database-for-tests.md b/docs/patterns/setting-up-and-tearing-down-database-for-tests.md new file mode 100644 index 00000000..e69de29b diff --git a/docs/start-here.md b/docs/start-here.md index 852dcb55..9eced18a 100644 --- a/docs/start-here.md +++ b/docs/start-here.md @@ -11,7 +11,7 @@ Welcome to Protean! Whether you're a seasoned developer or new to the framework, Ready to dive in? The first step is to get Protean up and running on your system. The [Installation guide](./guides/getting-started/installation.md) will take you through a brief tour to setup Protean locally. ## Core Concepts -Time to get acquainted with the building blocks of Protean. Get to know the driving principles and core ideas that shape this framework in [Introduction](./core-concepts/index.md). In [Building Blocks](./core-concepts/building-blocks/index.md) explore key DDD concepts like Aggregates, Repositories, Events, and more, to understand the core structures of Protean. +Get to know the driving principles and core ideas that shape this framework in [Introduction](./core-concepts/index.md). In [Building Blocks](./core-concepts/building-blocks/index.md), explore key DDD concepts like Aggregates, Repositories, Events, and more, to understand the core structures of Protean. ## Building with Protean diff --git a/docs_src/adapters/001.py b/docs_src/adapters/001.py new file mode 100644 index 00000000..a96bb393 --- /dev/null +++ b/docs_src/adapters/001.py @@ -0,0 +1,16 @@ +from protean import Domain + +domain = Domain(__file__, load_toml=False) + +# Non-existent database +domain.config["databases"]["default"] = { + "provider": "postgresql", + "database_uri": "postgresql://postgres:postgres@localhost:5444/foobar", # (1) +} + +domain.init(traverse=False) + +# Output +# +# protean.exceptions.ConfigurationError: +# Could not connect to database at postgresql://postgres:postgres@localhost:5444/foobar diff --git a/docs_src/adapters/database/postgresql/001.py b/docs_src/adapters/database/postgresql/001.py new file mode 100644 index 00000000..5441d18d --- /dev/null +++ b/docs_src/adapters/database/postgresql/001.py @@ -0,0 +1,29 @@ +import sqlalchemy as sa + +from protean import Domain +from protean.adapters.repository.sqlalchemy import SqlalchemyModel +from protean.fields import Integer, String + +domain = Domain(__file__, load_toml=False) +domain.config["databases"]["default"] = { + "provider": "postgresql", + "database_uri": "postgresql://postgres:postgres@localhost:5432/postgres", +} + + +@domain.aggregate +class Provider: + name = String() + age = Integer() + + +@domain.model(part_of=Provider) +class ProviderCustomModel: + name = sa.Column(sa.Text) + age = sa.Column(sa.Integer) + + +domain.init() +with domain.domain_context(): + model_cls = domain.repository_for(Provider)._model + assert issubclass(model_cls, SqlalchemyModel) diff --git a/mkdocs.yml b/mkdocs.yml index 827d25da..e3bd62d6 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -216,6 +216,7 @@ nav: # - faq.md - Adapters: - adapters/index.md + - adapters/internals.md - Database: - adapters/database/index.md - adapters/database/postgresql.md diff --git a/src/protean/adapters/repository/__init__.py b/src/protean/adapters/repository/__init__.py index c81c11b2..e4d52e99 100644 --- a/src/protean/adapters/repository/__init__.py +++ b/src/protean/adapters/repository/__init__.py @@ -109,6 +109,13 @@ def _initialize(self): ) provider = provider_cls(provider_name, self.domain, conn_info) + # Initialize a connection to check if everything is ok + conn = provider.is_alive() + if not conn: + raise ConfigurationError( + f"Could not connect to database at {conn_info['database_uri']}" + ) + provider_objects[provider_name] = provider self._providers = provider_objects diff --git a/src/protean/adapters/repository/elasticsearch.py b/src/protean/adapters/repository/elasticsearch.py index 1d0faca9..cbe117a6 100644 --- a/src/protean/adapters/repository/elasticsearch.py +++ b/src/protean/adapters/repository/elasticsearch.py @@ -376,6 +376,11 @@ def get_connection(self): verify_certs=self.conn_info.get("VERIFY_CERTS", False), ) + def is_alive(self) -> bool: + """Check if the connection is alive""" + conn = self.get_connection() + return conn.ping() + def get_dao(self, entity_cls, model_cls): """Return a DAO object configured with a live connection""" return ElasticsearchDAO(self.domain, self, entity_cls, model_cls) diff --git a/src/protean/adapters/repository/memory.py b/src/protean/adapters/repository/memory.py index ae4290be..0c004c62 100644 --- a/src/protean/adapters/repository/memory.py +++ b/src/protean/adapters/repository/memory.py @@ -119,6 +119,10 @@ def get_connection(self, session_cls=None): """Return the dictionary database object""" return MemorySession(self, new_connection=True) + def is_alive(self) -> bool: + """Check if the connection is alive""" + return True + def _data_reset(self): """Reset data""" self._databases = defaultdict(dict) diff --git a/src/protean/adapters/repository/sqlalchemy.py b/src/protean/adapters/repository/sqlalchemy.py index 7c42bd9d..bfa8391f 100644 --- a/src/protean/adapters/repository/sqlalchemy.py +++ b/src/protean/adapters/repository/sqlalchemy.py @@ -619,6 +619,15 @@ def get_connection(self, session_cls=None): return conn + def is_alive(self) -> bool: + """Check if the connection to the database is alive""" + try: + conn = self.get_connection() + conn.execute(text("SELECT 1")) + return True + except DatabaseError: + return False + def _data_reset(self): conn = self._engine.connect() diff --git a/src/protean/domain/__init__.py b/src/protean/domain/__init__.py index 2a1b8331..77b28b6c 100644 --- a/src/protean/domain/__init__.py +++ b/src/protean/domain/__init__.py @@ -11,8 +11,6 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Union from uuid import uuid4 -from werkzeug.datastructures import ImmutableDict - from protean.adapters import Brokers, Caches, EmailProviders, Providers from protean.adapters.event_store import EventStore from protean.container import Element @@ -49,6 +47,48 @@ _sentinel = object() +def _default_config(): + """Return the default configuration for a Protean application. + + This is placed in a separate function because we want to be absolutely + sure that we are using a copy of the defaults when we manipulate config + directly in tests. Housing it within the main `Domain` class can + potentially lead to issues because the config can be overwritten by accident. + """ + from protean.utils import IdentityStrategy, IdentityType + + return { + "env": None, + "debug": None, + "secret_key": None, + "identity_strategy": IdentityStrategy.UUID.value, + "identity_type": IdentityType.STRING.value, + "databases": { + "default": {"provider": "memory"}, + "memory": {"provider": "memory"}, + }, + "event_processing": EventProcessing.ASYNC.value, + "command_processing": CommandProcessing.ASYNC.value, + "event_store": { + "provider": "memory", + }, + "caches": { + "default": { + "provider": "memory", + "TTL": 300, + } + }, + "brokers": {"default": {"provider": "inline"}}, + "EMAIL_PROVIDERS": { + "default": { + "provider": "protean.adapters.DummyEmailProvider", + "DEFAULT_FROM_EMAIL": "admin@team8solutions.com", + }, + }, + "SNAPSHOT_THRESHOLD": 10, + } + + class Domain: """The domain object is a one-stop gateway to: @@ -98,39 +138,6 @@ class Domain: #: :data:`secret_key` configuration key. Defaults to ``None``. secret_key = ConfigAttribute("secret_key") - default_config = ImmutableDict( - { - "env": None, - "debug": None, - "secret_key": None, - "identity_strategy": IdentityStrategy.UUID.value, - "identity_type": IdentityType.STRING.value, - "databases": { - "default": {"provider": "memory"}, - "memory": {"provider": "memory"}, - }, - "event_processing": EventProcessing.ASYNC.value, - "command_processing": CommandProcessing.ASYNC.value, - "event_store": { - "provider": "memory", - }, - "caches": { - "default": { - "provider": "memory", - "TTL": 300, - } - }, - "brokers": {"default": {"provider": "inline"}}, - "EMAIL_PROVIDERS": { - "default": { - "provider": "protean.adapters.DummyEmailProvider", - "DEFAULT_FROM_EMAIL": "admin@team8solutions.com", - }, - }, - "SNAPSHOT_THRESHOLD": 10, - }, - ) - def __init__( self, root_path: str, @@ -306,13 +313,13 @@ def _initialize(self): def load_config(self, load_toml=True): """Load configuration from dist or a .toml file.""" - defaults = dict(self.default_config) + defaults = _default_config() defaults["env"] = get_env() defaults["debug"] = get_debug_flag() if load_toml: config = Config2.load_from_path(self.root_path, defaults) else: - config = Config2.load_from_dict(dict(self.default_config)) + config = Config2.load_from_dict(defaults) # Load Constants if "custom" in config: diff --git a/src/protean/port/provider.py b/src/protean/port/provider.py index d4fd3123..50611b86 100644 --- a/src/protean/port/provider.py +++ b/src/protean/port/provider.py @@ -48,6 +48,10 @@ def get_session(self): def get_connection(self): """Get the connection object for the repository""" + @abstractmethod + def is_alive(self) -> bool: + """Check if the connection is alive""" + @abstractmethod def get_dao(self, entity_cls, model_cls): """Return a DAO object configured with a live connection""" diff --git a/tests/adapters/repository/elasticsearch_repo/test_provider.py b/tests/adapters/repository/elasticsearch_repo/test_provider.py index 1f8ed76d..065205bc 100644 --- a/tests/adapters/repository/elasticsearch_repo/test_provider.py +++ b/tests/adapters/repository/elasticsearch_repo/test_provider.py @@ -1,11 +1,11 @@ -"""Module to test SQLAlchemy Provider Class""" - import pytest from elasticsearch import Elasticsearch from elasticsearch_dsl.response import Response +from protean import Domain from protean.adapters import Providers from protean.adapters.repository.elasticsearch import ESProvider +from protean.exceptions import ConfigurationError from .elements import Alien, Person @@ -39,6 +39,23 @@ def test_provider_get_connection(self, test_domain): assert conn is not None assert isinstance(conn, Elasticsearch) + def test_provider_is_alive(self, test_domain): + """Test ``is_alive`` method""" + assert test_domain.providers["default"].is_alive() + + @pytest.mark.no_test_domain + def test_exception_on_invalid_provider(self): + """Test exception on invalid provider""" + domain = Domain(__file__, load_toml=False) + domain.config["databases"]["default"] = { + "provider": "elasticsearch", + "database_uri": '{"hosts": ["imaginary"]}', + } + with pytest.raises(ConfigurationError) as exc: + domain.init(traverse=False) + + assert "Could not connect to database at" in str(exc.value) + @pytest.mark.pending def test_provider_raw(self, test_domain): """Test raw queries""" diff --git a/tests/adapters/repository/memory/test_provider.py b/tests/adapters/repository/memory/test_provider.py new file mode 100644 index 00000000..526ea121 --- /dev/null +++ b/tests/adapters/repository/memory/test_provider.py @@ -0,0 +1,3 @@ +def test_provider_is_alive(test_domain): + """Test ``is_alive`` method""" + assert test_domain.providers["default"].is_alive() diff --git a/tests/adapters/repository/sqlalchemy_repo/postgresql/test_provider.py b/tests/adapters/repository/sqlalchemy_repo/postgresql/test_provider.py index 518da824..a2e32150 100644 --- a/tests/adapters/repository/sqlalchemy_repo/postgresql/test_provider.py +++ b/tests/adapters/repository/sqlalchemy_repo/postgresql/test_provider.py @@ -4,8 +4,10 @@ from sqlalchemy.engine.result import Result from sqlalchemy.orm.session import Session +from protean import Domain from protean.adapters.repository import Providers from protean.adapters.repository.sqlalchemy import PostgresqlProvider +from protean.exceptions import ConfigurationError from .elements import Alien, Person @@ -39,6 +41,23 @@ def test_provider_get_connection(self, test_domain): assert conn is not None assert isinstance(conn, Session) + def test_provider_is_alive(self, test_domain): + """Test ``is_alive`` method""" + assert test_domain.providers["default"].is_alive() + + @pytest.mark.no_test_domain + def test_exception_on_invalid_provider(self): + """Test exception on invalid provider""" + domain = Domain(__file__, load_toml=False) + domain.config["databases"]["default"] = { + "provider": "postgresql", + "database_uri": "postgresql://postgres:postgres@localhost:5444/foobar", + } + with pytest.raises(ConfigurationError) as exc: + domain.init(traverse=False) + + assert "Could not connect to database at" in str(exc.value) + def test_provider_raw(self, test_domain): """Test raw queries""" test_domain.repository_for(Person)._dao.create( diff --git a/tests/adapters/repository/sqlalchemy_repo/sqlite/test_provider.py b/tests/adapters/repository/sqlalchemy_repo/sqlite/test_provider.py index b7526b64..46d772bf 100644 --- a/tests/adapters/repository/sqlalchemy_repo/sqlite/test_provider.py +++ b/tests/adapters/repository/sqlalchemy_repo/sqlite/test_provider.py @@ -4,8 +4,10 @@ from sqlalchemy.engine.result import Result from sqlalchemy.orm.session import Session +from protean import Domain from protean.adapters.repository import Providers from protean.adapters.repository.sqlalchemy import SqliteProvider +from protean.exceptions import ConfigurationError from .elements import Alien, Person @@ -40,6 +42,23 @@ def test_provider_get_connection(self, test_domain): assert isinstance(conn, Session) assert conn.is_active + def test_provider_is_alive(self, test_domain): + """Test ``is_alive`` method""" + assert test_domain.providers["default"].is_alive() + + @pytest.mark.no_test_domain + def test_exception_on_invalid_provider(self): + """Test exception on invalid provider""" + domain = Domain(__file__, load_toml=False) + domain.config["databases"]["default"] = { + "provider": "sqlite", + "database_uri": "sqlite:////C:/Users/username/foobar.db", + } + with pytest.raises(ConfigurationError) as exc: + domain.init(traverse=False) + + assert "Could not connect to database at" in str(exc.value) + def test_provider_raw(self, test_domain): """Test raw queries""" test_domain.repository_for(Person)._dao.create( diff --git a/tests/config/test_load.py b/tests/config/test_load.py index dd01d218..b2281627 100644 --- a/tests/config/test_load.py +++ b/tests/config/test_load.py @@ -6,7 +6,6 @@ import pytest -from protean import Domain from protean.domain.config import Config2 from protean.exceptions import ConfigurationError from protean.utils.domain_discovery import derive_domain @@ -126,7 +125,9 @@ def test_domain_throws_error_if_config_file_not_found(self): class TestLoadingDefaults: def test_that_config_is_loaded_from_dict(self): - config_dict = dict(Domain.default_config) + from protean.domain import _default_config + + config_dict = _default_config() config_dict["custom"] = {"FOO": "bar", "qux": "quux"} config = Config2.load_from_dict(config_dict) assert config["databases"]["default"]["provider"] == "memory" diff --git a/tests/domain/test_config_immutability.py b/tests/domain/test_config_immutability.py index d676dcab..a5ef3789 100644 --- a/tests/domain/test_config_immutability.py +++ b/tests/domain/test_config_immutability.py @@ -1,14 +1,7 @@ -import pytest - from protean import Domain from protean.utils import IdentityStrategy -def test_that_default_config_is_immutable(): - with pytest.raises(TypeError): - Domain.default_config["identity_strategy"] = "FOO" - - def test_that_config_is_unique_to_each_domain(): domain1 = Domain(__file__, load_toml=False) assert domain1.config["identity_strategy"] == IdentityStrategy.UUID.value