Upgrade packages and fix pytest warnings (#388)

* Fix warnings reported by Pytest
* Replace calls to utcnow with now(UTC)
* Introduce a new function called `utcnow_func` in protean.utils to use as default
* Replace pkgutil with importlib functions
* Remove support 3.8, 3.9, and 3.10
* Remove Protean's own importlib.py file
* Call datetime.now(UTC) on utcnow_func invocation
* Add test for utcnow_func
* Add test on comparing meta of different classes
* Upgrade coverage and linked packages

.github/workflows/ci.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.11", "3.12"]
     name: Python ${{ matrix.python-version }} Tests
 
     services:

setup.py

@@ -27,8 +27,8 @@ def read(*names, **kwargs):
 elasticsearch_requires = ["elasticsearch~=7.17.9", "elasticsearch-dsl~=7.4.1"]
 
 redis_requires = ["redis==3.5.2"]
-sqlite_requires = ["sqlalchemy==1.4.47"]
-postgresql_requires = ["psycopg2>=2.8.4", "sqlalchemy==1.4.47"]
+sqlite_requires = ["sqlalchemy~=1.4.50"]
+postgresql_requires = ["psycopg2>=2.9.9", "sqlalchemy~=1.4.50"]
 celery_requires = ["celery[redis]~=5.2.7"]
 sendgrid_requires = ["sendgrid>=6.1.3"]
 flask_requires = ["flask>=1.1.1"]
@@ -41,7 +41,7 @@ def read(*names, **kwargs):
     "cookiecutter>=1.7.0",
     "copier>=6.1.0",
     "inflection>=0.5.1",
-    "python-dateutil>=2.8.1",
+    "python-dateutil>=2.8.2",
     "werkzeug>=2.0.0",
 ]
 
@@ -58,20 +58,20 @@ def read(*names, **kwargs):
 
 testing_requires = all_external_requires + [
     "autoflake>=2.2.1",
-    "isort>=5.10.1",
-    "mock==4.0.2",
-    "pluggy==0.13.1",
-    "pytest-asyncio>=0.15.1",
-    "pytest-cov==2.8.1",
-    "pytest-flake8>=1.0.7",
-    "pytest-mock==3.1.0",
-    "pytest>=5.4.2",
+    "isort>=5.12.0",
+    "mock==5.1.0",
+    "pluggy==1.3.0",
+    "pytest-asyncio>=0.21.1",
+    "pytest-cov>=4.1.0",
+    "pytest-flake8>=1.1.1",
+    "pytest-mock==3.12.0",
+    "pytest>=7.4.3",
 ]
 
 docs_requires = [
     "livereload>=2.6.3",
-    "sphinx>=4.1.2",
-    "sphinx-tabs>=3.2.0",
+    "sphinx>=7.2.6",
+    "sphinx-tabs>=3.4.4",
 ]
 
 types_requires = [
@@ -86,13 +86,13 @@ def read(*names, **kwargs):
     + types_requires
     + testing_requires
     + [
-        "black==21.11b1",
-        "check-manifest==0.42",
-        "coverage==5.1",
-        "docutils==0.16",
+        "black>=23.11.0",
+        "check-manifest>=0.49",
+        "coverage>=7.3.2",
+        "docutils>=0.18.0",
         "pre-commit>=2.16.0",
-        "tox==3.15.0",
-        "twine==3.1.1",
+        "tox>=4.11.3",
+        "twine>=4.0.2",
     ]
 )
 
@@ -123,9 +123,6 @@ def read(*names, **kwargs):
         "License :: OSI Approved :: BSD License",
         "Operating System :: OS Independent",
         "Programming Language :: Python",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
-        "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
         "Programming Language :: Python :: 3.12",
         "Programming Language :: Python :: 3 :: Only",

src/protean/adapters/event_store/memory.py

@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import UTC, datetime
 from typing import Any, Dict, List
 
 from protean import BaseAggregate, BaseRepository
@@ -57,7 +57,7 @@ def write(
                 type=message_type,
                 data=data,
                 metadata=MessageMetadata(**metadata) if metadata else None,
-                time=datetime.utcnow(),
+                time=datetime.now(UTC),
             )
         )
 

src/protean/adapters/repository/elasticsearch.py

@@ -481,7 +481,7 @@ def _data_reset(self):
             if provider.conn_info[
                 "DATABASE"
             ] == Database.ELASTICSEARCH.value and conn.indices.exists(
-                model_cls._index._name
+                index=model_cls._index._name
             ):
                 conn.delete_by_query(
                     refresh=True,
@@ -522,7 +522,7 @@ def _drop_database_artifacts(self):
             if provider.conn_info[
                 "DATABASE"
             ] == Database.ELASTICSEARCH.value and model_cls._index.exists(using=conn):
-                conn.indices.delete(model_cls._index._name)
+                conn.indices.delete(index=model_cls._index._name)
 
 
 class DefaultLookup(BaseLookup):

src/protean/container.py

@@ -164,13 +164,6 @@ def __new__(mcs, name, bases, attrs, **kwargs):
             if attr_name not in fields_dict
         }
 
-        # Propagate `__classcell__` if present to the `type.__new__` call.
-        # Failing to do so will result in a RuntimeError in Python 3.8.
-        # https://docs.python.org/3/reference/datamodel.html#creating-the-class-object
-        classcell = attrs.pop("__classcell__", None)
-        if classcell is not None:
-            dup_attrs["__classcell__"] = classcell
-
         # Insert fields in the order in which they were specified
         #   When field names overlap, the last specified field wins
         dup_attrs.update(fields_dict)

src/protean/domain/helpers.py

@@ -1,6 +1,6 @@
+import importlib.util
 import logging
 import os
-import pkgutil
 import sys
 
 logger = logging.getLogger(__name__)
@@ -40,7 +40,7 @@ def get_root_path(import_name):
         return os.path.dirname(os.path.abspath(mod.__file__))
 
     # Next attempt: check the loader.
-    loader = pkgutil.get_loader(import_name)
+    loader = importlib.util.find_spec(import_name)
 
     # Loader does not exist or we're referring to an unloaded main module
     # or a main module without path (interactive sessions), go with the

src/protean/server/engine.py

@@ -10,7 +10,7 @@
 from protean.core.event_handler import BaseEventHandler
 from protean.exceptions import ConfigurationError
 from protean.globals import g
-from protean.utils.importlib import import_from_full_path
+from protean.utils import import_from_full_path
 from protean.utils.mixins import Message
 
 from .subscription import Subscription

src/protean/utils/__init__.py

@@ -4,14 +4,14 @@
 to the maximum extent possible.
 """
 import functools
+import importlib
 import logging
 
+from datetime import UTC, datetime
 from enum import Enum, auto
 from typing import Any, Tuple, Union
 from uuid import uuid4
 
-import pkg_resources
-
 from protean.exceptions import ConfigurationError, IncorrectUsageError
 from protean.globals import current_domain
 
@@ -64,8 +64,21 @@ def __eq__(self, other):
         return isinstance(other, self.expected_type)
 
 
+def utcnow_func():
+    """Return the current time in UTC with timezone information"""
+    return datetime.now(UTC)
+
+
 def get_version():
-    return pkg_resources.require("protean")[0].version
+    return importlib.metadata.version("protean")
+
+
+def import_from_full_path(domain, path):
+    spec = importlib.util.spec_from_file_location(domain, path)
+    mod = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(mod)
+
+    return getattr(mod, domain)
 
 
 def fully_qualified_name(cls):

src/protean/utils/importlib.py

@@ -1,40 +1 @@
-""" Module defines utilities for importing modules and packages """
 
-import importlib.util
-
-from importlib import import_module
-
-
-def perform_import(val):
-    """
-    If the given setting is a string import notation,
-    then perform the necessary import or imports.
-    """
-    if val is not None:
-        if isinstance(val, str):
-            return import_from_string(val)
-        elif isinstance(val, (list, tuple)):
-            return [import_from_string(item) for item in val]
-
-    return val
-
-
-def import_from_string(val, package=None):
-    """
-    Attempt to import a class from a string representation.
-    """
-    try:
-        module_path, class_name = val.rsplit(".", 1)
-        module = import_module(module_path, package=package)
-        return getattr(module, class_name)
-    except (ImportError, AttributeError) as e:
-        msg = f"Could not import {val}. {e.__class__.__name__}: {e}"
-        raise ImportError(msg)
-
-
-def import_from_full_path(domain, path):
-    spec = importlib.util.spec_from_file_location(domain, path)
-    mod = importlib.util.module_from_spec(spec)
-    spec.loader.exec_module(mod)
-
-    return getattr(mod, domain)

tests/adapters/model/elasticsearch_model/tests.py

@@ -346,7 +346,7 @@ class ReceiverInlineModel:
         model_cls = test_domain.repository_for(Receiver)._model
         conn = test_domain.providers["default"].get_connection()
         if model_cls._index.exists(using=conn):
-            conn.indices.delete(model_cls._index._name)
+            conn.indices.delete(index=model_cls._index._name)
         model_cls.init(using=conn)
 
         receiver_dao = test_domain.repository_for(Receiver)._dao
@@ -356,4 +356,4 @@ class ReceiverInlineModel:
         assert receiver is not None
         assert receiver.name == "John"
 
-        conn.indices.delete(model_cls._index._name)
+        conn.indices.delete(index=model_cls._index._name)

tests/adapters/model/sqlalchemy_model/postgresql/test_array_datatype.py

@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import UTC, datetime
 
 import pytest
 
@@ -110,7 +110,7 @@ def test_array_content_type_validation(test_domain):
         {"email": "[email protected]", "roles": [1, 2]},
         {"email": "[email protected]", "roles": ["1", 2]},
         {"email": "[email protected]", "roles": [1.0, 2.0]},
-        {"email": "[email protected]", "roles": [datetime.utcnow()]},
+        {"email": "[email protected]", "roles": [datetime.now(UTC)]},
     ]:
         with pytest.raises(ValidationError) as exception:
             ArrayUser(**kwargs)
@@ -127,7 +127,7 @@ def test_array_content_type_validation(test_domain):
     for kwargs in [
         {"email": "[email protected]", "roles": ["ADMIN", "USER"]},
         {"email": "[email protected]", "roles": ["1", "2"]},
-        {"email": "[email protected]", "roles": [datetime.utcnow()]},
+        {"email": "[email protected]", "roles": [datetime.now(UTC)]},
     ]:
         with pytest.raises(ValidationError) as exception:
             IntegerArrayUser(**kwargs)

tests/adapters/model/sqlalchemy_model/postgresql/test_json_datatype.py

@@ -1,17 +1,16 @@
-from datetime import datetime
-
 import pytest
 
 from sqlalchemy import types as sa_types
 
 from protean import BaseAggregate
 from protean.fields import DateTime, Dict, String
 from protean.globals import current_domain
+from protean.utils import utcnow_func
 
 
 class Event(BaseAggregate):
     name = String(max_length=255)
-    created_at = DateTime(default=datetime.utcnow())
+    created_at = DateTime(default=utcnow_func)
     payload = Dict()
 
 

tests/adapters/model/sqlalchemy_model/postgresql/test_list_datatype.py

@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import UTC, datetime
 
 import pytest
 
@@ -40,7 +40,7 @@ def test_array_content_type_validation(test_domain):
         {"email": "[email protected]", "roles": [1, 2]},
         {"email": "[email protected]", "roles": ["1", 2]},
         {"email": "[email protected]", "roles": [1.0, 2.0]},
-        {"email": "[email protected]", "roles": [datetime.utcnow()]},
+        {"email": "[email protected]", "roles": [datetime.now(UTC)]},
     ]:
         with pytest.raises(ValidationError) as exception:
             ListUser(**kwargs)
@@ -57,7 +57,7 @@ def test_array_content_type_validation(test_domain):
     for kwargs in [
         {"email": "[email protected]", "roles": ["ADMIN", "USER"]},
         {"email": "[email protected]", "roles": ["1", "2"]},
-        {"email": "[email protected]", "roles": [datetime.utcnow()]},
+        {"email": "[email protected]", "roles": [datetime.now(UTC)]},
     ]:
         with pytest.raises(ValidationError) as exception:
             IntegerListUser(**kwargs)

tests/adapters/model/sqlalchemy_model/sqlite/test_array_datatype.py

@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import UTC, datetime
 
 import pytest
 
@@ -60,7 +60,7 @@ def test_array_content_type_validation(test_domain):
         {"email": "[email protected]", "roles": [1, 2]},
         {"email": "[email protected]", "roles": ["1", 2]},
         {"email": "[email protected]", "roles": [1.0, 2.0]},
-        {"email": "[email protected]", "roles": [datetime.utcnow()]},
+        {"email": "[email protected]", "roles": [datetime.now(UTC)]},
     ]:
         with pytest.raises(ValidationError) as exception:
             ArrayUser(**kwargs)
@@ -77,7 +77,7 @@ def test_array_content_type_validation(test_domain):
     for kwargs in [
         {"email": "[email protected]", "roles": ["ADMIN", "USER"]},
         {"email": "[email protected]", "roles": ["1", "2"]},
-        {"email": "[email protected]", "roles": [datetime.utcnow()]},
+        {"email": "[email protected]", "roles": [datetime.now(UTC)]},
     ]:
         with pytest.raises(ValidationError) as exception:
             IntegerArrayUser(**kwargs)

tests/adapters/model/sqlalchemy_model/sqlite/test_json_datatype.py

@@ -1,16 +1,15 @@
-from datetime import datetime
-
 import pytest
 
 from sqlalchemy import types as sa_types
 
 from protean import BaseAggregate
 from protean.fields import DateTime, Dict, String
+from protean.utils import utcnow_func
 
 
 class Event(BaseAggregate):
     name = String(max_length=255)
-    created_at = DateTime(default=datetime.utcnow())
+    created_at = DateTime(default=utcnow_func)
     payload = Dict()
 
 

tests/adapters/repository/sqlalchemy_repo/postgresql/test_associations.py

@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import UTC, datetime
 
 import pytest
 
@@ -36,7 +36,7 @@ def test_updating_a_has_many_association(test_domain):
 
     post_repo = test_domain.repository_for(Post)
     post = Post(content="bar")
-    post.add_comments(Comment(content="foo", added_on=datetime.utcnow()))
+    post.add_comments(Comment(content="foo", added_on=datetime.now(UTC)))
     post_repo.add(post)
 
     with UnitOfWork():

tests/adapters/repository/sqlalchemy_repo/postgresql/test_persistence.py

@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import UTC, datetime
 
 import pytest
 
@@ -8,7 +8,7 @@
 
 class Event(BaseAggregate):
     name = String(max_length=255)
-    created_at = DateTime(default=datetime.utcnow())
+    created_at = DateTime(default=datetime.now(UTC))
     payload = Dict()