Ogc 508 replace elastic search by postgres v3

src/onegov/org/models/search.py

@@ -1,19 +1,18 @@
-import itertools
-
 from elasticsearch_dsl.function import SF  # type:ignore
 from elasticsearch_dsl.query import FunctionScore  # type:ignore
 from elasticsearch_dsl.query import Match
 from elasticsearch_dsl.query import MatchPhrase
 from elasticsearch_dsl.query import MultiMatch
 from functools import cached_property
+from itertools import chain, repeat
 from sedate import utcnow
 from sqlalchemy import func
 from typing import TYPE_CHECKING, Any
 
 from onegov.core.collection import Pagination, _M
 from onegov.event.models import Event
 from onegov.search.utils import (searchable_sqlalchemy_models,
-                                 filter_non_base_models)
+                                 filter_for_base_models)
 
 if TYPE_CHECKING:
     from onegov.org.request import OrgRequest
@@ -156,7 +155,7 @@
             first_entry = self.batch[0].load()
 
             # XXX the default view to the event should be doing the redirect
             if first_entry.es_type_name == 'events':
                 return self.request.link(first_entry, 'latest')
             else:
                 return self.request.link(first_entry)
@@ -193,9 +192,10 @@
         self.number_of_docs = 0
         self.number_of_results = 0
 
-        models = (model for base in self.request.app.session_manager.bases
-                  for model in searchable_sqlalchemy_models(base))
-        self.search_models = filter_non_base_models(set(models))
+        self.search_models = {
+            model for base in self.request.app.session_manager.bases
+            for model in searchable_sqlalchemy_models(base)}
+        self.search_models = filter_for_base_models(self.search_models)
 
     @cached_property
     def available_documents(self) -> int:
@@ -218,24 +218,24 @@
         return self.query
 
     def __eq__(self, other: object) -> bool:
         if not isinstance(other, SearchPostgres):
             return NotImplemented
         return self.page == other.page and self.query == other.query
 
     def subset(self) -> 'list[Searchable] | None':  # type:ignore[override]
         return self.batch
 
     @property
     def page_index(self) -> int:
         return self.page
 
     def page_by_index(self, index: int) -> 'SearchPostgres[_M]':
         return SearchPostgres(self.request, self.query, index)
 
     @cached_property
     def batch(self) -> 'list[Searchable]':  # type:ignore[override]
         if not self.query:
             return []
 
         if self.query.startswith('#'):
             results = self.hashtag_search()
@@ -289,7 +289,7 @@
             )
             for field, weight in zip(
                 model.es_properties.keys(),
-                itertools.chain('A', itertools.repeat('B')))
+                chain('A', repeat('B')))
             if not field.startswith('es_')  # TODO: rename to fts_
         ]
 
@@ -299,7 +299,7 @@
             for vector in weighted_vectors[1:]:
                 combined_vector = combined_vector.op('||')(vector)
         else:
             combined_vector = func.to_tsvector(language, '')
 
         return combined_vector
 
@@ -376,15 +376,15 @@
         return results
 
     def feeling_lucky(self) -> str | None:
         if self.batch:
             first_entry = self.batch[0]
 
             # XXX the default view to the event should be doing the redirect
             if isinstance(first_entry, Event):
                 return self.request.link(first_entry, 'latest')
             else:
                 return self.request.link(first_entry)
         return None
 
     @cached_property
     def subset_count(self) -> int:
@@ -393,37 +393,37 @@
     @cached_property
     def get_all_hashtags(self) -> list[str]:
         """ Returns all hashtags from the database in alphabetical order. """
         all_tags: set[str] = set()
 
         for base in self.request.app.session_manager.bases:
             for model in searchable_sqlalchemy_models(base):
                 query = self.request.session.query(
                     model.fts_idx_data['es_tags'].distinct())
                 for tag_list in query.all():
                     all_tags.update(tag_list[0]) if tag_list[0] else None
 
         # mark tags as hashtags; it also helps ot remain with the hashtag
         # search (url) when clicking on a suggestion
         all_tags = {f'#{tag}' for tag in all_tags}
         return sorted(all_tags)
 
     def suggestions(self) -> tuple[str, ...]:
         suggestions = []
         number_of_suggestions = 15
 
         if self.query.startswith('#'):  # hashtag search
             q = self.query.lstrip('#').lower()
             tags = self.get_all_hashtags
 
             if len(q) == 0:
                 return tuple(tags[:number_of_suggestions])
 
             suggestions = [tag for tag in tags if q in tag]
 
         else:
             for element in self.generic_search():
                 if element.es_type_name == 'files':
                     continue
                 suggest = getattr(element, 'es_suggestion', '')
                 if isinstance(suggest, tuple):
                     suggest = suggest[0]

src/onegov/search/integration.py

@@ -16,10 +16,9 @@
 from onegov.search.indexer import ORMEventTranslator
 from onegov.search.indexer import TypeMappingRegistry
 from onegov.search.utils import (searchable_sqlalchemy_models,
-                                 filter_non_base_models)
+                                 filter_for_base_models)
 from sortedcontainers import SortedSet
 from sedate import utcnow
-from sqlalchemy import inspect
 from sqlalchemy.orm import undefer
 from urllib3.exceptions import HTTPError
 
@@ -444,28 +443,24 @@
             """ Load all database objects and index them. """
             session = self.session()
             try:
-                q = session.query(model).options(undefer('*'))
-                i = inspect(model)
-
-                if i.polymorphic_on is not None:
-                    q = q.filter(i.polymorphic_on == i.polymorphic_identity)
-
-                for obj in q:
+                for obj in session.query(model).options(undefer('*')):
                     self.es_orm_events.index(schema, obj)
 
             except Exception as e:
-                print(f"Error psql indexing model '{model}': {e}")
+                print(f"Error psql indexing model '{model.__name__}': {e}")
             finally:
                 session.invalidate()
                 session.bind.dispose()
 
-        models = (model for base in self.session_manager.bases
-                  for model in searchable_sqlalchemy_models(base))
+        models = {
+            model
+            for base in self.session_manager.bases
+            for model in searchable_sqlalchemy_models(base)
+        }
+        base_models = filter_for_base_models(models)
+
         with ThreadPoolExecutor() as executor:
-            results = executor.map(
-                reindex_model, (
-                    model for model in filter_non_base_models(set(models)))
-            )
+            results = executor.map(reindex_model, base_models)
             if fail:
                 print(tuple(results))
 

src/onegov/search/utils.py

@@ -3,14 +3,17 @@
 import os
 import re
 
+from sqlalchemy import inspect
+
 from onegov.core.custom import json
 from langdetect import DetectorFactory, PROFILES_DIRECTORY
 from langdetect.utils.lang_profile import LangProfile
 from onegov.core.orm import find_models
 
 from typing import Any, Generic, TypeVar, TYPE_CHECKING
+
 if TYPE_CHECKING:
-    from collections.abc import Callable, Iterator, Sequence
+    from collections.abc import Callable, Iterable, Iterator, Sequence
     from langdetect.detector import Detector
     from langdetect.language import Language
     from onegov.search.mixins import Searchable
@@ -42,29 +45,34 @@ def searchable_sqlalchemy_models(
     )
 
 
-def filter_non_base_models(
-    models: 'set[type[T]]'
-) -> 'set[type[T]]':
-    """ Remove model classes that are base classes of other models in the set.
-    Args: models: set of model classes to filter
-    Returns: set: Model classes that are not base classes of any other model
-    in the set.
-
+def filter_for_base_models(
+    models: 'Iterable[type[Searchable]]'
+) -> 'set[type[Searchable]]':
     """
-    non_base_models = set()
+    Filter out models that are polymorphic subclasses of other
+    models in order to save on queries.
 
-    for model in models:
-        is_base = False
-        for other_model in models:
-            if (model is not other_model and issubclass(other_model, model)
-                    and model.__tablename__ == other_model.__tablename__):  # type:ignore[attr-defined]
-                is_base = True
-                break
+    """
+    from onegov.search.mixins import Searchable
 
-        if not is_base:
-            non_base_models.add(model)
+    new_models = set()
 
-    return non_base_models
+    for model in models:
+        i = inspect(model)
+        base_classes = {
+            e.base_mapper.class_ for e in
+            i.base_mapper.self_and_descendants
+            if issubclass(e.base_mapper.class_, Searchable)
+            if e.polymorphic_identity is not None
+            if e.base_mapper.class_ != model
+        }
+        if base_classes:
+            for base in base_classes:
+                new_models.add(base)
+        else:
+            new_models.add(model)
+
+    return new_models
 
 
 _invalid_index_characters = re.compile(r'[\\/?"<>|\s,A-Z:]+')

tests/onegov/search/test_utils.py

@@ -1,4 +1,4 @@
-from onegov.search import ORMSearchable, Searchable
+from onegov.search import ORMSearchable, Searchable, SearchableContent
 from onegov.search import utils
 from sqlalchemy import Column, Integer, Text, ForeignKey
 from sqlalchemy.ext.declarative import declarative_base
@@ -50,42 +50,66 @@ class B(Base):
     ]
 
 
-def test_filter_non_base_models():
+def test_filter_for_base_models():
     Base = declarative_base()
 
-    class Page(Base, Searchable):
+    class Ticket(Base, ORMSearchable):
+        __tablename__ = 'tickets'
         id = Column(Integer, primary_key=True)
-        __tablename__ = 'pages'
 
-    class Topic(Page):
-        pass
+    class XTicket(Ticket):
+        __mapper_args__ = {'polymorphic_identity': 'X'}  # type:ignore
 
-    class News(Page):
-        pass
+    class YTicket(Ticket):
+        __mapper_args__ = {'polymorphic_identity': 'Y'}  # type:ignore
 
-    assert utils.filter_non_base_models({Page, Topic, News}) == {Topic, News}
+    assert utils.filter_for_base_models({XTicket, YTicket, Ticket}) == {Ticket}
 
     class A(Base, Searchable):
+        __mapper_args__ = {'polymorphic_identity': 'a'}  # type:ignore
         id = Column(Integer, primary_key=True)
         __tablename__ = 'a'
 
     class AA(A):
+        __mapper_args__ = {'polymorphic_identity': 'aa'}  # type:ignore
         pass
 
     class B(Base, Searchable):
+        __mapper_args__ = {'polymorphic_identity': 'b'}  # type:ignore
         id = Column(Integer, primary_key=True)
         __tablename__ = 'b'
 
     class C(Base, Searchable):
+        __mapper_args__ = {'polymorphic_identity': 'c'}  # type:ignore
         id = Column(Integer, primary_key=True)
         __tablename__ = 'c'
 
     class CC(C):
+        __mapper_args__ = {'polymorphic_identity': 'cc'}  # type:ignore
         id_2 = Column(Integer, primary_key=True)
         c_id = Column(Integer, ForeignKey('c.id'))
         __tablename__ = 'cc'
 
-    assert utils.filter_non_base_models({A, AA, B, C, CC}) == {AA, B, C, CC}
+    assert utils.filter_for_base_models({A, AA, B, C, CC}) == {A, B, C}
+
+    class AdjacencyList(Base):
+        __abstract__ = True
+        __mapper_args__ = {'polymorphic_identity': 'generic'}
+
+    class Page(AdjacencyList):
+        __tablename__ = 'pages'
+        id = Column(Integer, primary_key=True)
+
+    class Topic(Page, SearchableContent):
+        __mapper_args__ = {'polymorphic_identity': 'topic'}
+
+    class News(Page, SearchableContent):
+        __mapper_args__ = {'polymorphic_identity': 'news'}
+
+    searchable_models = {
+        m for m in utils.searchable_sqlalchemy_models(Base)}
+    assert utils.filter_for_base_models(searchable_models) == {
+        Topic, News, Ticket, A, B, C}
 
 
 def test_related_types():