Merge pull request #1626 from freedomofpress/search-filter-performance

Incident search filter improvements

common/blocks.py

@@ -1,4 +1,5 @@
 import re
+import bleach
 
 from django.forms.utils import ErrorList
 
@@ -11,6 +12,7 @@
 from common.choices import BACKGROUND_COLOR_CHOICES
 from common.models.helpers import get_tags
 from common.templatetags.render_as_template import render_as_template
+from common.search import get_searchable_content_for_fields
 from common.utils import unescape
 from common.validators import validate_template
 
@@ -94,6 +96,11 @@ class AlignedCaptionedImageBlock(blocks.StructBlock):
     )
     alignment = blocks.ChoiceBlock(choices=ALIGNMENT_CHOICES)
 
+    def get_searchable_content(self, value):
+        return get_searchable_content_for_fields(
+            value, self.child_blocks, ['caption']
+        )
+
     class Meta:
         template = 'common/blocks/aligned_captioned_image.html'
         icon = 'image'
@@ -113,6 +120,11 @@ class AlignedCaptionedEmbedBlock(blocks.StructBlock):
     )
     alignment = blocks.ChoiceBlock(choices=ALIGNMENT_CHOICES)
 
+    def get_searchable_content(self, value):
+        return get_searchable_content_for_fields(
+            value, self.child_blocks, ['caption', 'attribution'],
+        )
+
     class Meta:
         template = 'common/blocks/aligned_captioned_embed.html'
         icon = 'media'
@@ -143,6 +155,13 @@ def clean(self, value):
 
         return super().clean(value)
 
+    def get_searchable_content(self, value):
+        tweet_content = value.get('tweet', None)
+        if tweet_content and tweet_content.html:
+            return [bleach.clean(tweet_content.html, strip=True, tags={})]
+        else:
+            return []
+
 
 class AsideBlock(blocks.StructBlock):
     text = blocks.RichTextBlock(

common/search.py

@@ -0,0 +1,14 @@
+def get_searchable_content_for_fields(value, child_blocks, fields_to_index):
+    """Returns the searchable content for a subset of indexable fields
+
+    Intended to be used with StructBlocks or other collections of
+    blocks where not all fields should be included in the public
+    index.
+
+    `fields_to_index` should be a list of field names.
+
+    """
+    content = []
+    for field in fields_to_index:
+        content.extend(child_blocks[field].get_searchable_content(value[field]))
+    return content

common/tests/factories.py

@@ -1,13 +1,21 @@
 import factory
 import wagtail_factories
+from wagtail_factories.blocks import BlockFactory
 from wagtail import blocks
 from wagtail.rich_text import RichText
 
 from common.blocks import (
+    ALIGNMENT_CHOICES,
     Heading1,
     Heading2,
     Heading3,
     StyledTextBlock,
+    RichTextTemplateBlock,
+    AlignedCaptionedImageBlock,
+    TweetEmbedBlock,
+    PullQuoteBlock,
+    RichTextBlockQuoteBlock,
+    AlignedCaptionedEmbedBlock,
 )
 from common.choices import CATEGORY_SYMBOL_CHOICES
 from common.models import (
@@ -24,6 +32,55 @@
 )
 
 
+class RichTextTemplateBlockFactory(BlockFactory):
+    class Meta:
+        model = RichTextTemplateBlock
+
+
+class AlignedCaptionedImageBlockFactory(wagtail_factories.StructBlockFactory):
+    image = factory.SubFactory(
+        wagtail_factories.blocks.ImageChooserBlockFactory,
+    )
+    caption = RichText('Caption')
+    alignment = ALIGNMENT_CHOICES[0][0]
+
+    class Meta:
+        model = AlignedCaptionedImageBlock
+
+
+class TweetEmbedBlockFactory(wagtail_factories.StructBlockFactory):
+    tweet = factory.SubFactory(wagtail_factories.blocks.CharBlockFactory)
+
+    class Meta:
+        model = TweetEmbedBlock
+
+
+class RichTextBlockQuoteBlockFactory(wagtail_factories.StructBlockFactory):
+    text = RichText('Quote content')
+    source_text = RichText('Name of source')
+    source_url = 'https://freedom.press'
+
+    class Meta:
+        model = RichTextBlockQuoteBlock
+
+
+class PullQuoteBlockFactory(wagtail_factories.StructBlockFactory):
+    text = 'Text of Quote'
+
+    class Meta:
+        model = PullQuoteBlock
+
+
+class AlignedCaptionedEmbedBlockFactory(wagtail_factories.StructBlockFactory):
+    caption = RichText('Embed caption')
+    attribution = 'Attribution of embed'
+    video = factory.SubFactory(wagtail_factories.blocks.CharBlockFactory)
+    alignment = ALIGNMENT_CHOICES[0][0]
+
+    class Meta:
+        model = AlignedCaptionedEmbedBlock
+
+
 class DevelopmentSiteFactory(wagtail_factories.SiteFactory):
     class Meta:
         django_get_or_create = ('is_default_site',)

incident/models/incident_page.py

@@ -31,7 +31,7 @@
 from wagtail.fields import StreamField, RichTextField
 from wagtail.models import Page, Orderable, PageManager, PageQuerySet
 from wagtail.images.blocks import ImageChooserBlock
-
+from wagtail.search import index
 from wagtailautocomplete.edit_handlers import AutocompletePanel
 from common.blocks import (
     RichTextBlockQuoteBlock,
@@ -770,6 +770,24 @@ class IncidentPage(MetadataPageMixin, Page):
 
     parent_page_types = ['incident.IncidentIndexPage']
 
+    search_fields = Page.search_fields + [
+        index.SearchField('body'),
+        index.SearchField('city'),
+        index.RelatedFields('state', [
+            index.SearchField('name'),
+        ]),
+        index.SearchField('introduction'),
+        index.SearchField('teaser'),
+        index.RelatedFields('teaser_image', [
+            index.SearchField('attribution'),
+        ]),
+        index.SearchField('image_caption'),
+        index.RelatedFields('updates', [
+            index.SearchField('title'),
+            index.SearchField('body'),
+        ]),
+    ]
+
     def get_context(self, request, *args, **kwargs):
         context = super(IncidentPage, self).get_context(request, *args, **kwargs)
 

incident/tests/factories.py

@@ -30,9 +30,17 @@
     Venue,
 )
 from common.tests.factories import (
+    CustomImageFactory,
     CategoryPageFactory,
     CommonTagFactory,
     PersonPageFactory,
+    RichTextTemplateBlockFactory,
+    AlignedCaptionedImageBlockFactory,
+    RawHTMLBlockFactory,
+    TweetEmbedBlockFactory,
+    RichTextBlockQuoteBlockFactory,
+    PullQuoteBlockFactory,
+    AlignedCaptionedEmbedBlockFactory,
 )
 from common.tests.utils import StreamfieldProvider
 from menus.factories import MainMenuItemFactory
@@ -106,6 +114,19 @@ class Meta:
     body = None
 
 
+class IncidentUpdateWithBodyFactory(IncidentUpdateFactory):
+    body = wagtail_factories.StreamFieldFactory({
+        'rich_text': factory.SubFactory(RichTextTemplateBlockFactory),
+        'image': factory.SubFactory(
+            wagtail_factories.blocks.ImageChooserBlockFactory
+        ),
+        'raw_html': factory.SubFactory(RawHTMLBlockFactory),
+        'tweet': factory.SubFactory(TweetEmbedBlockFactory),
+        'blockquote': factory.SubFactory(RichTextBlockQuoteBlockFactory),
+        'video': factory.SubFactory(AlignedCaptionedEmbedBlockFactory),
+    })
+
+
 class IncidentLinkFactory(factory.django.DjangoModelFactory):
     class Meta:
         model = IncidentPageLinks
@@ -140,7 +161,7 @@ class Meta:
     title = factory.Sequence(lambda n: f'Incident {n}')
     date = factory.LazyFunction(datetime.date.today)
     city = None
-    state = None
+    state = factory.SubFactory(StateFactory)
     longitude = None
     latitude = None
     body = None
@@ -408,6 +429,22 @@ def related_incidents(self, create, extracted, **kwargs):
             self.related_incidents.set(extracted)
 
 
+class IncidentPageWithBodyFactory(IncidentPageFactory):
+    teaser_image = factory.SubFactory(CustomImageFactory)
+    body = wagtail_factories.StreamFieldFactory({
+        'rich_text': factory.SubFactory(RichTextTemplateBlockFactory),
+        'image': factory.SubFactory(
+            wagtail_factories.blocks.ImageChooserBlockFactory
+        ),
+        'aligned_image': factory.SubFactory(AlignedCaptionedImageBlockFactory),
+        'raw_html': factory.SubFactory(RawHTMLBlockFactory),
+        'tweet': factory.SubFactory(TweetEmbedBlockFactory),
+        'blockquote': factory.SubFactory(RichTextBlockQuoteBlockFactory),
+        'pull_quote': factory.SubFactory(PullQuoteBlockFactory),
+        'video': factory.SubFactory(AlignedCaptionedEmbedBlockFactory),
+    })
+
+
 class InexactDateIncidentPageFactory(IncidentPageFactory):
     exact_date_unknown = True
     date = datetime.date(2017, 3, 1)