Fix sched-a by-employer,by-occupation and schedule_b/by_recipient fulltext and n/a search issue

tests/test_aggregates.py

@@ -1,3 +1,4 @@
+import sqlalchemy as sa
 from tests import factories
 from tests.common import ApiBaseTest, assert_dicts_subset
 
@@ -133,16 +134,15 @@ def test_disbursement_recipient(self):
         aggregate = factories.ScheduleBByRecipientFactory(
             committee_id=committee.committee_id,
             cycle=committee.cycle,
-            recipient_name='STARBOARD STRATEGIES, INC.',
+            recipient_name_text=sa.func.to_tsvector('STARBOARD STRATEGIES, INC.'),
+            recipient_name='STARBOARD STRATEGIES, INC.'
         )
         results = self._results(
             api.url_for(
                 ScheduleBByRecipientView,
                 committee_id=committee.committee_id,
                 cycle=2012,
-                recipient_name='Starboard Strategies',
-            )
-        )
+                recipient_name='Starboard Strategies',))
         self.assertEqual(len(results), 1)
         expected = {
             'committee_id': committee.committee_id,

webservices/common/models/aggregates.py

@@ -1,3 +1,4 @@
+from sqlalchemy.dialects.postgresql import TSVECTOR
 from webservices import docs, utils
 
 from .base import db, BaseModel
@@ -33,15 +34,15 @@ class ScheduleAByZip(BaseAggregate):
 
 
 class ScheduleAByEmployer(BaseAggregate):
-    __table_args__ = {'schema': 'disclosure'}
-    __tablename__ = 'dsc_sched_a_aggregate_employer'
+    __tablename__ = 'ofec_sched_a_aggregate_employer_mv'
     employer = db.Column(db.String, primary_key=True, doc=docs.EMPLOYER)
+    employer_text = db.Column(TSVECTOR)
 
 
 class ScheduleAByOccupation(BaseAggregate):
-    __table_args__ = {'schema': 'disclosure'}
-    __tablename__ = 'dsc_sched_a_aggregate_occupation'
+    __tablename__ = 'ofec_sched_a_aggregate_occupation_mv'
     occupation = db.Column(db.String, primary_key=True, doc=docs.OCCUPATION)
+    occupation_text = db.Column(TSVECTOR)
 
 
 class BaseDisbursementAggregate(BaseAggregate):
@@ -58,6 +59,7 @@ class ScheduleBByRecipient(BaseDisbursementAggregate):
 
     recipient_name = db.Column('recipient_nm', db.String, primary_key=True, doc=docs.RECIPIENT_NAME)
     committee_total_disbursements = db.Column('disbursements', db.Numeric(30, 2), index=True, doc=docs.DISBURSEMENTS)
+    recipient_name_text = db.Column('recipient_nm_text', TSVECTOR)
 
     @property
     def recipient_disbursement_percent(self):

webservices/common/views.py

@@ -19,6 +19,7 @@ class ApiResource(utils.Resource):
     filter_multi_fields = []
     filter_range_fields = []
     filter_fulltext_fields = []
+    filter_fulltext_fields_NA = []
     filter_overlap_fields = []
     query_options = []
     join_columns = {}
@@ -53,6 +54,7 @@ def build_query(self, *args, _apply_options=True, **kwargs):
         query = filters.filter_multi(query, kwargs, self.filter_multi_fields)
         query = filters.filter_range(query, kwargs, self.filter_range_fields)
         query = filters.filter_fulltext(query, kwargs, self.filter_fulltext_fields)
+        query = filters.filter_fulltext_NA(query, kwargs, self.filter_fulltext_fields_NA)
         query = filters.filter_overlap(query, kwargs, self.filter_overlap_fields)
         if _apply_options:
             query = query.options(*self.query_options)

webservices/filters.py

@@ -100,6 +100,33 @@
     return query
 
 
+# Support N/A search, 'column' is tsvector datatype, 'original_column' is varchar datatype.
+def filter_fulltext_NA(query, kwargs, fields):
+    for key, column, original_column in fields:
+        if kwargs.get(key):
+            exclude_list = build_exclude_list(kwargs.get(key))
+            include_list = build_include_list(kwargs.get(key))
+            if exclude_list:
+                filters = []
+                for value in exclude_list:
+                    if value.strip().upper() == "N/A":
+                        query = query.filter(original_column != 'N/A')
+                    else:
+                        filters.append(sa.not_(column.match(utils.parse_fulltext(value))))
+                query = query.filter(sa.and_(*filters))
+            if include_list:
+                filters = []
+                for value in include_list:
+                    if value.strip().upper() == "N/A":
+                        query = query.filter(original_column == 'N/A')
+                    else:
+                        filters.append(column.match(utils.parse_fulltext(value)))
+                        if value.upper() == 'NULL':
+                            filters.append(column.is_(None))
+                query = query.filter(sa.or_(*filters))
+    return query
+
+
 def filter_multi_start_with(query, kwargs, fields):
     for key, column in fields:
         if kwargs.get(key):

webservices/resources/aggregates.py

@@ -54,8 +54,9 @@ class ScheduleAByEmployerView(AggregateResource):
         ('cycle', models.ScheduleAByEmployer.cycle),
         ('committee_id', models.ScheduleAByEmployer.committee_id),
     ]
-    filter_fulltext_fields = [
-        ('employer', models.ScheduleAByEmployer.employer),
+
+    filter_fulltext_fields_NA = [
+        ('employer', models.ScheduleAByEmployer.employer_text, models.ScheduleAByEmployer.employer),
     ]
 
 
@@ -76,8 +77,8 @@ class ScheduleAByOccupationView(AggregateResource):
         ('cycle', models.ScheduleAByOccupation.cycle),
         ('committee_id', models.ScheduleAByOccupation.committee_id),
     ]
-    filter_fulltext_fields = [
-        ('occupation', models.ScheduleAByOccupation.occupation),
+    filter_fulltext_fields_NA = [
+        ('occupation', models.ScheduleAByOccupation.occupation_text, models.ScheduleAByOccupation.occupation),
     ]
 
 
@@ -179,7 +180,8 @@ class ScheduleBByPurposeView(AggregateResource):
 
 # used for endpoint:`/schedules/schedule_b/by_recipient/`
 # under tag: disbursements
-# Ex: http://127.0.0.1:5000/v1/schedules/schedule_b/by_recipient/
+# Ex: http://127.0.0.1:5000/v1/schedules/schedule_b/by_recipient/?cycle=2022
+# &committee_id=C00160937&recipient_name=zoom.com
 @doc(
     tags=['disbursements'],
     description=docs.SCHEDULE_B_BY_RECIPIENT,
@@ -194,8 +196,8 @@ class ScheduleBByRecipientView(AggregateResource):
         ('cycle', models.ScheduleBByRecipient.cycle),
         ('committee_id', models.ScheduleBByRecipient.committee_id),
     ]
-    filter_fulltext_fields = [
-        ('recipient_name', models.ScheduleBByRecipient.recipient_name),
+    filter_fulltext_fields_NA = [
+        ('recipient_name', models.ScheduleBByRecipient.recipient_name_text, models.ScheduleBByRecipient.recipient_name),
     ]
 
 

webservices/schemas.py

@@ -1060,9 +1060,10 @@ class CandidateHistoryTotalSchema(schemas['CandidateHistorySchema'],
         'recipient_disbursement_percent': ma.fields.Float(),
         'committee_total_disbursements': ma.fields.Float(),
         'total': ma.fields.Float(),
-        'memo_total': ma.fields.Float()
+        'memo_total': ma.fields.Float(),
+        'recipient_name': ma.fields.Str(),
     },
-    options={'exclude': ('idx', 'committee')}
+    options={'exclude': ('idx', 'committee', 'recipient_name_text')}
 )
 
 ScheduleBByRecipientPageSchema = make_page_schema(ScheduleBByRecipientSchema, page_type=paging_schemas.SeekPageSchema)
@@ -1075,9 +1076,31 @@ class CandidateHistoryTotalSchema(schemas['CandidateHistorySchema'],
     models.ScheduleAByZip,
     models.ScheduleABySize,
     models.ScheduleAByState,
+    models.ScheduleBByPurpose,
+)
+
+ScheduleAByEmployerSchema = make_schema(
     models.ScheduleAByEmployer,
+    fields={
+        'committee': ma.fields.Nested(schemas['CommitteeHistorySchema']),
+        'employer': ma.fields.Str(),
+    },
+    options={'exclude': ('idx', 'committee', 'employer_text',)}
+)
+ScheduleAByEmployerPageSchema = make_page_schema(
+    ScheduleAByEmployerSchema
+)
+
+ScheduleAByOccupationSchema = make_schema(
     models.ScheduleAByOccupation,
-    models.ScheduleBByPurpose,
+    fields={
+        'committee': ma.fields.Nested(schemas['CommitteeHistorySchema']),
+        'occupation': ma.fields.Str(),
+    },
+    options={'exclude': ('idx', 'committee', 'occupation_text',)}
+)
+ScheduleAByOccupationPageSchema = make_page_schema(
+    ScheduleAByOccupationSchema
 )
 
 make_aggregate_schema = functools.partial(