From 469ad5f701c7b06b6338cd51d7ab2e6b4c391cce Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Wed, 11 Oct 2023 13:31:38 -0400 Subject: [PATCH 01/70] Adds calendar_ytd calculation (WIP) --- .../fecfiler/transactions/managers.py | 37 ++++++++++++++++++- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/django-backend/fecfiler/transactions/managers.py b/django-backend/fecfiler/transactions/managers.py index 5274699369..6ae7aaeee2 100644 --- a/django-backend/fecfiler/transactions/managers.py +++ b/django-backend/fecfiler/transactions/managers.py @@ -21,10 +21,13 @@ BooleanField, TextField, DecimalField, + UUIDField, + CASCADE, ) from decimal import Decimal from enum import Enum from .schedule_b.managers import refunds as schedule_b_refunds +from fecfiler.contacts.models import Contact """Manager to deterimine fields that are used the same way across transactions, but are called different names""" @@ -59,10 +62,12 @@ def get_queryset(self): "schedule_d__incurred_amount", ), effective_amount=self.get_amount_clause(), + candidate_contact_id=self.get_candidate_contact_clause(), ) ) - contact_clause = Q(contact_1_id=OuterRef("contact_1_id")) + contact_1_clause = Q(contact_1_id=OuterRef("contact_1_id")) + candidate_contact_clause = Q(candidate_contact_id__isnull=False) & Q(candidate_contact_id=OuterRef("candidate_contact_id")) year_clause = Q(date__year=OuterRef("date__year")) date_clause = Q(date__lt=OuterRef("date")) | Q( date=OuterRef("date"), created__lte=OuterRef("created") @@ -72,7 +77,7 @@ def get_queryset(self): aggregate_clause = ( queryset.filter( - contact_clause, + contact_1_clause, year_clause, date_clause, group_clause, @@ -83,6 +88,19 @@ def get_queryset(self): .values("aggregate") ) + calendar_ytd_clause = ( + queryset.filter( + candidate_contact_clause, + year_clause, + date_clause, + group_clause, + force_unaggregated_clause, + ) + .values("committee_account_id") + .annotate(calendar_ytd=Sum("effective_amount")) + .values("calendar_ytd") + ) + loan_payment_to_date_clause = ( queryset.filter( loan__transaction_id=OuterRef("transaction_id"), @@ -133,6 +151,7 @@ def get_queryset(self): return ( queryset.annotate( aggregate=Coalesce(Subquery(aggregate_clause), Value(Decimal(0))), + calendar_ytd=Coalesce(Subquery(calendar_ytd_clause), Value(Decimal(0))), loan_payment_to_date=Case( When( schedule_c__isnull=False, @@ -272,6 +291,20 @@ def get_amount_clause(self): output_field=DecimalField(), ) + def get_candidate_contact_clause(self): + return Case( + When( + contact_2__type=Contact.ContactType.CANDIDATE, + then=F("contact_2_id") + ), + When( + contact_3__type=Contact.ContactType.CANDIDATE, + then=F("contact_3_id") + ), + default=None, + output_field=UUIDField() + ) + class Schedule(Enum): A = Value("A") From 9042547f7cdd117f2a1941f5682757044cd2d94d Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Wed, 11 Oct 2023 14:19:49 -0400 Subject: [PATCH 02/70] calendar_ytd is populating --- django-backend/fecfiler/transactions/managers.py | 12 +++++++++--- django-backend/fecfiler/transactions/serializers.py | 2 ++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/django-backend/fecfiler/transactions/managers.py b/django-backend/fecfiler/transactions/managers.py index 6ae7aaeee2..bef39f0a88 100644 --- a/django-backend/fecfiler/transactions/managers.py +++ b/django-backend/fecfiler/transactions/managers.py @@ -22,7 +22,6 @@ TextField, DecimalField, UUIDField, - CASCADE, ) from decimal import Decimal from enum import Enum @@ -52,6 +51,14 @@ def get_queryset(self): "schedule_a__contribution_date", "schedule_b__expenditure_date", "schedule_c__loan_incurred_date", + Case( + When( + schedule_e__isnull=False, then=Case( + When(schedule_e__disbursement_date__isnull=True, then=F("schedule_e__dissemination_date")), + default=F("schedule_e__disbursement_date") + ) + ) + ) ), amount=Coalesce( "schedule_a__contribution_amount", @@ -60,6 +67,7 @@ def get_queryset(self): "schedule_c2__guaranteed_amount", "debt__schedule_d__incurred_amount", "schedule_d__incurred_amount", + "schedule_e__expenditure_amount", ), effective_amount=self.get_amount_clause(), candidate_contact_id=self.get_candidate_contact_clause(), @@ -87,14 +95,12 @@ def get_queryset(self): .annotate(aggregate=Sum("effective_amount")) .values("aggregate") ) - calendar_ytd_clause = ( queryset.filter( candidate_contact_clause, year_clause, date_clause, group_clause, - force_unaggregated_clause, ) .values("committee_account_id") .annotate(calendar_ytd=Sum("effective_amount")) diff --git a/django-backend/fecfiler/transactions/serializers.py b/django-backend/fecfiler/transactions/serializers.py index 67993c1240..b297b2947e 100644 --- a/django-backend/fecfiler/transactions/serializers.py +++ b/django-backend/fecfiler/transactions/serializers.py @@ -129,6 +129,7 @@ class TransactionSerializerBase( date = DateField(read_only=True) amount = DecimalField(max_digits=11, decimal_places=2, read_only=True) aggregate = DecimalField(max_digits=11, decimal_places=2, read_only=True) + calendar_ytd = DecimalField(max_digits=11, decimal_places=2, read_only=True) balance = DecimalField(max_digits=11, decimal_places=2, read_only=True) loan_payment_to_date = DecimalField(max_digits=11, decimal_places=2, read_only=True) loan_balance = DecimalField(max_digits=11, decimal_places=2, read_only=True) @@ -311,6 +312,7 @@ def get_fields(): "date", "amount", "aggregate", + "calendar_ytd", "loan_payment_to_date", "balance", "loan_balance", From ffa3c5d64b8af50066382392988a811e79047d68 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Mon, 16 Oct 2023 02:51:10 -0400 Subject: [PATCH 03/70] Adds election_code support --- django-backend/fecfiler/transactions/views.py | 45 +++++++++++++------ 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/django-backend/fecfiler/transactions/views.py b/django-backend/fecfiler/transactions/views.py index 9a933bdaf6..5b3de22766 100644 --- a/django-backend/fecfiler/transactions/views.py +++ b/django-backend/fecfiler/transactions/views.py @@ -176,31 +176,48 @@ def get_queryset(self): @action(detail=False, methods=["get"]) def previous(self, request): """Retrieves transaction that comes before this transactions, - while bieng in the same group for aggregation""" + while being in the same group for aggregation""" transaction_id = request.query_params.get("transaction_id", None) contact_1_id = request.query_params.get("contact_1_id", None) + candidate_id = request.query_params.get("candidate_id", None) date = request.query_params.get("date", None) aggregation_group = request.query_params.get("aggregation_group", None) - if not (contact_1_id and date): + election_code = request.query_params.get("election_code", None) + + if not date or not (contact_1_id or candidate_id): return Response( - "Please provide contact_1_id and date in query params.", + "Please provide date and either contact_1_id or candidate_id in query params.", status=status.HTTP_400_BAD_REQUEST, ) date = datetime.fromisoformat(date) - previous_transaction = ( - self.get_queryset() - .filter( - ~Q(id=transaction_id or None), - Q(contact_1_id=contact_1_id), - Q(date__year=date.year), - Q(date__lte=date), - Q(aggregation_group=aggregation_group), + if candidate_id == None: + previous_transactions = ( + self.get_queryset() + .filter( + ~Q(id=transaction_id or None), + Q(contact_1_id=contact_1_id), + Q(date__year=date.year), + Q(date__lte=date), + Q(aggregation_group=aggregation_group), + ) ) - .order_by("-date", "-created") - .first() - ) + else: + previous_transactions = ( + self.get_queryset() + .filter( + ~Q(id=transaction_id or None), + Q(contact_2_id=candidate_id) | Q(contact_3_id=candidate_id), + Q(date__year=date.year), + Q(date__lte=date), + Q(aggregation_group=aggregation_group), + Q(schedule_e__election_code=election_code) + ) + ) + + previous_transaction = previous_transactions.order_by("-date", "-created").first() + if previous_transaction: serializer = self.get_serializer(previous_transaction) From 39370ba134d0006ad16759ccb2b5dd45745e4081 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Mon, 16 Oct 2023 09:55:50 -0400 Subject: [PATCH 04/70] Fixes linting errors --- .../fecfiler/transactions/managers.py | 11 +++++++++-- django-backend/fecfiler/transactions/views.py | 18 ++++++++++++++---- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/django-backend/fecfiler/transactions/managers.py b/django-backend/fecfiler/transactions/managers.py index bef39f0a88..396d89e438 100644 --- a/django-backend/fecfiler/transactions/managers.py +++ b/django-backend/fecfiler/transactions/managers.py @@ -54,7 +54,10 @@ def get_queryset(self): Case( When( schedule_e__isnull=False, then=Case( - When(schedule_e__disbursement_date__isnull=True, then=F("schedule_e__dissemination_date")), + When( + schedule_e__disbursement_date__isnull=True, + then=F("schedule_e__dissemination_date") + ), default=F("schedule_e__disbursement_date") ) ) @@ -75,7 +78,11 @@ def get_queryset(self): ) contact_1_clause = Q(contact_1_id=OuterRef("contact_1_id")) - candidate_contact_clause = Q(candidate_contact_id__isnull=False) & Q(candidate_contact_id=OuterRef("candidate_contact_id")) + candidate_contact_clause = Q( + candidate_contact_id__isnull=False + ) & Q( + candidate_contact_id=OuterRef("candidate_contact_id") + ) year_clause = Q(date__year=OuterRef("date__year")) date_clause = Q(date__lt=OuterRef("date")) | Q( date=OuterRef("date"), created__lte=OuterRef("created") diff --git a/django-backend/fecfiler/transactions/views.py b/django-backend/fecfiler/transactions/views.py index 5b3de22766..7dc7c4f10d 100644 --- a/django-backend/fecfiler/transactions/views.py +++ b/django-backend/fecfiler/transactions/views.py @@ -184,15 +184,26 @@ def previous(self, request): aggregation_group = request.query_params.get("aggregation_group", None) election_code = request.query_params.get("election_code", None) - if not date or not (contact_1_id or candidate_id): + missing_params = [] + if not date: + missing_params.append("date") + if not aggregation_group: + missing_params.append("aggregation_group") + if not contact_1_id and not candidate_id: + missing_params.append("and either contact_1_id or candidate_id") + if candidate_id and not election_code: + missing_params.append("election_code") + + if len(missing_params) > 0: + response_str = "Please provide"+",".join(missing_params)+" in query params" return Response( - "Please provide date and either contact_1_id or candidate_id in query params.", + response_str, status=status.HTTP_400_BAD_REQUEST, ) date = datetime.fromisoformat(date) - if candidate_id == None: + if candidate_id is None: previous_transactions = ( self.get_queryset() .filter( @@ -218,7 +229,6 @@ def previous(self, request): previous_transaction = previous_transactions.order_by("-date", "-created").first() - if previous_transaction: serializer = self.get_serializer(previous_transaction) return Response(data=serializer.data) From 185acb370f6b119b6740a6bab751a1f993261a59 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Mon, 16 Oct 2023 11:12:21 -0400 Subject: [PATCH 05/70] Simplifies 'date' annotation --- django-backend/fecfiler/transactions/managers.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/django-backend/fecfiler/transactions/managers.py b/django-backend/fecfiler/transactions/managers.py index 59025deaaf..43d6890abe 100644 --- a/django-backend/fecfiler/transactions/managers.py +++ b/django-backend/fecfiler/transactions/managers.py @@ -51,17 +51,8 @@ def get_queryset(self): "schedule_a__contribution_date", "schedule_b__expenditure_date", "schedule_c__loan_incurred_date", - Case( - When( - schedule_e__isnull=False, then=Case( - When( - schedule_e__disbursement_date__isnull=True, - then=F("schedule_e__dissemination_date") - ), - default=F("schedule_e__disbursement_date") - ) - ) - ) + "schedule_e__disbursement_date" + "schedule_e__dissemination_date" ), amount=Coalesce( "schedule_a__contribution_amount", From 8349ca64a1151daf2c0cded333fce97eb8e013e7 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Tue, 17 Oct 2023 12:39:41 -0400 Subject: [PATCH 06/70] Separates Entity Aggregate from Election Aggregate --- .../fecfiler/transactions/managers.py | 2 +- django-backend/fecfiler/transactions/views.py | 101 +++++++++++++----- 2 files changed, 73 insertions(+), 30 deletions(-) diff --git a/django-backend/fecfiler/transactions/managers.py b/django-backend/fecfiler/transactions/managers.py index 43d6890abe..d454562088 100644 --- a/django-backend/fecfiler/transactions/managers.py +++ b/django-backend/fecfiler/transactions/managers.py @@ -51,7 +51,7 @@ def get_queryset(self): "schedule_a__contribution_date", "schedule_b__expenditure_date", "schedule_c__loan_incurred_date", - "schedule_e__disbursement_date" + "schedule_e__disbursement_date", "schedule_e__dissemination_date" ), amount=Coalesce( diff --git a/django-backend/fecfiler/transactions/views.py b/django-backend/fecfiler/transactions/views.py index 1b528965d1..b4f3285ff4 100644 --- a/django-backend/fecfiler/transactions/views.py +++ b/django-backend/fecfiler/transactions/views.py @@ -14,6 +14,7 @@ from fecfiler.transactions.models import Transaction from fecfiler.transactions.serializers import TransactionSerializerBase from fecfiler.contacts.serializers import ContactSerializer +from fecfiler.contacts.models import Contact from fecfiler.transactions.schedule_a.serializers import ScheduleATransactionSerializer from fecfiler.transactions.schedule_b.serializers import ScheduleBTransactionSerializer from fecfiler.transactions.schedule_c.serializers import ScheduleCTransactionSerializer @@ -177,26 +178,81 @@ def get_queryset(self): queryset = queryset.filter(schedule_e__isnull=True) return queryset - @action(detail=False, methods=["get"]) - def previous(self, request): + @action(detail=False, methods=["get"], url_path=r"previous/election") + def previous_election(self, request): """Retrieves transaction that comes before this transactions, while being in the same group for aggregation""" transaction_id = request.query_params.get("transaction_id", None) - contact_1_id = request.query_params.get("contact_1_id", None) - candidate_id = request.query_params.get("candidate_id", None) date = request.query_params.get("date", None) aggregation_group = request.query_params.get("aggregation_group", None) election_code = request.query_params.get("election_code", None) + candidate_office = request.query_params.get("candidate_office", None) + candidate_state = request.query_params.get("candidate_state", None) + candidate_district = request.query_params.get("candidate_district", None) missing_params = [] if not date: missing_params.append("date") if not aggregation_group: missing_params.append("aggregation_group") - if not contact_1_id and not candidate_id: - missing_params.append("and either contact_1_id or candidate_id") - if candidate_id and not election_code: + if not election_code: missing_params.append("election_code") + if not candidate_office: + missing_params.append("candidate_office") + if candidate_office != Contact.CandidateOffice.PRESIDENTIAL and not candidate_state: + missing_params.append("candidate_state") + if candidate_office == Contact.CandidateOffice.HOUSE and not candidate_district: + missing_params.append("candidate_district") + + response_str = "" + if len(missing_params) > 0: + response_str += "Please provide"+",".join(missing_params)+" in query params" + return Response( + response_str, + status=status.HTTP_400_BAD_REQUEST, + ) + + date = datetime.fromisoformat(date) + + previous_transactions = ( + self.get_queryset() + .filter( + ~Q(id=transaction_id or None), + Q(date__year=date.year), + Q(date__lte=date), + Q(aggregation_group=aggregation_group), + Q(schedule_e__election_code=election_code), + Q(schedule_e__so_candidate_office=candidate_office), + Q(schedule_e__so_candidate_state=candidate_state), + Q(schedule_e__so_candidate_district=candidate_district), + ) + ) + + previous_transaction = previous_transactions.order_by("-date", "-created").first() + + if previous_transaction: + serializer = self.get_serializer(previous_transaction) + return Response(data=serializer.data) + + response = {"message": "No previous transaction found."} + return Response(response, status=status.HTTP_404_NOT_FOUND) + + @action(detail=False, methods=["get"], url_path=r"previous/entity") + def previous_entity(self, request): + """Retrieves transaction that comes before this transactions, + while being in the same group for aggregation""" + transaction_id = request.query_params.get("transaction_id", None) + contact_1_id = request.query_params.get("contact_1_id", None) + date = request.query_params.get("date", None) + aggregation_group = request.query_params.get("aggregation_group", None) + + missing_params = [] + if not contact_1_id: + missing_params.append("contact_1_id") + if not date: + missing_params.append("date") + if not aggregation_group: + missing_params.append("aggregation_group") if len(missing_params) > 0: response_str = "Please provide"+",".join(missing_params)+" in query params" @@ -207,29 +263,16 @@ def previous(self, request): date = datetime.fromisoformat(date) - if candidate_id is None: - previous_transactions = ( - self.get_queryset() - .filter( - ~Q(id=transaction_id or None), - Q(contact_1_id=contact_1_id), - Q(date__year=date.year), - Q(date__lte=date), - Q(aggregation_group=aggregation_group), - ) - ) - else: - previous_transactions = ( - self.get_queryset() - .filter( - ~Q(id=transaction_id or None), - Q(contact_2_id=candidate_id) | Q(contact_3_id=candidate_id), - Q(date__year=date.year), - Q(date__lte=date), - Q(aggregation_group=aggregation_group), - Q(schedule_e__election_code=election_code) - ) + previous_transactions = ( + self.get_queryset() + .filter( + ~Q(id=transaction_id or None), + Q(contact_1_id=contact_1_id), + Q(date__year=date.year), + Q(date__lte=date), + Q(aggregation_group=aggregation_group), ) + ) previous_transaction = previous_transactions.order_by("-date", "-created").first() From 9cb7d41c8432f5c2fad1977c38e7502cd75faa9b Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Tue, 17 Oct 2023 13:05:53 -0400 Subject: [PATCH 07/70] Fixes linting errors --- django-backend/fecfiler/transactions/views.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/django-backend/fecfiler/transactions/views.py b/django-backend/fecfiler/transactions/views.py index b4f3285ff4..feff2f0085 100644 --- a/django-backend/fecfiler/transactions/views.py +++ b/django-backend/fecfiler/transactions/views.py @@ -199,16 +199,18 @@ def previous_election(self, request): missing_params.append("election_code") if not candidate_office: missing_params.append("candidate_office") - if candidate_office != Contact.CandidateOffice.PRESIDENTIAL and not candidate_state: + if ( + candidate_office != Contact.CandidateOffice.PRESIDENTIAL + and not candidate_state + ): missing_params.append("candidate_state") if candidate_office == Contact.CandidateOffice.HOUSE and not candidate_district: missing_params.append("candidate_district") - response_str = "" if len(missing_params) > 0: - response_str += "Please provide"+",".join(missing_params)+" in query params" + error_msg += "Please provide " + ",".join(missing_params) + " in query params" return Response( - response_str, + error_msg, status=status.HTTP_400_BAD_REQUEST, ) @@ -255,9 +257,9 @@ def previous_entity(self, request): missing_params.append("aggregation_group") if len(missing_params) > 0: - response_str = "Please provide"+",".join(missing_params)+" in query params" + error_msg = "Please provide " + ",".join(missing_params) + " in query params" return Response( - response_str, + error_msg, status=status.HTTP_400_BAD_REQUEST, ) From 47488feb22ad046ad318b9712e60f31b9dfa3fbe Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Tue, 17 Oct 2023 13:14:17 -0400 Subject: [PATCH 08/70] 1438 add transaction line_label annotation --- .../fecfiler/transactions/managers.py | 33 +++++++++++++++++++ .../fecfiler/transactions/schedule_a/views.py | 2 +- .../fecfiler/transactions/schedule_b/views.py | 2 +- .../fecfiler/transactions/serializers.py | 2 ++ django-backend/fecfiler/transactions/views.py | 2 +- 5 files changed, 38 insertions(+), 3 deletions(-) diff --git a/django-backend/fecfiler/transactions/managers.py b/django-backend/fecfiler/transactions/managers.py index 646f48cc52..0157bb9ad0 100644 --- a/django-backend/fecfiler/transactions/managers.py +++ b/django-backend/fecfiler/transactions/managers.py @@ -180,6 +180,39 @@ def get_queryset(self): - F("payment_amount"), ), ), + line_label=Case( + # Schedule A + When(_form_type="SA11A", then=Value("11(a)")), + When(_form_type="SA11AI", then=Value("11(a)(i)")), + When(_form_type="SA11AII", then=Value("11(a)(ii)")), + When(_form_type="SA11B", then=Value("11(b)")), + When(_form_type="SA11C", then=Value("11(c)")), + When(_form_type="SA12", then=Value("12")), + When(_form_type="SA13", then=Value("13")), + When(_form_type="SA14", then=Value("14")), + When(_form_type="SA15", then=Value("15")), + When(_form_type="SA16", then=Value("16")), + When(_form_type="SA17", then=Value("17")), + # Schedule B + When(_form_type="SB21B", then=Value("21(b)")), + When(_form_type="SB22", then=Value("22")), + When(_form_type="SB23", then=Value("23")), + When(_form_type="SB26", then=Value("26")), + When(_form_type="SB27", then=Value("27")), + When(_form_type="SB28A", then=Value("28(a)")), + When(_form_type="SB28B", then=Value("28(b)")), + When(_form_type="SB28C", then=Value("28(c)")), + When(_form_type="SB29", then=Value("29")), + When(_form_type="SB30B", then=Value("30(b)")), + # Schedule C + When(_form_type="SC/10", then=Value("10")), + When(_form_type="SC/9", then=Value("9")), + # Schedule D + When(_form_type="SD9", then=Value("9")), + When(_form_type="SD10", then=Value("10")), + # Schedule E + When(_form_type="SE", then=Value("24")), + ), form_type=Case( When(_form_type="SA11AI", itemized=False, then=Value("SA11AII")), When(_form_type="SA11AII", itemized=True, then=Value("SA11AI")), diff --git a/django-backend/fecfiler/transactions/schedule_a/views.py b/django-backend/fecfiler/transactions/schedule_a/views.py index 41bb359f7c..b1543e8c97 100644 --- a/django-backend/fecfiler/transactions/schedule_a/views.py +++ b/django-backend/fecfiler/transactions/schedule_a/views.py @@ -32,7 +32,7 @@ class ScheduleATransactionViewSet(TransactionViewSet): ) serializer_class = ScheduleATransactionSerializer ordering_fields = [ - "form_type", + "line_label", "transaction_type_identifier", "contributor_name", "contribution_date", diff --git a/django-backend/fecfiler/transactions/schedule_b/views.py b/django-backend/fecfiler/transactions/schedule_b/views.py index 1b396c9941..c5ee4ecdc7 100644 --- a/django-backend/fecfiler/transactions/schedule_b/views.py +++ b/django-backend/fecfiler/transactions/schedule_b/views.py @@ -32,7 +32,7 @@ class ScheduleBTransactionViewSet(TransactionViewSet): ) serializer_class = ScheduleBTransactionSerializer ordering_fields = [ - "form_type", + "line_label", "transaction_type_identifier", "payee_name", "expenditure_date", diff --git a/django-backend/fecfiler/transactions/serializers.py b/django-backend/fecfiler/transactions/serializers.py index a740e73bb9..fbe0044f20 100644 --- a/django-backend/fecfiler/transactions/serializers.py +++ b/django-backend/fecfiler/transactions/serializers.py @@ -137,6 +137,7 @@ class TransactionSerializerBase( balance_at_close = DecimalField( max_digits=11, decimal_places=2, read_only=True ) # debt payments + line_label = CharField(read_only=True) schedule_a = ScheduleASerializer(required=False) schedule_b = ScheduleBSerializer(required=False) @@ -317,6 +318,7 @@ def get_fields(): "beginning_balance", "payment_amount", "balance_at_close", + "line_label", "schedule_a", "schedule_b", "schedule_c", diff --git a/django-backend/fecfiler/transactions/views.py b/django-backend/fecfiler/transactions/views.py index be5f74f2a2..61d7dff93c 100644 --- a/django-backend/fecfiler/transactions/views.py +++ b/django-backend/fecfiler/transactions/views.py @@ -141,7 +141,7 @@ class TransactionViewSet(CommitteeOwnedViewSet, ReportViewMixin): filter_backends = [filters.OrderingFilter] ordering_fields = [ - "form_type", + "line_label", "transaction_type_identifier", "memo_code", "name", From 8845d57fe68a951a6b248890c6ea1078345aecf2 Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Tue, 17 Oct 2023 13:48:36 -0400 Subject: [PATCH 09/70] 1483 add transaction line_label test --- django-backend/fecfiler/transactions/tests/test_manager.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/django-backend/fecfiler/transactions/tests/test_manager.py b/django-backend/fecfiler/transactions/tests/test_manager.py index 51e9f533eb..d29c504275 100644 --- a/django-backend/fecfiler/transactions/tests/test_manager.py +++ b/django-backend/fecfiler/transactions/tests/test_manager.py @@ -49,3 +49,7 @@ def test_debt_repayment(self): def test_force_unaggregated(self): txn = Transaction.objects.get(id="12345678-1596-4ef1-a1aa-c4386b8d1234") self.assertEqual(txn.aggregate, 3333) + + def test_line_label(self): + refund = Transaction.objects.get(id="bbbbbbbb-3274-47d8-9388-7294a3fd4321") + self.assertEqual(refund.line_label, "21(b)") From 168f688d7ead9d96fc55faf739969c890c8aa647 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Tue, 17 Oct 2023 14:43:39 -0400 Subject: [PATCH 10/70] Fix lint issues reported by SonarCloud --- django-backend/fecfiler/transactions/views.py | 60 +++++++++---------- 1 file changed, 28 insertions(+), 32 deletions(-) diff --git a/django-backend/fecfiler/transactions/views.py b/django-backend/fecfiler/transactions/views.py index a38b335782..ebd72ef9b4 100644 --- a/django-backend/fecfiler/transactions/views.py +++ b/django-backend/fecfiler/transactions/views.py @@ -200,7 +200,7 @@ def previous_election(self, request): if not candidate_office: missing_params.append("candidate_office") if ( - candidate_office != Contact.CandidateOffice.PRESIDENTIAL + candidate_office != Contact.CandidateOffice.PRESIDENTIAL and not candidate_state ): missing_params.append("candidate_state") @@ -208,29 +208,27 @@ def previous_election(self, request): missing_params.append("candidate_district") if len(missing_params) > 0: - error_msg += "Please provide " + ",".join(missing_params) + " in query params" - return Response( - error_msg, - status=status.HTTP_400_BAD_REQUEST, + error_msg = ( + "Please provide " + ",".join(missing_params) + " in query params" ) + return Response(error_msg, status=status.HTTP_400_BAD_REQUEST,) date = datetime.fromisoformat(date) - previous_transactions = ( - self.get_queryset() - .filter( - ~Q(id=transaction_id or None), - Q(date__year=date.year), - Q(date__lte=date), - Q(aggregation_group=aggregation_group), - Q(schedule_e__election_code=election_code), - Q(schedule_e__so_candidate_office=candidate_office), - Q(schedule_e__so_candidate_state=candidate_state), - Q(schedule_e__so_candidate_district=candidate_district), - ) + previous_transactions = self.get_queryset().filter( + ~Q(id=transaction_id or None), + Q(date__year=date.year), + Q(date__lte=date), + Q(aggregation_group=aggregation_group), + Q(schedule_e__election_code=election_code), + Q(schedule_e__so_candidate_office=candidate_office), + Q(schedule_e__so_candidate_state=candidate_state), + Q(schedule_e__so_candidate_district=candidate_district), ) - previous_transaction = previous_transactions.order_by("-date", "-created").first() + previous_transaction = previous_transactions.order_by( + "-date", "-created" + ).first() if previous_transaction: serializer = self.get_serializer(previous_transaction) @@ -257,26 +255,24 @@ def previous_entity(self, request): missing_params.append("aggregation_group") if len(missing_params) > 0: - error_msg = "Please provide " + ",".join(missing_params) + " in query params" - return Response( - error_msg, - status=status.HTTP_400_BAD_REQUEST, + error_msg = ( + "Please provide " + ",".join(missing_params) + " in query params" ) + return Response(error_msg, status=status.HTTP_400_BAD_REQUEST,) date = datetime.fromisoformat(date) - previous_transactions = ( - self.get_queryset() - .filter( - ~Q(id=transaction_id or None), - Q(contact_1_id=contact_1_id), - Q(date__year=date.year), - Q(date__lte=date), - Q(aggregation_group=aggregation_group), - ) + previous_transactions = self.get_queryset().filter( + ~Q(id=transaction_id or None), + Q(contact_1_id=contact_1_id), + Q(date__year=date.year), + Q(date__lte=date), + Q(aggregation_group=aggregation_group), ) - previous_transaction = previous_transactions.order_by("-date", "-created").first() + previous_transaction = previous_transactions.order_by( + "-date", "-created" + ).first() if previous_transaction: serializer = self.get_serializer(previous_transaction) From 87c964bc0a792b988207c2f1a5a297260ab61fb6 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Tue, 17 Oct 2023 14:55:18 -0400 Subject: [PATCH 11/70] Corrects a syntax error that was also breaking flake8 --- django-backend/fecfiler/transactions/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django-backend/fecfiler/transactions/views.py b/django-backend/fecfiler/transactions/views.py index a38b335782..5c714e0d87 100644 --- a/django-backend/fecfiler/transactions/views.py +++ b/django-backend/fecfiler/transactions/views.py @@ -208,7 +208,7 @@ def previous_election(self, request): missing_params.append("candidate_district") if len(missing_params) > 0: - error_msg += "Please provide " + ",".join(missing_params) + " in query params" + error_msg = "Please provide " + ",".join(missing_params) + " in query params" return Response( error_msg, status=status.HTTP_400_BAD_REQUEST, From d286291d8ed275dc9bed248cb0793301b5d40cbd Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Tue, 17 Oct 2023 16:31:56 -0400 Subject: [PATCH 12/70] Fixes linting errors --- .../fecfiler/transactions/managers.py | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/django-backend/fecfiler/transactions/managers.py b/django-backend/fecfiler/transactions/managers.py index f87b6a1998..7253567818 100644 --- a/django-backend/fecfiler/transactions/managers.py +++ b/django-backend/fecfiler/transactions/managers.py @@ -69,25 +69,35 @@ def get_queryset(self): ) ) - contact_1_clause = Q(contact_1_id=OuterRef("contact_1_id")) + primary_contact_clause = Q(contact_1_id=OuterRef("contact_1_id")) election_clause = ( Q( schedule_e__isnull=False ) & Q( - schedule_e__election_code=OuterRef("schedule_e__election_code") + schedule_e__election_code=OuterRef( + "schedule_e__election_code" + ) ) & Q( - schedule_e__so_candidate_office=OuterRef("schedule_e__so_candidate_office") + schedule_e__so_candidate_office=OuterRef( + "schedule_e__so_candidate_office" + ) ) & Q( Q( - schedule_e__so_candidate_state=OuterRef("schedule_e__so_candidate_state") + schedule_e__so_candidate_state=OuterRef( + "schedule_e__so_candidate_state" + ) ) | ( - Q(schedule_e__so_candidate_office__isnull=True) & Q(outer_candidate_state__isnull=True) + Q(schedule_e__so_candidate_office__isnull=True) & + Q(outer_candidate_state__isnull=True) ) ) & ( Q( - schedule_e__so_candidate_district=OuterRef("schedule_e__so_candidate_district") + schedule_e__so_candidate_district=OuterRef( + "schedule_e__so_candidate_district" + ) ) | ( - Q(schedule_e__so_candidate_district__isnull=True) & Q(outer_candidate_district__isnull=True) + Q(schedule_e__so_candidate_district__isnull=True) & + Q(outer_candidate_district__isnull=True) ) ) ) @@ -100,7 +110,7 @@ def get_queryset(self): aggregate_clause = ( queryset.filter( - contact_1_clause, + primary_contact_clause, year_clause, date_clause, group_clause, @@ -111,7 +121,7 @@ def get_queryset(self): .values("aggregate") ) calendar_ytd_clause = ( - queryset.alias( # Needed to get around null-matching bug with Q + queryset.alias( # Needed to get around null-matching bug with Q() outer_candidate_district=ExpressionWrapper( OuterRef('schedule_e__so_candidate_district'), output_field=TextField() From d5e23a3964a39aa4c6623db60256d24e6ea1db1e Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Tue, 17 Oct 2023 17:11:37 -0400 Subject: [PATCH 13/70] 1438 add line_label sort option to other schedules --- django-backend/fecfiler/transactions/schedule_c/views.py | 1 + django-backend/fecfiler/transactions/schedule_c1/views.py | 1 + django-backend/fecfiler/transactions/schedule_c2/views.py | 1 + django-backend/fecfiler/transactions/schedule_d/views.py | 1 + django-backend/fecfiler/transactions/schedule_e/views.py | 1 + 5 files changed, 5 insertions(+) diff --git a/django-backend/fecfiler/transactions/schedule_c/views.py b/django-backend/fecfiler/transactions/schedule_c/views.py index 60e6a6c804..73e42e8f9b 100644 --- a/django-backend/fecfiler/transactions/schedule_c/views.py +++ b/django-backend/fecfiler/transactions/schedule_c/views.py @@ -30,6 +30,7 @@ class ScheduleCTransactionViewSet(TransactionViewSet): serializer_class = ScheduleCTransactionSerializer ordering_fields = [ "id", + "line_label", "transaction_type_identifier", "lender_name", "memo_code", diff --git a/django-backend/fecfiler/transactions/schedule_c1/views.py b/django-backend/fecfiler/transactions/schedule_c1/views.py index 4b2781b56a..250da16f22 100644 --- a/django-backend/fecfiler/transactions/schedule_c1/views.py +++ b/django-backend/fecfiler/transactions/schedule_c1/views.py @@ -16,6 +16,7 @@ class ScheduleC1TransactionViewSet(TransactionViewSet): serializer_class = ScheduleC1TransactionSerializer ordering_fields = [ "id", + "line_label", "transaction_type_identifier", "lender_organization_name", ] diff --git a/django-backend/fecfiler/transactions/schedule_c2/views.py b/django-backend/fecfiler/transactions/schedule_c2/views.py index 3fda771a8f..efb056ff59 100644 --- a/django-backend/fecfiler/transactions/schedule_c2/views.py +++ b/django-backend/fecfiler/transactions/schedule_c2/views.py @@ -16,6 +16,7 @@ class ScheduleC2TransactionViewSet(TransactionViewSet): serializer_class = ScheduleC2TransactionSerializer ordering_fields = [ "id", + "line_label", "transaction_type_identifier", "guarantor_last_name", "guarantor_first_name", diff --git a/django-backend/fecfiler/transactions/schedule_d/views.py b/django-backend/fecfiler/transactions/schedule_d/views.py index b955beb1ab..e5f6516aea 100644 --- a/django-backend/fecfiler/transactions/schedule_d/views.py +++ b/django-backend/fecfiler/transactions/schedule_d/views.py @@ -30,6 +30,7 @@ class ScheduleDTransactionViewSet(TransactionViewSet): serializer_class = ScheduleDTransactionSerializer ordering_fields = [ "id", + "line_label", "transaction_type_identifier", "creditor_name", ] diff --git a/django-backend/fecfiler/transactions/schedule_e/views.py b/django-backend/fecfiler/transactions/schedule_e/views.py index 05fea56d86..a0819ef6a7 100644 --- a/django-backend/fecfiler/transactions/schedule_e/views.py +++ b/django-backend/fecfiler/transactions/schedule_e/views.py @@ -30,6 +30,7 @@ class ScheduleETransactionViewSet(TransactionViewSet): serializer_class = ScheduleETransactionSerializer ordering_fields = [ "id", + "line_label", "transaction_type_identifier", "payee_name", ] From cf9d399391c2e2d51bc0f35cb154d05beedeea52 Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Tue, 17 Oct 2023 17:31:23 -0400 Subject: [PATCH 14/70] 1438 fixed itemize/unitemize issue --- .../fecfiler/transactions/managers.py | 66 +++++++++---------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/django-backend/fecfiler/transactions/managers.py b/django-backend/fecfiler/transactions/managers.py index 0157bb9ad0..6b667bbd6f 100644 --- a/django-backend/fecfiler/transactions/managers.py +++ b/django-backend/fecfiler/transactions/managers.py @@ -180,39 +180,6 @@ def get_queryset(self): - F("payment_amount"), ), ), - line_label=Case( - # Schedule A - When(_form_type="SA11A", then=Value("11(a)")), - When(_form_type="SA11AI", then=Value("11(a)(i)")), - When(_form_type="SA11AII", then=Value("11(a)(ii)")), - When(_form_type="SA11B", then=Value("11(b)")), - When(_form_type="SA11C", then=Value("11(c)")), - When(_form_type="SA12", then=Value("12")), - When(_form_type="SA13", then=Value("13")), - When(_form_type="SA14", then=Value("14")), - When(_form_type="SA15", then=Value("15")), - When(_form_type="SA16", then=Value("16")), - When(_form_type="SA17", then=Value("17")), - # Schedule B - When(_form_type="SB21B", then=Value("21(b)")), - When(_form_type="SB22", then=Value("22")), - When(_form_type="SB23", then=Value("23")), - When(_form_type="SB26", then=Value("26")), - When(_form_type="SB27", then=Value("27")), - When(_form_type="SB28A", then=Value("28(a)")), - When(_form_type="SB28B", then=Value("28(b)")), - When(_form_type="SB28C", then=Value("28(c)")), - When(_form_type="SB29", then=Value("29")), - When(_form_type="SB30B", then=Value("30(b)")), - # Schedule C - When(_form_type="SC/10", then=Value("10")), - When(_form_type="SC/9", then=Value("9")), - # Schedule D - When(_form_type="SD9", then=Value("9")), - When(_form_type="SD10", then=Value("10")), - # Schedule E - When(_form_type="SE", then=Value("24")), - ), form_type=Case( When(_form_type="SA11AI", itemized=False, then=Value("SA11AII")), When(_form_type="SA11AII", itemized=True, then=Value("SA11AI")), @@ -228,6 +195,39 @@ def get_queryset(self): default=F("_form_type"), output_field=TextField(), ), + line_label=Case( + # Schedule A + When(form_type="SA11A", then=Value("11(a)")), + When(form_type="SA11AI", then=Value("11(a)(i)")), + When(form_type="SA11AII", then=Value("11(a)(ii)")), + When(form_type="SA11B", then=Value("11(b)")), + When(form_type="SA11C", then=Value("11(c)")), + When(form_type="SA12", then=Value("12")), + When(form_type="SA13", then=Value("13")), + When(form_type="SA14", then=Value("14")), + When(form_type="SA15", then=Value("15")), + When(form_type="SA16", then=Value("16")), + When(form_type="SA17", then=Value("17")), + # Schedule B + When(form_type="SB21B", then=Value("21(b)")), + When(form_type="SB22", then=Value("22")), + When(form_type="SB23", then=Value("23")), + When(form_type="SB26", then=Value("26")), + When(form_type="SB27", then=Value("27")), + When(form_type="SB28A", then=Value("28(a)")), + When(form_type="SB28B", then=Value("28(b)")), + When(form_type="SB28C", then=Value("28(c)")), + When(form_type="SB29", then=Value("29")), + When(form_type="SB30B", then=Value("30(b)")), + # Schedule C + When(form_type="SC/10", then=Value("10")), + When(form_type="SC/9", then=Value("9")), + # Schedule D + When(form_type="SD9", then=Value("9")), + When(form_type="SD10", then=Value("10")), + # Schedule E + When(form_type="SE", then=Value("24")), + ), back_reference_tran_id_number=Coalesce( F("parent_transaction__transaction_id"), F("debt__transaction_id"), From 8c4ec97d4f687818881d90924aa40205020178a9 Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Wed, 18 Oct 2023 12:15:59 -0400 Subject: [PATCH 15/70] 1438 sort time optimization --- .../fecfiler/transactions/managers.py | 68 ++++++++++--------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/django-backend/fecfiler/transactions/managers.py b/django-backend/fecfiler/transactions/managers.py index 6b667bbd6f..2ad4b95657 100644 --- a/django-backend/fecfiler/transactions/managers.py +++ b/django-backend/fecfiler/transactions/managers.py @@ -195,39 +195,6 @@ def get_queryset(self): default=F("_form_type"), output_field=TextField(), ), - line_label=Case( - # Schedule A - When(form_type="SA11A", then=Value("11(a)")), - When(form_type="SA11AI", then=Value("11(a)(i)")), - When(form_type="SA11AII", then=Value("11(a)(ii)")), - When(form_type="SA11B", then=Value("11(b)")), - When(form_type="SA11C", then=Value("11(c)")), - When(form_type="SA12", then=Value("12")), - When(form_type="SA13", then=Value("13")), - When(form_type="SA14", then=Value("14")), - When(form_type="SA15", then=Value("15")), - When(form_type="SA16", then=Value("16")), - When(form_type="SA17", then=Value("17")), - # Schedule B - When(form_type="SB21B", then=Value("21(b)")), - When(form_type="SB22", then=Value("22")), - When(form_type="SB23", then=Value("23")), - When(form_type="SB26", then=Value("26")), - When(form_type="SB27", then=Value("27")), - When(form_type="SB28A", then=Value("28(a)")), - When(form_type="SB28B", then=Value("28(b)")), - When(form_type="SB28C", then=Value("28(c)")), - When(form_type="SB29", then=Value("29")), - When(form_type="SB30B", then=Value("30(b)")), - # Schedule C - When(form_type="SC/10", then=Value("10")), - When(form_type="SC/9", then=Value("9")), - # Schedule D - When(form_type="SD9", then=Value("9")), - When(form_type="SD10", then=Value("10")), - # Schedule E - When(form_type="SE", then=Value("24")), - ), back_reference_tran_id_number=Coalesce( F("parent_transaction__transaction_id"), F("debt__transaction_id"), @@ -240,6 +207,41 @@ def get_queryset(self): F("loan___form_type"), Value(None), ), + line_label=Case( + # Schedule A + When(_form_type="SA11A", then=Value("11(a)")), + When(_form_type="SA11AI", itemized=False, then=Value("11(a)(ii)")), + When(_form_type="SA11AI", then=Value("11(a)(i)")), + When(_form_type="SA11AII", itemized=True, then=Value("11(a)(i)")), + When(_form_type="SA11AII", then=Value("11(a)(ii)")), + When(_form_type="SA11B", then=Value("11(b)")), + When(_form_type="SA11C", then=Value("11(c)")), + When(_form_type="SA12", then=Value("12")), + When(_form_type="SA13", then=Value("13")), + When(_form_type="SA14", then=Value("14")), + When(_form_type="SA15", then=Value("15")), + When(_form_type="SA16", then=Value("16")), + When(_form_type="SA17", then=Value("17")), + # Schedule B + When(_form_type="SB21B", then=Value("21(b)")), + When(_form_type="SB22", then=Value("22")), + When(_form_type="SB23", then=Value("23")), + When(_form_type="SB26", then=Value("26")), + When(_form_type="SB27", then=Value("27")), + When(_form_type="SB28A", then=Value("28(a)")), + When(_form_type="SB28B", then=Value("28(b)")), + When(_form_type="SB28C", then=Value("28(c)")), + When(_form_type="SB29", then=Value("29")), + When(_form_type="SB30B", then=Value("30(b)")), + # Schedule C + When(_form_type="SC/10", then=Value("10")), + When(_form_type="SC/9", then=Value("9")), + # Schedule D + When(_form_type="SD9", then=Value("9")), + When(_form_type="SD10", then=Value("10")), + # Schedule E + When(_form_type="SE", then=Value("24")), + ), ) .annotate( balance=Coalesce( From 0cbc4be77f4a2ad1806bc11702bde7e2e995550b Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Wed, 18 Oct 2023 13:32:22 -0400 Subject: [PATCH 16/70] 1438 remove itemized check for line_label --- django-backend/fecfiler/transactions/managers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/django-backend/fecfiler/transactions/managers.py b/django-backend/fecfiler/transactions/managers.py index 2ad4b95657..86a95ae878 100644 --- a/django-backend/fecfiler/transactions/managers.py +++ b/django-backend/fecfiler/transactions/managers.py @@ -210,9 +210,7 @@ def get_queryset(self): line_label=Case( # Schedule A When(_form_type="SA11A", then=Value("11(a)")), - When(_form_type="SA11AI", itemized=False, then=Value("11(a)(ii)")), When(_form_type="SA11AI", then=Value("11(a)(i)")), - When(_form_type="SA11AII", itemized=True, then=Value("11(a)(i)")), When(_form_type="SA11AII", then=Value("11(a)(ii)")), When(_form_type="SA11B", then=Value("11(b)")), When(_form_type="SA11C", then=Value("11(c)")), From bbb7dc7f410f06681d1739f7aed5f43299f9d79e Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Wed, 18 Oct 2023 16:25:40 -0400 Subject: [PATCH 17/70] 1438 interleaving sort fix --- django-backend/fecfiler/transactions/managers.py | 3 ++- django-backend/fecfiler/transactions/schedule_a/views.py | 2 +- django-backend/fecfiler/transactions/schedule_b/views.py | 2 +- django-backend/fecfiler/transactions/schedule_c/views.py | 2 +- django-backend/fecfiler/transactions/schedule_c1/views.py | 2 +- django-backend/fecfiler/transactions/schedule_c2/views.py | 2 +- django-backend/fecfiler/transactions/schedule_d/views.py | 2 +- django-backend/fecfiler/transactions/schedule_e/views.py | 2 +- django-backend/fecfiler/transactions/views.py | 2 +- 9 files changed, 10 insertions(+), 9 deletions(-) diff --git a/django-backend/fecfiler/transactions/managers.py b/django-backend/fecfiler/transactions/managers.py index 86a95ae878..d847570c46 100644 --- a/django-backend/fecfiler/transactions/managers.py +++ b/django-backend/fecfiler/transactions/managers.py @@ -211,7 +211,7 @@ def get_queryset(self): # Schedule A When(_form_type="SA11A", then=Value("11(a)")), When(_form_type="SA11AI", then=Value("11(a)(i)")), - When(_form_type="SA11AII", then=Value("11(a)(ii)")), + When(_form_type="SA11AII", then=Value("11(a)(i)")), When(_form_type="SA11B", then=Value("11(b)")), When(_form_type="SA11C", then=Value("11(c)")), When(_form_type="SA12", then=Value("12")), @@ -240,6 +240,7 @@ def get_queryset(self): # Schedule E When(_form_type="SE", then=Value("24")), ), + line_label_order_key=Concat(F("line_label"), F("form_type"), output_field=TextField()), ) .annotate( balance=Coalesce( diff --git a/django-backend/fecfiler/transactions/schedule_a/views.py b/django-backend/fecfiler/transactions/schedule_a/views.py index b1543e8c97..23f07252b9 100644 --- a/django-backend/fecfiler/transactions/schedule_a/views.py +++ b/django-backend/fecfiler/transactions/schedule_a/views.py @@ -32,7 +32,7 @@ class ScheduleATransactionViewSet(TransactionViewSet): ) serializer_class = ScheduleATransactionSerializer ordering_fields = [ - "line_label", + "line_label_order_key", "transaction_type_identifier", "contributor_name", "contribution_date", diff --git a/django-backend/fecfiler/transactions/schedule_b/views.py b/django-backend/fecfiler/transactions/schedule_b/views.py index c5ee4ecdc7..65471698c4 100644 --- a/django-backend/fecfiler/transactions/schedule_b/views.py +++ b/django-backend/fecfiler/transactions/schedule_b/views.py @@ -32,7 +32,7 @@ class ScheduleBTransactionViewSet(TransactionViewSet): ) serializer_class = ScheduleBTransactionSerializer ordering_fields = [ - "line_label", + "line_label_order_key", "transaction_type_identifier", "payee_name", "expenditure_date", diff --git a/django-backend/fecfiler/transactions/schedule_c/views.py b/django-backend/fecfiler/transactions/schedule_c/views.py index 73e42e8f9b..d088b1fddc 100644 --- a/django-backend/fecfiler/transactions/schedule_c/views.py +++ b/django-backend/fecfiler/transactions/schedule_c/views.py @@ -30,7 +30,7 @@ class ScheduleCTransactionViewSet(TransactionViewSet): serializer_class = ScheduleCTransactionSerializer ordering_fields = [ "id", - "line_label", + "line_label_order_key", "transaction_type_identifier", "lender_name", "memo_code", diff --git a/django-backend/fecfiler/transactions/schedule_c1/views.py b/django-backend/fecfiler/transactions/schedule_c1/views.py index 250da16f22..631d8810fe 100644 --- a/django-backend/fecfiler/transactions/schedule_c1/views.py +++ b/django-backend/fecfiler/transactions/schedule_c1/views.py @@ -16,7 +16,7 @@ class ScheduleC1TransactionViewSet(TransactionViewSet): serializer_class = ScheduleC1TransactionSerializer ordering_fields = [ "id", - "line_label", + "line_label_order_key", "transaction_type_identifier", "lender_organization_name", ] diff --git a/django-backend/fecfiler/transactions/schedule_c2/views.py b/django-backend/fecfiler/transactions/schedule_c2/views.py index efb056ff59..9bbde350d3 100644 --- a/django-backend/fecfiler/transactions/schedule_c2/views.py +++ b/django-backend/fecfiler/transactions/schedule_c2/views.py @@ -16,7 +16,7 @@ class ScheduleC2TransactionViewSet(TransactionViewSet): serializer_class = ScheduleC2TransactionSerializer ordering_fields = [ "id", - "line_label", + "line_label_order_key", "transaction_type_identifier", "guarantor_last_name", "guarantor_first_name", diff --git a/django-backend/fecfiler/transactions/schedule_d/views.py b/django-backend/fecfiler/transactions/schedule_d/views.py index e5f6516aea..5c002715f4 100644 --- a/django-backend/fecfiler/transactions/schedule_d/views.py +++ b/django-backend/fecfiler/transactions/schedule_d/views.py @@ -30,7 +30,7 @@ class ScheduleDTransactionViewSet(TransactionViewSet): serializer_class = ScheduleDTransactionSerializer ordering_fields = [ "id", - "line_label", + "line_label_order_key", "transaction_type_identifier", "creditor_name", ] diff --git a/django-backend/fecfiler/transactions/schedule_e/views.py b/django-backend/fecfiler/transactions/schedule_e/views.py index a0819ef6a7..5c4bc60254 100644 --- a/django-backend/fecfiler/transactions/schedule_e/views.py +++ b/django-backend/fecfiler/transactions/schedule_e/views.py @@ -30,7 +30,7 @@ class ScheduleETransactionViewSet(TransactionViewSet): serializer_class = ScheduleETransactionSerializer ordering_fields = [ "id", - "line_label", + "line_label_order_key", "transaction_type_identifier", "payee_name", ] diff --git a/django-backend/fecfiler/transactions/views.py b/django-backend/fecfiler/transactions/views.py index 61d7dff93c..e393859b6d 100644 --- a/django-backend/fecfiler/transactions/views.py +++ b/django-backend/fecfiler/transactions/views.py @@ -141,7 +141,7 @@ class TransactionViewSet(CommitteeOwnedViewSet, ReportViewMixin): filter_backends = [filters.OrderingFilter] ordering_fields = [ - "line_label", + "line_label_order_key", "transaction_type_identifier", "memo_code", "name", From aeb47c8d6a746a9be0ffc32f1ee5d1f9e1048dd2 Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Wed, 18 Oct 2023 16:30:48 -0400 Subject: [PATCH 18/70] 1438 fix --- django-backend/fecfiler/transactions/managers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django-backend/fecfiler/transactions/managers.py b/django-backend/fecfiler/transactions/managers.py index d847570c46..df403e6481 100644 --- a/django-backend/fecfiler/transactions/managers.py +++ b/django-backend/fecfiler/transactions/managers.py @@ -240,7 +240,7 @@ def get_queryset(self): # Schedule E When(_form_type="SE", then=Value("24")), ), - line_label_order_key=Concat(F("line_label"), F("form_type"), output_field=TextField()), + line_label_order_key=Concat("line_label", "form_type", output_field=TextField()), ) .annotate( balance=Coalesce( From 8eb9ef33fa89b3fa14d77009693e88056f56800c Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Wed, 18 Oct 2023 16:35:30 -0400 Subject: [PATCH 19/70] lint fix --- django-backend/fecfiler/transactions/managers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/django-backend/fecfiler/transactions/managers.py b/django-backend/fecfiler/transactions/managers.py index df403e6481..7b7a445c6d 100644 --- a/django-backend/fecfiler/transactions/managers.py +++ b/django-backend/fecfiler/transactions/managers.py @@ -240,7 +240,9 @@ def get_queryset(self): # Schedule E When(_form_type="SE", then=Value("24")), ), - line_label_order_key=Concat("line_label", "form_type", output_field=TextField()), + line_label_order_key=Concat( + "line_label", "form_type", output_field=TextField() + ), ) .annotate( balance=Coalesce( From 1277e4ed56c9b565bd08fd6a1a47ded4d9bc06da Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Wed, 18 Oct 2023 18:31:15 -0400 Subject: [PATCH 20/70] Fixes linting errors --- django-backend/fecfiler/transactions/managers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/django-backend/fecfiler/transactions/managers.py b/django-backend/fecfiler/transactions/managers.py index 7253567818..6a9394af4e 100644 --- a/django-backend/fecfiler/transactions/managers.py +++ b/django-backend/fecfiler/transactions/managers.py @@ -87,8 +87,8 @@ def get_queryset(self): "schedule_e__so_candidate_state" ) ) | ( - Q(schedule_e__so_candidate_office__isnull=True) & - Q(outer_candidate_state__isnull=True) + Q(schedule_e__so_candidate_office__isnull=True) + & Q(outer_candidate_state__isnull=True) ) ) & ( Q( @@ -96,8 +96,8 @@ def get_queryset(self): "schedule_e__so_candidate_district" ) ) | ( - Q(schedule_e__so_candidate_district__isnull=True) & - Q(outer_candidate_district__isnull=True) + Q(schedule_e__so_candidate_district__isnull=True) + & Q(outer_candidate_district__isnull=True) ) ) ) From 656366f421415fc26ca69c5b1f1e03ebe2606edb Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Thu, 19 Oct 2023 10:10:20 -0400 Subject: [PATCH 21/70] WIP adding unit tests --- .../test_election_aggregation_data.json | 430 ++++++++++++++++++ 1 file changed, 430 insertions(+) create mode 100644 django-backend/fecfiler/transactions/fixtures/test_election_aggregation_data.json diff --git a/django-backend/fecfiler/transactions/fixtures/test_election_aggregation_data.json b/django-backend/fecfiler/transactions/fixtures/test_election_aggregation_data.json new file mode 100644 index 0000000000..1eb16eb9df --- /dev/null +++ b/django-backend/fecfiler/transactions/fixtures/test_election_aggregation_data.json @@ -0,0 +1,430 @@ +[ +{ + "model": "contacts.contact", + "pk": "112301a6-2461-4cac-9048-d9568bd74786", + "fields": { + "deleted": null, + "committee_account": "355fda97-57f8-424d-94fd-a3161a9161c7", + "type": "IND", + "candidate_id": null, + "committee_id": null, + "name": null, + "last_name": "Testerson", + "first_name": "George", + "middle_name": null, + "prefix": null, + "suffix": null, + "street_1": "1234 Test Dr", + "street_2": null, + "city": "Testville", + "state": "AK", + "zip": "54321", + "employer": "Employer", + "occupation": "Occupation", + "candidate_office": null, + "candidate_state": null, + "candidate_district": null, + "telephone": null, + "country": "USA", + "created": "2023-10-17T18:35:07.353Z", + "updated": "2023-10-17T18:35:07.354Z" + } +}, +{ + "model": "contacts.contact", + "pk": "6ab0f6d8-c9b1-4068-88a1-dbabda1dee73", + "fields": { + "deleted": null, + "committee_account": "355fda97-57f8-424d-94fd-a3161a9161c7", + "type": "CAN", + "candidate_id": "S6AK00162", + "committee_id": null, + "name": null, + "last_name": "Testerson", + "first_name": "Phil", + "middle_name": null, + "prefix": null, + "suffix": null, + "street_1": "1212 TEST LN", + "street_2": null, + "city": "TESTOPOLIS", + "state": "AK", + "zip": "59624", + "employer": null, + "occupation": null, + "candidate_office": "S", + "candidate_state": "AK", + "candidate_district": null, + "telephone": null, + "country": "USA", + "created": "2023-10-17T18:35:15.492Z", + "updated": "2023-10-17T18:35:15.492Z" + } +}, +{ + "model": "contacts.contact", + "pk": "72fd8435-3d5d-429f-9f23-56ea04fc8b2f", + "fields": { + "deleted": null, + "committee_account": "355fda97-57f8-424d-94fd-a3161a9161c7", + "type": "ORG", + "candidate_id": null, + "committee_id": null, + "name": "The Most Organized Organization", + "last_name": null, + "first_name": null, + "middle_name": null, + "prefix": null, + "suffix": null, + "street_1": "1234 Testing Ln", + "street_2": null, + "city": "Testopolis", + "state": "AK", + "zip": "12345", + "employer": null, + "occupation": null, + "candidate_office": null, + "candidate_state": null, + "candidate_district": null, + "telephone": null, + "country": "USA", + "created": "2023-10-17T18:36:26.899Z", + "updated": "2023-10-17T18:36:26.899Z" + } +}, +{ + "model": "contacts.contact", + "pk": "e43e5af9-9de2-4d60-8107-2c37e981720d", + "fields": { + "deleted": null, + "committee_account": "355fda97-57f8-424d-94fd-a3161a9161c7", + "type": "COM", + "candidate_id": null, + "committee_id": "C01244210", + "name": "ALASKAN TEST COMMITTEE", + "last_name": null, + "first_name": null, + "middle_name": null, + "prefix": null, + "suffix": null, + "street_1": "1212 TEST LN", + "street_2": null, + "city": "TESTOPOLIS", + "state": "AK", + "zip": "12345", + "employer": null, + "occupation": null, + "candidate_office": null, + "candidate_state": null, + "candidate_district": null, + "telephone": "", + "country": "USA", + "created": "2023-10-17T18:35:22.641Z", + "updated": "2023-10-17T18:35:22.641Z" + } +}, +{ + "model": "transactions.schedulea", + "pk": "5c039ffa-cf15-4c03-81e2-c6dcf7dcf0e1", + "fields": { + "contributor_organization_name": null, + "contributor_last_name": "Testerson", + "contributor_first_name": "George", + "contributor_middle_name": null, + "contributor_prefix": null, + "contributor_suffix": null, + "contributor_street_1": "1234 Test Dr", + "contributor_street_2": null, + "contributor_city": "Testville", + "contributor_state": "AK", + "contributor_zip": "54321", + "contribution_date": "2023-10-01", + "contribution_amount": "255.00", + "contribution_purpose_descrip": null, + "contributor_employer": "Employer", + "contributor_occupation": "Occupation", + "donor_committee_fec_id": null, + "donor_committee_name": null, + "donor_candidate_fec_id": null, + "donor_candidate_last_name": null, + "donor_candidate_first_name": null, + "donor_candidate_middle_name": null, + "donor_candidate_prefix": null, + "donor_candidate_suffix": null, + "donor_candidate_office": null, + "donor_candidate_state": null, + "donor_candidate_district": null, + "election_code": null, + "election_other_description": null, + "conduit_name": null, + "conduit_street_1": null, + "conduit_street_2": null, + "conduit_city": null, + "conduit_state": null, + "conduit_zip": null, + "memo_text_description": null, + "reference_to_si_or_sl_system_code_that_identifies_the_account": null + } +}, +{ + "model": "transactions.schedulee", + "pk": "21f75558-9ac7-4aaf-9471-be2d7f8d2902", + "fields": { + "payee_organization_name": null, + "payee_last_name": "Testerson", + "payee_first_name": "George", + "payee_middle_name": null, + "payee_prefix": null, + "payee_suffix": null, + "payee_street_1": "1234 Test Dr", + "payee_street_2": null, + "payee_city": "Testville", + "payee_state": "AK", + "payee_zip": "54321", + "election_code": "C2012", + "election_other_description": null, + "dissemination_date": null, + "expenditure_amount": "42.00", + "disbursement_date": "2023-10-19", + "calendar_ytd": "42.00", + "expenditure_purpose_descrip": "Reimbursement: See Below", + "category_code": null, + "payee_cmtte_fec_id_number": null, + "support_oppose_code": "S", + "so_candidate_id_number": "S6AK00162", + "so_candidate_last_name": "Testerson", + "so_candidate_first_name": "Phil", + "so_candinate_middle_name": null, + "so_candidate_prefix": null, + "so_candidate_suffix": null, + "so_candidate_office": "S", + "so_candidate_district": null, + "so_candidate_state": "AK", + "completing_last_name": "T", + "completing_first_name": "T", + "completing_middle_name": null, + "completing_prefix": null, + "completing_suffix": null, + "date_signed": "2023-10-01", + "memo_text_description": null + } +}, +{ + "model": "transactions.schedulee", + "pk": "dea48c3a-c47b-4596-b905-e2df0a3da773", + "fields": { + "payee_organization_name": "The Most Organized Organization", + "payee_last_name": null, + "payee_first_name": null, + "payee_middle_name": null, + "payee_prefix": null, + "payee_suffix": null, + "payee_street_1": "1234 Testing Ln", + "payee_street_2": null, + "payee_city": "Testopolis", + "payee_state": "AK", + "payee_zip": "12345", + "election_code": "C2012", + "election_other_description": null, + "dissemination_date": null, + "expenditure_amount": "520.00", + "disbursement_date": "2023-11-16", + "calendar_ytd": "520.00", + "expenditure_purpose_descrip": "yes", + "category_code": null, + "payee_cmtte_fec_id_number": null, + "support_oppose_code": "S", + "so_candidate_id_number": "S6AK00162", + "so_candidate_last_name": "Testerson", + "so_candidate_first_name": "Phil", + "so_candinate_middle_name": null, + "so_candidate_prefix": null, + "so_candidate_suffix": null, + "so_candidate_office": "S", + "so_candidate_district": null, + "so_candidate_state": "AK", + "completing_last_name": "T", + "completing_first_name": "T", + "completing_middle_name": null, + "completing_prefix": null, + "completing_suffix": null, + "date_signed": "2023-10-01", + "memo_text_description": null + } +}, +{ + "model": "transactions.schedulee", + "pk": "ef41ee6e-9220-4cd0-9df3-440a7b4b6dd5", + "fields": { + "payee_organization_name": "The Most Organized Organization", + "payee_last_name": null, + "payee_first_name": null, + "payee_middle_name": null, + "payee_prefix": null, + "payee_suffix": null, + "payee_street_1": "1234 Testing Ln", + "payee_street_2": null, + "payee_city": "Testopolis", + "payee_state": "AK", + "payee_zip": "12345", + "election_code": "C2012", + "election_other_description": null, + "dissemination_date": "2023-10-16", + "expenditure_amount": "16.00", + "disbursement_date": "2023-10-31", + "calendar_ytd": "16.00", + "expenditure_purpose_descrip": "nooo", + "category_code": null, + "payee_cmtte_fec_id_number": null, + "support_oppose_code": "O", + "so_candidate_id_number": "S6AK00162", + "so_candidate_last_name": "Testerson", + "so_candidate_first_name": "Phil", + "so_candinate_middle_name": null, + "so_candidate_prefix": null, + "so_candidate_suffix": null, + "so_candidate_office": "S", + "so_candidate_district": null, + "so_candidate_state": "AK", + "completing_last_name": "T", + "completing_first_name": "T", + "completing_middle_name": null, + "completing_prefix": null, + "completing_suffix": null, + "date_signed": "2023-10-01", + "memo_text_description": null + } +}, +{ + "model": "transactions.transaction", + "pk": "0b0b9776-df8b-4f5f-b4c5-d751167417e7", + "fields": { + "deleted": null, + "committee_account": "355fda97-57f8-424d-94fd-a3161a9161c7", + "report": "b6d60d2d-d926-4e89-ad4b-c47d152a66ae", + "transaction_type_identifier": "INDIVIDUAL_RECEIPT", + "aggregation_group": "GENERAL", + "parent_transaction": null, + "debt": null, + "loan": null, + "_form_type": "SA11AI", + "transaction_id": "3DBFB03D68B950AC8015", + "entity_type": "IND", + "memo_code": null, + "force_itemized": null, + "force_unaggregated": null, + "created": "2023-10-17T18:40:33.783Z", + "updated": "2023-10-17T18:40:33.783Z", + "contact_1": "112301a6-2461-4cac-9048-d9568bd74786", + "contact_2": null, + "contact_3": null, + "memo_text": null, + "schedule_a": "5c039ffa-cf15-4c03-81e2-c6dcf7dcf0e1", + "schedule_b": null, + "schedule_c": null, + "schedule_c1": null, + "schedule_c2": null, + "schedule_d": null, + "schedule_e": null + } +}, +{ + "model": "transactions.transaction", + "pk": "18d1d052-ee32-4679-bb9b-11ea88ca2c12", + "fields": { + "deleted": null, + "committee_account": "355fda97-57f8-424d-94fd-a3161a9161c7", + "report": "b6d60d2d-d926-4e89-ad4b-c47d152a66ae", + "transaction_type_identifier": "INDEPENDENT_EXPENDITURE_STAFF_REIMBURSEMENT", + "aggregation_group": "INDEPENDENT_EXPENDITURE", + "parent_transaction": null, + "debt": null, + "loan": null, + "_form_type": "SE", + "transaction_id": "3B3F887FF7AE9B9159DC", + "entity_type": "IND", + "memo_code": null, + "force_itemized": null, + "force_unaggregated": null, + "created": "2023-10-17T18:42:09.498Z", + "updated": "2023-10-17T20:02:16.534Z", + "contact_1": "112301a6-2461-4cac-9048-d9568bd74786", + "contact_2": "6ab0f6d8-c9b1-4068-88a1-dbabda1dee73", + "contact_3": null, + "memo_text": null, + "schedule_a": null, + "schedule_b": null, + "schedule_c": null, + "schedule_c1": null, + "schedule_c2": null, + "schedule_d": null, + "schedule_e": "21f75558-9ac7-4aaf-9471-be2d7f8d2902" + } +}, +{ + "model": "transactions.transaction", + "pk": "a92359eb-b3a4-495c-8ed0-5e3afa655212", + "fields": { + "deleted": null, + "committee_account": "355fda97-57f8-424d-94fd-a3161a9161c7", + "report": "b6d60d2d-d9264e89-ad4b-c47d152a66ae", + "transaction_type_identifier": "INDEPENDENT_EXPENDITURE", + "aggregation_group": "INDEPENDENT_EXPENDITURE", + "parent_transaction": null, + "debt": null, + "loan": null, + "_form_type": "SE", + "transaction_id": "0AD8839F08FE0CC342A5", + "entity_type": "ORG", + "memo_code": null, + "force_itemized": null, + "force_unaggregated": null, + "created": "2023-10-17T18:41:14.799Z", + "updated": "2023-10-17T20:02:09.054Z", + "contact_1": "72fd8435-3d5d-429f-9f23-56ea04fc8b2f", + "contact_2": "6ab0f6d8-c9b1-4068-88a1-dbabda1dee73", + "contact_3": null, + "memo_text": null, + "schedule_a": null, + "schedule_b": null, + "schedule_c": null, + "schedule_c1": null, + "schedule_c2": null, + "schedule_d": null, + "schedule_e": "ef41ee6e-9220-4cd0-9df3-440a7b4b6dd5" + } +}, +{ + "model": "transactions.transaction", + "pk": "c4ba684a-607f-4f5d-bfb4-0fa1776d4e35", + "fields": { + "deleted": null, + "committee_account": "355fda97-57f8-424d-94fd-a3161a9161c7", + "report": "b6d60d2d-d9264e89-ad4b-c47d152a66ae", + "transaction_type_identifier": "INDEPENDENT_EXPENDITURE", + "aggregation_group": "INDEPENDENT_EXPENDITURE", + "parent_transaction": null, + "debt": null, + "loan": null, + "_form_type": "SE", + "transaction_id": "55EE8AF7180CC73A9CE6", + "entity_type": "ORG", + "memo_code": null, + "force_itemized": null, + "force_unaggregated": null, + "created": "2023-10-17T18:44:29.869Z", + "updated": "2023-10-17T20:01:58.399Z", + "contact_1": "72fd8435-3d5d-429f-9f23-56ea04fc8b2f", + "contact_2": "6ab0f6d8-c9b1-4068-88a1-dbabda1dee73", + "contact_3": null, + "memo_text": null, + "schedule_a": null, + "schedule_b": null, + "schedule_c": null, + "schedule_c1": null, + "schedule_c2": null, + "schedule_d": null, + "schedule_e": "dea48c3a-c47b-4596-b905-e2df0a3da773" + } +} +] From 28979ada99c8e7401ae2e0046b353d4d08c7ba05 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Thu, 19 Oct 2023 11:10:29 -0400 Subject: [PATCH 22/70] Adds line_9 to summary calculation --- .../test_schedulec_summary_transactions.json | 74 +++++++++++++++++++ .../test_scheduled_summary_transactions.json | 71 ++++++++++++++++++ .../fecfiler/web_services/summary/summary.py | 10 +++ .../fecfiler/web_services/summary/tasks.py | 2 + .../web_services/summary/test_summary.py | 3 + 5 files changed, 160 insertions(+) create mode 100644 django-backend/fecfiler/web_services/fixtures/test_schedulec_summary_transactions.json create mode 100644 django-backend/fecfiler/web_services/fixtures/test_scheduled_summary_transactions.json diff --git a/django-backend/fecfiler/web_services/fixtures/test_schedulec_summary_transactions.json b/django-backend/fecfiler/web_services/fixtures/test_schedulec_summary_transactions.json new file mode 100644 index 0000000000..fa4df1bbd5 --- /dev/null +++ b/django-backend/fecfiler/web_services/fixtures/test_schedulec_summary_transactions.json @@ -0,0 +1,74 @@ +[ + { + "comment": "SC/9 within dates that should count", + "model": "transactions.schedulec", + "fields": { + "id": "c11aefe8-9b5f-4539-b7e5-210000000001", + "loan_amount": 150, + "loan_incurred_date": "2005-02-01" + } + }, + { + "comment": "SC/9 within dates that should count", + "model": "transactions.transaction", + "fields": { + "id": "c12aefe8-9b5f-4539-b7e5-210000000001", + "schedule_c_id": "c11aefe8-9b5f-4539-b7e5-210000000001", + "committee_account_id": "735db943-9446-462a-9be0-c820baadb622", + "report_id": "b6d60d2d-d926-4e89-ad4b-c47d152a66ae", + "_form_type": "SC/9", + "memo_code": false, + "created": "2022-02-09T00:00:00.000Z", + "updated": "2022-02-09T00:00:00.000Z", + "transaction_type_identifier": "TRANSFER_TO_AFFILIATES" + } + }, + { + "comment": "SC/9 before dates but same year, and that should count in Column B", + "model": "transactions.schedulec", + "fields": { + "id": "c11aefe8-9b5f-4539-b7e5-210000000002", + "loan_amount": 100, + "loan_incurred_date": "2005-01-01" + } + }, + { + "comment": "SC/9 before dates but same year, and that should count in Column B", + "model": "transactions.transaction", + "fields": { + "id": "c12aefe8-9b5f-4539-b7e5-210000000002", + "schedule_c_id": "c11aefe8-9b5f-4539-b7e5-210000000002", + "committee_account_id": "735db943-9446-462a-9be0-c820baadb622", + "report_id": "1406535e-f99f-42c4-97a8-247904b7d297", + "_form_type": "SC/9", + "memo_code": false, + "created": "2022-02-09T00:00:00.000Z", + "updated": "2022-02-09T00:00:00.000Z", + "transaction_type_identifier": "TRANSFER_TO_AFFILIATES" + } + }, + { + "comment": "SC/9 different year, and that should not count", + "model": "transactions.schedulec", + "fields": { + "id": "c11aefe8-9b5f-4539-b7e5-210000000003", + "loan_amount": 100, + "loan_incurred_date": "2004-12-01" + } + }, + { + "comment": "SC/9 different year, and that should not count", + "model": "transactions.transaction", + "fields": { + "id": "c12aefe8-9b5f-4539-b7e5-210000000003", + "schedule_c_id": "c11aefe8-9b5f-4539-b7e5-210000000003", + "committee_account_id": "735db943-9446-462a-9be0-c820baadb622", + "report_id": "1406535e-f99f-42c4-97a8-247904b7d297", + "_form_type": "SC/9", + "memo_code": false, + "created": "2022-02-09T00:00:00.000Z", + "updated": "2022-02-09T00:00:00.000Z", + "transaction_type_identifier": "TRANSFER_TO_AFFILIATES" + } + } +] \ No newline at end of file diff --git a/django-backend/fecfiler/web_services/fixtures/test_scheduled_summary_transactions.json b/django-backend/fecfiler/web_services/fixtures/test_scheduled_summary_transactions.json new file mode 100644 index 0000000000..bb48b82313 --- /dev/null +++ b/django-backend/fecfiler/web_services/fixtures/test_scheduled_summary_transactions.json @@ -0,0 +1,71 @@ +[ + { + "comment": "SD/9 within dates that should count", + "model": "transactions.scheduled", + "fields": { + "id": "d11aefe8-9b5f-4539-b7e5-210000000001", + "incurred_amount": 75 + } + }, + { + "comment": "SD/9 within dates that should count", + "model": "transactions.transaction", + "fields": { + "id": "d12aefe8-9b5f-4539-b7e5-210000000001", + "schedule_d_id": "d11aefe8-9b5f-4539-b7e5-210000000001", + "committee_account_id": "735db943-9446-462a-9be0-c820baadb622", + "report_id": "b6d60d2d-d926-4e89-ad4b-c47d152a66ae", + "_form_type": "SD/9", + "memo_code": false, + "created": "2022-02-09T00:00:00.000Z", + "updated": "2022-02-09T00:00:00.000Z", + "transaction_type_identifier": "TRANSFER_TO_AFFILIATES" + } + }, + { + "comment": "SD/9 before dates but same year, and that should count in Column B", + "model": "transactions.scheduled", + "fields": { + "id": "d11aefe8-9b5f-4539-b7e5-210000000002", + "incurred_amount": 100 + } + }, + { + "comment": "SD/9 before dates but same year, and that should count in Column B", + "model": "transactions.transaction", + "fields": { + "id": "d12aefe8-9b5f-4539-b7e5-210000000002", + "schedule_d_id": "d11aefe8-9b5f-4539-b7e5-210000000002", + "committee_account_id": "735db943-9446-462a-9be0-c820baadb622", + "report_id": "1406535e-f99f-42c4-97a8-247904b7d297", + "_form_type": "SD/9", + "memo_code": false, + "created": "2022-02-09T00:00:00.000Z", + "updated": "2022-02-09T00:00:00.000Z", + "transaction_type_identifier": "TRANSFER_TO_AFFILIATES" + } + }, + { + "comment": "SD/9 different year, and that should not count", + "model": "transactions.scheduled", + "fields": { + "id": "d11aefe8-9b5f-4539-b7e5-210000000003", + "incurred_amount": 100 + } + }, + { + "comment": "SD/9 different year, and that should not count", + "model": "transactions.transaction", + "fields": { + "id": "d12aefe8-9b5f-4539-b7e5-210000000003", + "schedule_d_id": "d11aefe8-9b5f-4539-b7e5-210000000003", + "committee_account_id": "735db943-9446-462a-9be0-c820baadb622", + "report_id": "1406535e-f99f-42c4-97a8-247904b7d297", + "_form_type": "SD/9", + "memo_code": false, + "created": "2022-02-09T00:00:00.000Z", + "updated": "2022-02-09T00:00:00.000Z", + "transaction_type_identifier": "TRANSFER_TO_AFFILIATES" + } + } +] \ No newline at end of file diff --git a/django-backend/fecfiler/web_services/summary/summary.py b/django-backend/fecfiler/web_services/summary/summary.py index bdcca18a41..4af8655c1e 100644 --- a/django-backend/fecfiler/web_services/summary/summary.py +++ b/django-backend/fecfiler/web_services/summary/summary.py @@ -42,7 +42,11 @@ def calculate_summary_column_a(self): line_28c=self.get_line("SB28C"), line_29=self.get_line("SB29"), line_30b=self.get_line("SB30B"), + # Temporary aggregations + temp_sc9=self.get_line("SC/9"), + temp_sd9=self.get_line("SD/9") ) + summary["line_9"]= summary["temp_sc9"] + summary["temp_sd9"] summary["line_11aiii"] = summary["line_11ai"] + summary["line_11aii"] summary["line_11d"] = ( summary["line_11aiii"] + summary["line_11b"] + summary["line_11c"] @@ -54,6 +58,12 @@ def calculate_summary_column_a(self): summary["line_34"] = summary["line_28d"] summary["line_35"] = summary["line_33"] - summary["line_34"] summary["line_37"] = summary["line_15"] + + # Remove temporary aggregations to clean up the summary + for key in list(summary.keys()): + if key.startswith("temp_"): + summary.pop(key) + return summary def calculate_summary_column_b(self): diff --git a/django-backend/fecfiler/web_services/summary/tasks.py b/django-backend/fecfiler/web_services/summary/tasks.py index a2b1ee0a98..961adeb449 100644 --- a/django-backend/fecfiler/web_services/summary/tasks.py +++ b/django-backend/fecfiler/web_services/summary/tasks.py @@ -33,6 +33,8 @@ def calculate_summary(report_id): a = summary["a"] b = summary["b"] + # line 9 + report.form_3x.L9_debts_to_period = a["line_9"] # line 11ai report.form_3x.L11ai_itemized_period = a["line_11ai"] report.form_3x.L11ai_itemized_ytd = b["line_11ai"] diff --git a/django-backend/fecfiler/web_services/summary/test_summary.py b/django-backend/fecfiler/web_services/summary/test_summary.py index 6bcb243f17..565e77507f 100644 --- a/django-backend/fecfiler/web_services/summary/test_summary.py +++ b/django-backend/fecfiler/web_services/summary/test_summary.py @@ -11,6 +11,8 @@ class F3XReportTestCase(TestCase): "test_f3x_reports", "test_schedulea_summary_transactions", "test_scheduleb_summary_transactions", + "test_schedulec_summary_transactions", + "test_scheduled_summary_transactions", "test_contacts", ] @@ -18,6 +20,7 @@ def test_calculate_summary_column_a(self): f3x = Report.objects.get(id="b6d60d2d-d926-4e89-ad4b-c47d152a66ae") summary_service = SummaryService(f3x) summary = summary_service.calculate_summary() + self.assertEqual(summary["a"]["line_9"], Decimal("225.00")) self.assertEqual(summary["a"]["line_11ai"], Decimal("10000.23")) self.assertEqual(summary["a"]["line_11aii"], Decimal("3.77")) self.assertEqual( From 833f8103ac85673b896315e6acad1f759ab71549 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Thu, 19 Oct 2023 14:14:48 -0400 Subject: [PATCH 23/70] Adds whitespace around an = --- django-backend/fecfiler/web_services/summary/summary.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django-backend/fecfiler/web_services/summary/summary.py b/django-backend/fecfiler/web_services/summary/summary.py index 4af8655c1e..9d7846f749 100644 --- a/django-backend/fecfiler/web_services/summary/summary.py +++ b/django-backend/fecfiler/web_services/summary/summary.py @@ -46,7 +46,7 @@ def calculate_summary_column_a(self): temp_sc9=self.get_line("SC/9"), temp_sd9=self.get_line("SD/9") ) - summary["line_9"]= summary["temp_sc9"] + summary["temp_sd9"] + summary["line_9"] = summary["temp_sc9"] + summary["temp_sd9"] summary["line_11aiii"] = summary["line_11ai"] + summary["line_11aii"] summary["line_11d"] = ( summary["line_11aiii"] + summary["line_11b"] + summary["line_11c"] From a0ac17ddb1aaa795e191d71ac66c1b7c9ca80757 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Thu, 19 Oct 2023 14:17:49 -0400 Subject: [PATCH 24/70] Add indepenent expenditure memos --- django-backend/fecfiler/transactions/schedule_e/managers.py | 3 +++ requirements.txt | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/django-backend/fecfiler/transactions/schedule_e/managers.py b/django-backend/fecfiler/transactions/schedule_e/managers.py index 432b2bad4d..9ee47be00a 100644 --- a/django-backend/fecfiler/transactions/schedule_e/managers.py +++ b/django-backend/fecfiler/transactions/schedule_e/managers.py @@ -4,4 +4,7 @@ "INDEPENDENT_EXPENDITURE_CREDIT_CARD_PAYMENT", "INDEPENDENT_EXPENDITURE_PAYMENT_TO_PAYROLL", "INDEPENDENT_EXPENDITURE_STAFF_REIMBURSEMENT", + "INDEPENDENT_EXPENDITURE_CREDIT_CARD_PAYMENT_MEMO", + "INDEPENDENT_EXPENDITURE_PAYMENT_TO_PAYROLL_MEMO", + "INDEPENDENT_EXPENDITURE_STAFF_REIMBURSEMENT_MEMO", ] diff --git a/requirements.txt b/requirements.txt index 767ecdcd61..80c69e0d7a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ django-cors-headers==3.13.0 django-storages==1.13.1 djangorestframework==3.14.0 drf-spectacular==0.24.2 -git+https://github.com/fecgov/fecfile-validate@57fd1af27991bca418df51b43b2999fef7ef4119#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@4c8954efb9dde7da4591d6d348557b83b6cc18df#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.30 gunicorn==20.1.0 Jinja2==3.1.2 From 2293fc77edb780ea5df420b81b1e160001301a84 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Thu, 19 Oct 2023 14:38:49 -0400 Subject: [PATCH 25/70] Adds a calendar_ytd test --- django-backend/fecfiler/transactions/tests/test_manager.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/django-backend/fecfiler/transactions/tests/test_manager.py b/django-backend/fecfiler/transactions/tests/test_manager.py index 51e9f533eb..2536fdff1a 100644 --- a/django-backend/fecfiler/transactions/tests/test_manager.py +++ b/django-backend/fecfiler/transactions/tests/test_manager.py @@ -2,6 +2,7 @@ from fecfiler.transactions.models import Transaction, Schedule import uuid +from decimal import Decimal class TransactionManagerTestCase(TestCase): @@ -9,6 +10,7 @@ class TransactionManagerTestCase(TestCase): "test_committee_accounts", "test_f3x_reports", "test_transaction_manager_transactions", + "test_election_aggregation_data", ] def test_order_of_transactions(self): @@ -39,6 +41,10 @@ def test_refund_aggregation(self): refund = Transaction.objects.get(id="bbbbbbbb-3274-47d8-9388-7294a3fd4321") self.assertEqual(refund.aggregate, 4444) + def test_election_aggregation(self): + transaction = Transaction.objects.get(id="c4ba684a-607f-4f5d-bfb4-0fa1776d4e35") + self.assertEqual(transaction.calendar_ytd, Decimal("578.00")) + def test_debt_repayment(self): repayment = Transaction.objects.get(id="dbdbdbdb-62f7-4a11-ac8e-27ea2afa9491") debt = Transaction.objects.get(id="dddddddd-3274-47d8-9388-7294a3fd4321") From 725997082a04368b9296fbfa607f400079e9d345 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Thu, 19 Oct 2023 14:40:13 -0400 Subject: [PATCH 26/70] Removes no longer necessary method --- django-backend/fecfiler/transactions/managers.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/django-backend/fecfiler/transactions/managers.py b/django-backend/fecfiler/transactions/managers.py index 6a9394af4e..f5521f6a8f 100644 --- a/django-backend/fecfiler/transactions/managers.py +++ b/django-backend/fecfiler/transactions/managers.py @@ -65,7 +65,6 @@ def get_queryset(self): "schedule_e__expenditure_amount", ), effective_amount=self.get_amount_clause(), - candidate_contact_id=self.get_candidate_contact_clause(), ) ) @@ -329,20 +328,6 @@ def get_amount_clause(self): output_field=DecimalField(), ) - def get_candidate_contact_clause(self): - return Case( - When( - contact_2__type=Contact.ContactType.CANDIDATE, - then=F("contact_2_id") - ), - When( - contact_3__type=Contact.ContactType.CANDIDATE, - then=F("contact_3_id") - ), - default=None, - output_field=UUIDField() - ) - class Schedule(Enum): A = Value("A") From 0140c45360e8cc6e9c2e56289a205a4d21f1b730 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Thu, 19 Oct 2023 14:46:16 -0400 Subject: [PATCH 27/70] Fixes a couple linting errors --- django-backend/fecfiler/transactions/managers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/django-backend/fecfiler/transactions/managers.py b/django-backend/fecfiler/transactions/managers.py index f5521f6a8f..cce2d006ab 100644 --- a/django-backend/fecfiler/transactions/managers.py +++ b/django-backend/fecfiler/transactions/managers.py @@ -21,13 +21,11 @@ BooleanField, TextField, DecimalField, - UUIDField, ExpressionWrapper ) from decimal import Decimal from enum import Enum from .schedule_b.managers import refunds as schedule_b_refunds -from fecfiler.contacts.models import Contact """Manager to deterimine fields that are used the same way across transactions, but are called different names""" From cd7c93d7465eeb7872e419187ad0247ca75b294c Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Thu, 19 Oct 2023 15:16:53 -0400 Subject: [PATCH 28/70] Adds summary line 10 calculation --- .../test_schedulec_summary_transactions.json | 90 +++++++++++++++++-- .../test_scheduled_summary_transactions.json | 87 ++++++++++++++++-- .../fecfiler/web_services/summary/summary.py | 5 +- .../fecfiler/web_services/summary/tasks.py | 2 + .../web_services/summary/test_summary.py | 1 + 5 files changed, 166 insertions(+), 19 deletions(-) diff --git a/django-backend/fecfiler/web_services/fixtures/test_schedulec_summary_transactions.json b/django-backend/fecfiler/web_services/fixtures/test_schedulec_summary_transactions.json index fa4df1bbd5..27e92d6758 100644 --- a/django-backend/fecfiler/web_services/fixtures/test_schedulec_summary_transactions.json +++ b/django-backend/fecfiler/web_services/fixtures/test_schedulec_summary_transactions.json @@ -3,7 +3,7 @@ "comment": "SC/9 within dates that should count", "model": "transactions.schedulec", "fields": { - "id": "c11aefe8-9b5f-4539-b7e5-210000000001", + "id": "c11aefe8-9b5f-4539-b7e5-c09000000001", "loan_amount": 150, "loan_incurred_date": "2005-02-01" } @@ -12,8 +12,8 @@ "comment": "SC/9 within dates that should count", "model": "transactions.transaction", "fields": { - "id": "c12aefe8-9b5f-4539-b7e5-210000000001", - "schedule_c_id": "c11aefe8-9b5f-4539-b7e5-210000000001", + "id": "c12aefe8-9b5f-4539-b7e5-c09000000001", + "schedule_c_id": "c11aefe8-9b5f-4539-b7e5-c09000000001", "committee_account_id": "735db943-9446-462a-9be0-c820baadb622", "report_id": "b6d60d2d-d926-4e89-ad4b-c47d152a66ae", "_form_type": "SC/9", @@ -27,7 +27,7 @@ "comment": "SC/9 before dates but same year, and that should count in Column B", "model": "transactions.schedulec", "fields": { - "id": "c11aefe8-9b5f-4539-b7e5-210000000002", + "id": "c11aefe8-9b5f-4539-b7e5-c09000000002", "loan_amount": 100, "loan_incurred_date": "2005-01-01" } @@ -36,8 +36,8 @@ "comment": "SC/9 before dates but same year, and that should count in Column B", "model": "transactions.transaction", "fields": { - "id": "c12aefe8-9b5f-4539-b7e5-210000000002", - "schedule_c_id": "c11aefe8-9b5f-4539-b7e5-210000000002", + "id": "c12aefe8-9b5f-4539-b7e5-c09000000002", + "schedule_c_id": "c11aefe8-9b5f-4539-b7e5-c09000000002", "committee_account_id": "735db943-9446-462a-9be0-c820baadb622", "report_id": "1406535e-f99f-42c4-97a8-247904b7d297", "_form_type": "SC/9", @@ -51,7 +51,7 @@ "comment": "SC/9 different year, and that should not count", "model": "transactions.schedulec", "fields": { - "id": "c11aefe8-9b5f-4539-b7e5-210000000003", + "id": "c11aefe8-9b5f-4539-b7e5-c09000000003", "loan_amount": 100, "loan_incurred_date": "2004-12-01" } @@ -60,8 +60,8 @@ "comment": "SC/9 different year, and that should not count", "model": "transactions.transaction", "fields": { - "id": "c12aefe8-9b5f-4539-b7e5-210000000003", - "schedule_c_id": "c11aefe8-9b5f-4539-b7e5-210000000003", + "id": "c12aefe8-9b5f-4539-b7e5-c09000000003", + "schedule_c_id": "c11aefe8-9b5f-4539-b7e5-c09000000003", "committee_account_id": "735db943-9446-462a-9be0-c820baadb622", "report_id": "1406535e-f99f-42c4-97a8-247904b7d297", "_form_type": "SC/9", @@ -70,5 +70,77 @@ "updated": "2022-02-09T00:00:00.000Z", "transaction_type_identifier": "TRANSFER_TO_AFFILIATES" } + }, + { + "comment": "SC/10 within dates that should count", + "model": "transactions.schedulec", + "fields": { + "id": "c11aefe8-9b5f-4539-b7e5-c10000000001", + "loan_amount": 30, + "loan_incurred_date": "2005-02-01" + } + }, + { + "comment": "SC/10 within dates that should count", + "model": "transactions.transaction", + "fields": { + "id": "c12aefe8-9b5f-4539-b7e5-c10000000001", + "schedule_c_id": "c11aefe8-9b5f-4539-b7e5-c10000000001", + "committee_account_id": "735db943-9446-462a-9be0-c820baadb622", + "report_id": "b6d60d2d-d926-4e89-ad4b-c47d152a66ae", + "_form_type": "SC/10", + "memo_code": false, + "created": "2022-02-09T00:00:00.000Z", + "updated": "2022-02-09T00:00:00.000Z", + "transaction_type_identifier": "TRANSFER_TO_AFFILIATES" + } + }, + { + "comment": "SC/10 before dates but same year, and that should count in Column B", + "model": "transactions.schedulec", + "fields": { + "id": "c11aefe8-9b5f-4539-b7e5-c10000000002", + "loan_amount": 100, + "loan_incurred_date": "2005-01-01" + } + }, + { + "comment": "SC/10 before dates but same year, and that should count in Column B", + "model": "transactions.transaction", + "fields": { + "id": "c12aefe8-9b5f-4539-b7e5-c10000000002", + "schedule_c_id": "c11aefe8-9b5f-4539-b7e5-c10000000002", + "committee_account_id": "735db943-9446-462a-9be0-c820baadb622", + "report_id": "1406535e-f99f-42c4-97a8-247904b7d297", + "_form_type": "SC/10", + "memo_code": false, + "created": "2022-02-09T00:00:00.000Z", + "updated": "2022-02-09T00:00:00.000Z", + "transaction_type_identifier": "TRANSFER_TO_AFFILIATES" + } + }, + { + "comment": "SC/10 different year, and that should not count", + "model": "transactions.schedulec", + "fields": { + "id": "c11aefe8-9b5f-4539-b7e5-c10000000003", + "loan_amount": 100, + "loan_incurred_date": "2004-12-01" + } + }, + { + "comment": "SC/10 different year, and that should not count", + "model": "transactions.transaction", + "fields": { + "id": "c12aefe8-9b5f-4539-b7e5-c10000000003", + "schedule_c_id": "c11aefe8-9b5f-4539-b7e5-c10000000003", + "committee_account_id": "735db943-9446-462a-9be0-c820baadb622", + "report_id": "1406535e-f99f-42c4-97a8-247904b7d297", + "_form_type": "SC/10", + "memo_code": false, + "created": "2022-02-09T00:00:00.000Z", + "updated": "2022-02-09T00:00:00.000Z", + "transaction_type_identifier": "TRANSFER_TO_AFFILIATES" + } } ] \ No newline at end of file diff --git a/django-backend/fecfiler/web_services/fixtures/test_scheduled_summary_transactions.json b/django-backend/fecfiler/web_services/fixtures/test_scheduled_summary_transactions.json index bb48b82313..3b868f8f23 100644 --- a/django-backend/fecfiler/web_services/fixtures/test_scheduled_summary_transactions.json +++ b/django-backend/fecfiler/web_services/fixtures/test_scheduled_summary_transactions.json @@ -3,7 +3,7 @@ "comment": "SD/9 within dates that should count", "model": "transactions.scheduled", "fields": { - "id": "d11aefe8-9b5f-4539-b7e5-210000000001", + "id": "d11aefe8-9b5f-4539-b7e5-d09000000001", "incurred_amount": 75 } }, @@ -11,8 +11,8 @@ "comment": "SD/9 within dates that should count", "model": "transactions.transaction", "fields": { - "id": "d12aefe8-9b5f-4539-b7e5-210000000001", - "schedule_d_id": "d11aefe8-9b5f-4539-b7e5-210000000001", + "id": "d12aefe8-9b5f-4539-b7e5-d09000000001", + "schedule_d_id": "d11aefe8-9b5f-4539-b7e5-d09000000001", "committee_account_id": "735db943-9446-462a-9be0-c820baadb622", "report_id": "b6d60d2d-d926-4e89-ad4b-c47d152a66ae", "_form_type": "SD/9", @@ -26,7 +26,7 @@ "comment": "SD/9 before dates but same year, and that should count in Column B", "model": "transactions.scheduled", "fields": { - "id": "d11aefe8-9b5f-4539-b7e5-210000000002", + "id": "d11aefe8-9b5f-4539-b7e5-d09000000002", "incurred_amount": 100 } }, @@ -34,8 +34,8 @@ "comment": "SD/9 before dates but same year, and that should count in Column B", "model": "transactions.transaction", "fields": { - "id": "d12aefe8-9b5f-4539-b7e5-210000000002", - "schedule_d_id": "d11aefe8-9b5f-4539-b7e5-210000000002", + "id": "d12aefe8-9b5f-4539-b7e5-d09000000002", + "schedule_d_id": "d11aefe8-9b5f-4539-b7e5-d09000000002", "committee_account_id": "735db943-9446-462a-9be0-c820baadb622", "report_id": "1406535e-f99f-42c4-97a8-247904b7d297", "_form_type": "SD/9", @@ -49,7 +49,7 @@ "comment": "SD/9 different year, and that should not count", "model": "transactions.scheduled", "fields": { - "id": "d11aefe8-9b5f-4539-b7e5-210000000003", + "id": "d11aefe8-9b5f-4539-b7e5-d09000000003", "incurred_amount": 100 } }, @@ -57,8 +57,8 @@ "comment": "SD/9 different year, and that should not count", "model": "transactions.transaction", "fields": { - "id": "d12aefe8-9b5f-4539-b7e5-210000000003", - "schedule_d_id": "d11aefe8-9b5f-4539-b7e5-210000000003", + "id": "d12aefe8-9b5f-4539-b7e5-d09000000003", + "schedule_d_id": "d11aefe8-9b5f-4539-b7e5-d09000000003", "committee_account_id": "735db943-9446-462a-9be0-c820baadb622", "report_id": "1406535e-f99f-42c4-97a8-247904b7d297", "_form_type": "SD/9", @@ -67,5 +67,74 @@ "updated": "2022-02-09T00:00:00.000Z", "transaction_type_identifier": "TRANSFER_TO_AFFILIATES" } + }, + { + "comment": "SD/10 within dates that should count", + "model": "transactions.scheduled", + "fields": { + "id": "d11aefe8-9b5f-4539-b7e5-d10000000001", + "incurred_amount": 220 + } + }, + { + "comment": "SD/10 within dates that should count", + "model": "transactions.transaction", + "fields": { + "id": "d12aefe8-9b5f-4539-b7e5-d10000000001", + "schedule_d_id": "d11aefe8-9b5f-4539-b7e5-d10000000001", + "committee_account_id": "735db943-9446-462a-9be0-c820baadb622", + "report_id": "b6d60d2d-d926-4e89-ad4b-c47d152a66ae", + "_form_type": "SD/10", + "memo_code": false, + "created": "2022-02-09T00:00:00.000Z", + "updated": "2022-02-09T00:00:00.000Z", + "transaction_type_identifier": "TRANSFER_TO_AFFILIATES" + } + }, + { + "comment": "SD/10 before dates but same year, and that should count in Column B", + "model": "transactions.scheduled", + "fields": { + "id": "d11aefe8-9b5f-4539-b7e5-d10000000002", + "incurred_amount": 100 + } + }, + { + "comment": "SD/10 before dates but same year, and that should count in Column B", + "model": "transactions.transaction", + "fields": { + "id": "d12aefe8-9b5f-4539-b7e5-d10000000002", + "schedule_d_id": "d11aefe8-9b5f-4539-b7e5-d10000000002", + "committee_account_id": "735db943-9446-462a-9be0-c820baadb622", + "report_id": "1406535e-f99f-42c4-97a8-247904b7d297", + "_form_type": "SD/10", + "memo_code": false, + "created": "2022-02-09T00:00:00.000Z", + "updated": "2022-02-09T00:00:00.000Z", + "transaction_type_identifier": "TRANSFER_TO_AFFILIATES" + } + }, + { + "comment": "SD/10 different year, and that should not count", + "model": "transactions.scheduled", + "fields": { + "id": "d11aefe8-9b5f-4539-b7e5-d10000000003", + "incurred_amount": 100 + } + }, + { + "comment": "SD/10 different year, and that should not count", + "model": "transactions.transaction", + "fields": { + "id": "d12aefe8-9b5f-4539-b7e5-d10000000003", + "schedule_d_id": "d11aefe8-9b5f-4539-b7e5-d10000000003", + "committee_account_id": "735db943-9446-462a-9be0-c820baadb622", + "report_id": "1406535e-f99f-42c4-97a8-247904b7d297", + "_form_type": "SD/10", + "memo_code": false, + "created": "2022-02-09T00:00:00.000Z", + "updated": "2022-02-09T00:00:00.000Z", + "transaction_type_identifier": "TRANSFER_TO_AFFILIATES" + } } ] \ No newline at end of file diff --git a/django-backend/fecfiler/web_services/summary/summary.py b/django-backend/fecfiler/web_services/summary/summary.py index 9d7846f749..360ef6d53f 100644 --- a/django-backend/fecfiler/web_services/summary/summary.py +++ b/django-backend/fecfiler/web_services/summary/summary.py @@ -44,9 +44,12 @@ def calculate_summary_column_a(self): line_30b=self.get_line("SB30B"), # Temporary aggregations temp_sc9=self.get_line("SC/9"), - temp_sd9=self.get_line("SD/9") + temp_sd9=self.get_line("SD/9"), + temp_sc10=self.get_line("SC/10"), + temp_sd10=self.get_line("SD/10") ) summary["line_9"] = summary["temp_sc9"] + summary["temp_sd9"] + summary["line_10"] = summary["temp_sc10"] + summary["temp_sd10"] summary["line_11aiii"] = summary["line_11ai"] + summary["line_11aii"] summary["line_11d"] = ( summary["line_11aiii"] + summary["line_11b"] + summary["line_11c"] diff --git a/django-backend/fecfiler/web_services/summary/tasks.py b/django-backend/fecfiler/web_services/summary/tasks.py index 961adeb449..892743f2f4 100644 --- a/django-backend/fecfiler/web_services/summary/tasks.py +++ b/django-backend/fecfiler/web_services/summary/tasks.py @@ -35,6 +35,8 @@ def calculate_summary(report_id): # line 9 report.form_3x.L9_debts_to_period = a["line_9"] + # line 10 + report.form_3x.L10_debts_by_period = a["line_10"] # line 11ai report.form_3x.L11ai_itemized_period = a["line_11ai"] report.form_3x.L11ai_itemized_ytd = b["line_11ai"] diff --git a/django-backend/fecfiler/web_services/summary/test_summary.py b/django-backend/fecfiler/web_services/summary/test_summary.py index 565e77507f..d734905418 100644 --- a/django-backend/fecfiler/web_services/summary/test_summary.py +++ b/django-backend/fecfiler/web_services/summary/test_summary.py @@ -21,6 +21,7 @@ def test_calculate_summary_column_a(self): summary_service = SummaryService(f3x) summary = summary_service.calculate_summary() self.assertEqual(summary["a"]["line_9"], Decimal("225.00")) + self.assertEqual(summary["a"]["line_10"], Decimal("250.00")) self.assertEqual(summary["a"]["line_11ai"], Decimal("10000.23")) self.assertEqual(summary["a"]["line_11aii"], Decimal("3.77")) self.assertEqual( From 0113deb18e20ecf40605352291719f7ffb30240c Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Thu, 19 Oct 2023 15:31:14 -0400 Subject: [PATCH 29/70] Adds summary calculation for lines 6c and 19 --- django-backend/fecfiler/web_services/summary/summary.py | 6 ++++++ django-backend/fecfiler/web_services/summary/tasks.py | 4 ++++ .../fecfiler/web_services/summary/test_summary.py | 2 ++ 3 files changed, 12 insertions(+) diff --git a/django-backend/fecfiler/web_services/summary/summary.py b/django-backend/fecfiler/web_services/summary/summary.py index 360ef6d53f..dfaf3191cc 100644 --- a/django-backend/fecfiler/web_services/summary/summary.py +++ b/django-backend/fecfiler/web_services/summary/summary.py @@ -48,12 +48,18 @@ def calculate_summary_column_a(self): temp_sc10=self.get_line("SC/10"), temp_sd10=self.get_line("SD/10") ) + summary["line_6c"] = ( + summary["line_11c"] + summary["line_12"] + summary["line_13"] + + summary["line_14"] + summary["line_15"] + summary["line_16"] + + summary["line_17"] + summary.get("line_18c", Decimal("0.00")) + ) summary["line_9"] = summary["temp_sc9"] + summary["temp_sd9"] summary["line_10"] = summary["temp_sc10"] + summary["temp_sd10"] summary["line_11aiii"] = summary["line_11ai"] + summary["line_11aii"] summary["line_11d"] = ( summary["line_11aiii"] + summary["line_11b"] + summary["line_11c"] ) + summary["line_19"] = summary["line_6c"] summary["line_28d"] = ( summary["line_28a"] + summary["line_28b"] + summary["line_28c"] ) diff --git a/django-backend/fecfiler/web_services/summary/tasks.py b/django-backend/fecfiler/web_services/summary/tasks.py index 892743f2f4..a03fa16ed1 100644 --- a/django-backend/fecfiler/web_services/summary/tasks.py +++ b/django-backend/fecfiler/web_services/summary/tasks.py @@ -33,6 +33,8 @@ def calculate_summary(report_id): a = summary["a"] b = summary["b"] + # line 6 + report.form_3x.L6c_total_receipts_period = a["line_6c"] # line 9 report.form_3x.L9_debts_to_period = a["line_9"] # line 10 @@ -73,6 +75,8 @@ def calculate_summary(report_id): # line 17 report.form_3x.L17_other_federal_receipts_dividends_period = a["line_17"] report.form_3x.L17_other_federal_receipts_dividends_ytd = b["line_17"] + # line 19 + report.form_3x.L19_total_receipts_period = a["line_19"] # line 21b report.form_3x.L21b_other_federal_operating_expenditures_period = a["line_21b"] report.form_3x.L21b_other_federal_operating_expenditures_ytd = b["line_21b"] diff --git a/django-backend/fecfiler/web_services/summary/test_summary.py b/django-backend/fecfiler/web_services/summary/test_summary.py index d734905418..50897610bc 100644 --- a/django-backend/fecfiler/web_services/summary/test_summary.py +++ b/django-backend/fecfiler/web_services/summary/test_summary.py @@ -20,6 +20,7 @@ def test_calculate_summary_column_a(self): f3x = Report.objects.get(id="b6d60d2d-d926-4e89-ad4b-c47d152a66ae") summary_service = SummaryService(f3x) summary = summary_service.calculate_summary() + self.assertEqual(summary["a"]["line_6c"], Decimal("7636.73")) self.assertEqual(summary["a"]["line_9"], Decimal("225.00")) self.assertEqual(summary["a"]["line_10"], Decimal("250.00")) self.assertEqual(summary["a"]["line_11ai"], Decimal("10000.23")) @@ -40,6 +41,7 @@ def test_calculate_summary_column_a(self): self.assertEqual(summary["a"]["line_15"], summary["a"]["line_37"]) self.assertEqual(summary["a"]["line_16"], Decimal("16.00")) self.assertEqual(summary["a"]["line_17"], Decimal("1000.00")) + self.assertEqual(summary["a"]["line_19"], Decimal("7636.73")) self.assertEqual(summary["a"]["line_21b"], Decimal("150.00")) self.assertEqual(summary["a"]["line_22"], Decimal("22.00")) self.assertEqual(summary["a"]["line_23"], Decimal("14.00")) From c866ce6a66f72840703e6bf711e2472dccde7e85 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Thu, 19 Oct 2023 15:43:54 -0400 Subject: [PATCH 30/70] Adds column B to line_6c and line_19 calculation --- django-backend/fecfiler/web_services/summary/summary.py | 6 ++++++ django-backend/fecfiler/web_services/summary/tasks.py | 2 ++ .../fecfiler/web_services/summary/test_summary.py | 2 ++ 3 files changed, 10 insertions(+) diff --git a/django-backend/fecfiler/web_services/summary/summary.py b/django-backend/fecfiler/web_services/summary/summary.py index dfaf3191cc..f2aef1ad4a 100644 --- a/django-backend/fecfiler/web_services/summary/summary.py +++ b/django-backend/fecfiler/web_services/summary/summary.py @@ -107,10 +107,16 @@ def calculate_summary_column_b(self): line_29=self.get_line("SB29"), line_30b=self.get_line("SB30B"), ) + summary["line_6c"] = ( + summary["line_11c"] + summary["line_12"] + summary["line_13"] + + summary["line_14"] + summary["line_15"] + summary["line_16"] + + summary["line_17"] + summary.get("line_18c", Decimal("0.00")) + ) summary["line_11aiii"] = summary["line_11ai"] + summary["line_11aii"] summary["line_11d"] = ( summary["line_11aiii"] + summary["line_11b"] + summary["line_11c"] ) + summary["line_19"] = summary["line_6c"] summary["line_28d"] = ( summary["line_28a"] + summary["line_28b"] + summary["line_28c"] ) diff --git a/django-backend/fecfiler/web_services/summary/tasks.py b/django-backend/fecfiler/web_services/summary/tasks.py index a03fa16ed1..bc32cf5722 100644 --- a/django-backend/fecfiler/web_services/summary/tasks.py +++ b/django-backend/fecfiler/web_services/summary/tasks.py @@ -35,6 +35,7 @@ def calculate_summary(report_id): # line 6 report.form_3x.L6c_total_receipts_period = a["line_6c"] + report.form_3x.L6c_total_receipts_ytd = a["line_6c"] # line 9 report.form_3x.L9_debts_to_period = a["line_9"] # line 10 @@ -77,6 +78,7 @@ def calculate_summary(report_id): report.form_3x.L17_other_federal_receipts_dividends_ytd = b["line_17"] # line 19 report.form_3x.L19_total_receipts_period = a["line_19"] + report.form_3x.L19_total_receipts_ytd = b["line_19"] # line 21b report.form_3x.L21b_other_federal_operating_expenditures_period = a["line_21b"] report.form_3x.L21b_other_federal_operating_expenditures_ytd = b["line_21b"] diff --git a/django-backend/fecfiler/web_services/summary/test_summary.py b/django-backend/fecfiler/web_services/summary/test_summary.py index 50897610bc..39e2fbfb78 100644 --- a/django-backend/fecfiler/web_services/summary/test_summary.py +++ b/django-backend/fecfiler/web_services/summary/test_summary.py @@ -80,6 +80,7 @@ def test_calculate_summary_column_b(self): t = Transaction.objects.get(id="aaaaaaaa-4d75-46f0-bce2-111000000001") self.assertEqual(t.itemized, False) + self.assertEqual(summary["b"]["line_6c"], Decimal("8336.73")) self.assertEqual(summary["b"]["line_11ai"], Decimal("10000.23")) self.assertEqual(summary["b"]["line_11aii"], Decimal("103.77")) self.assertEqual(summary["b"]["line_11aiii"], Decimal(10104.00)) @@ -95,6 +96,7 @@ def test_calculate_summary_column_b(self): self.assertEqual(summary["b"]["line_15"], Decimal("2225.79")) self.assertEqual(summary["b"]["line_16"], Decimal("116.00")) self.assertEqual(summary["b"]["line_17"], Decimal("1100.00")) + self.assertEqual(summary["b"]["line_19"], Decimal("8336.73")) self.assertEqual(summary["b"]["line_21b"], Decimal("250.00")) self.assertEqual(summary["b"]["line_23"], Decimal("64.00")) self.assertEqual(summary["b"]["line_26"], Decimal("61.00")) From 4274a8fcac4367294238b47eaa940652aefe998f Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Thu, 19 Oct 2023 17:20:41 -0400 Subject: [PATCH 31/70] 1405 started dev --- .../transactions/schedule_c/models.py | 39 ++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/django-backend/fecfiler/transactions/schedule_c/models.py b/django-backend/fecfiler/transactions/schedule_c/models.py index 8e483f0644..bb79779928 100644 --- a/django-backend/fecfiler/transactions/schedule_c/models.py +++ b/django-backend/fecfiler/transactions/schedule_c/models.py @@ -1,5 +1,7 @@ -from django.db import models +from fecfiler.reports.models import Report +from django.db import models, transaction as db_transaction import uuid +import copy class ScheduleC(models.Model): @@ -44,6 +46,41 @@ class ScheduleC(models.Model): lender_candidate_district = models.TextField(null=True, blank=True) memo_text_description = models.TextField(null=True, blank=True) + def save(self, *args, **kwargs): + with db_transaction.atomic(): + super().save(*args, **kwargs) + transaction = self.get_transaction() + if not transaction.memo_code: + report = transaction.report + future_reports = Report.objects.get_queryset().filter( + committee_account=report.committee_id, + upload_submission__isnull=True, + coverage_through_date__gte=report.coverage_through_date, + ) + transactions_to_update = ScheduleC.objects.filter( + transaction__transaction_id=transaction.transaction_id, + transaction__report_id__in=models.Subquery( + future_reports.values('id') + ) + ) + transactions_to_update.update(**transaction) + reports_ids_to_create_transaction_for = list(future_reports.exclude( + id__in=models.Subquery( + transactions_to_update.values('transaction__report_id') + ) + ).values_list('id', flat=True).distinct()) + + for report_id in reports_ids_to_create_transaction_for: + schedule_c_copy = copy.deepcopy(self) + schedule_c_copy.id = None + schedule_c_copy.save() + + transaction_copy = copy.deepcopy(transaction) + transaction_copy.id = None + transaction_copy.report_id = report_id + transaction_copy.schedule_c = schedule_c_copy + transaction_copy.save() + def get_date(self): return self.transaction.report.through_date From facdd2428ad1e6e3db0f1001df811c8b7990c838 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Fri, 20 Oct 2023 16:54:39 -0400 Subject: [PATCH 32/70] Fixes a bug preventing Calendar YTD from working with Presidential candidates --- django-backend/fecfiler/transactions/managers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django-backend/fecfiler/transactions/managers.py b/django-backend/fecfiler/transactions/managers.py index f5c89e2bdf..a0594b294c 100644 --- a/django-backend/fecfiler/transactions/managers.py +++ b/django-backend/fecfiler/transactions/managers.py @@ -84,7 +84,7 @@ def get_queryset(self): "schedule_e__so_candidate_state" ) ) | ( - Q(schedule_e__so_candidate_office__isnull=True) + Q(schedule_e__so_candidate_state__isnull=True) & Q(outer_candidate_state__isnull=True) ) ) & ( From 51983fc916998835c856bda906c0117222607abf Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Mon, 23 Oct 2023 16:55:45 -0400 Subject: [PATCH 33/70] 1405 local testing fixed create --- .../transactions/schedule_c/models.py | 39 +--------------- .../transactions/schedule_c/serializers.py | 45 ++++++++++++++++++- 2 files changed, 44 insertions(+), 40 deletions(-) diff --git a/django-backend/fecfiler/transactions/schedule_c/models.py b/django-backend/fecfiler/transactions/schedule_c/models.py index bb79779928..8e483f0644 100644 --- a/django-backend/fecfiler/transactions/schedule_c/models.py +++ b/django-backend/fecfiler/transactions/schedule_c/models.py @@ -1,7 +1,5 @@ -from fecfiler.reports.models import Report -from django.db import models, transaction as db_transaction +from django.db import models import uuid -import copy class ScheduleC(models.Model): @@ -46,41 +44,6 @@ class ScheduleC(models.Model): lender_candidate_district = models.TextField(null=True, blank=True) memo_text_description = models.TextField(null=True, blank=True) - def save(self, *args, **kwargs): - with db_transaction.atomic(): - super().save(*args, **kwargs) - transaction = self.get_transaction() - if not transaction.memo_code: - report = transaction.report - future_reports = Report.objects.get_queryset().filter( - committee_account=report.committee_id, - upload_submission__isnull=True, - coverage_through_date__gte=report.coverage_through_date, - ) - transactions_to_update = ScheduleC.objects.filter( - transaction__transaction_id=transaction.transaction_id, - transaction__report_id__in=models.Subquery( - future_reports.values('id') - ) - ) - transactions_to_update.update(**transaction) - reports_ids_to_create_transaction_for = list(future_reports.exclude( - id__in=models.Subquery( - transactions_to_update.values('transaction__report_id') - ) - ).values_list('id', flat=True).distinct()) - - for report_id in reports_ids_to_create_transaction_for: - schedule_c_copy = copy.deepcopy(self) - schedule_c_copy.id = None - schedule_c_copy.save() - - transaction_copy = copy.deepcopy(transaction) - transaction_copy.id = None - transaction_copy.report_id = report_id - transaction_copy.schedule_c = schedule_c_copy - transaction_copy.save() - def get_date(self): return self.transaction.report.through_date diff --git a/django-backend/fecfiler/transactions/schedule_c/serializers.py b/django-backend/fecfiler/transactions/schedule_c/serializers.py index b6006f7a55..6c105a20c9 100644 --- a/django-backend/fecfiler/transactions/schedule_c/serializers.py +++ b/django-backend/fecfiler/transactions/schedule_c/serializers.py @@ -1,10 +1,14 @@ import logging -from django.db import transaction +from django.db import transaction, models +from django.db.models import Q from fecfiler.transactions.schedule_c.models import ScheduleC +from fecfiler.transactions.models import Transaction +from fecfiler.reports.models import Report from fecfiler.transactions.serializers import TransactionSerializerBase from fecfiler.shared.utilities import get_model_data from rest_framework.fields import DecimalField, CharField, DateField, BooleanField +import copy logger = logging.getLogger(__name__) @@ -34,6 +38,8 @@ def create(self, validated_data: dict): del transaction_data[key] schedule_c_transaction = super().create(transaction_data) + if not schedule_c_transaction.memo_code: + self.create_future_reports(schedule_c_transaction) return schedule_c_transaction def update(self, instance, validated_data: dict): @@ -42,7 +48,42 @@ def update(self, instance, validated_data: dict): if attr != "id": setattr(instance.schedule_c, attr, value) instance.schedule_c.save() - return super().update(instance, validated_data) + schedule_c_transaction = super().update(instance, validated_data) + self.update_future_reports(schedule_c_transaction) + return schedule_c_transaction + + def get_future_in_progress_reports(self, report: Report): + print(str(report.__dict__)) + return Report.objects.get_queryset().filter( + ~Q(id=report.id), + committee_account=report.committee_account_id, + upload_submission__isnull=True, + coverage_through_date__gte=report.coverage_through_date, + ) + + def create_future_reports(self, schedule_c_transaction: Transaction): + report = schedule_c_transaction.report + future_reports = self.get_future_in_progress_reports(report) + for report in future_reports: + schedule_c_copy = copy.deepcopy(schedule_c_transaction.schedule_c) + schedule_c_copy.id = None + schedule_c_copy.save() + schedule_c_transaction_copy = copy.deepcopy(schedule_c_transaction) + schedule_c_transaction_copy.id = None + schedule_c_transaction_copy.report = report + schedule_c_transaction_copy.schedule_c = schedule_c_copy + schedule_c_transaction_copy.save() + + def update_future_reports(self, schedule_c_transaction: Transaction): + report = schedule_c_transaction.report + future_reports = self.get_future_in_progress_reports(report) + transactions_to_update = Transaction.objects.filter( + transaction_id=schedule_c_transaction.transaction_id, + report_id__in=models.Subquery( + future_reports.values('id') + ) + ) + transactions_to_update.update(**schedule_c_transaction) receipt_line_number = CharField(required=False, allow_null=True) lender_organization_name = CharField(required=False, allow_null=True) From 2cc140af653628b1d1d17ae3dc746c5dbf05467f Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Tue, 24 Oct 2023 10:22:03 -0400 Subject: [PATCH 34/70] Adds balance to the sorting fields --- django-backend/fecfiler/transactions/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/django-backend/fecfiler/transactions/views.py b/django-backend/fecfiler/transactions/views.py index 8bc7056dea..6694d552c5 100644 --- a/django-backend/fecfiler/transactions/views.py +++ b/django-backend/fecfiler/transactions/views.py @@ -149,6 +149,7 @@ class TransactionViewSet(CommitteeOwnedViewSet, ReportViewMixin): "date", "amount", "aggregate", + "balance", "back_reference_tran_id_number", ] ordering = ["-created"] From d76316ca14a0ff86d5c546cf5b6958b82796157f Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Tue, 24 Oct 2023 11:22:45 -0400 Subject: [PATCH 35/70] Fix misspelling of so_candidate_middle_name in sched E --- .../test_election_aggregation_data.json | 6 +++--- ..._name_schedulee_so_candidate_middle_name.py | 18 ++++++++++++++++++ .../fecfiler/transactions/schedule_e/models.py | 2 +- .../transactions/schedule_e/serializers.py | 2 +- 4 files changed, 23 insertions(+), 5 deletions(-) create mode 100644 django-backend/fecfiler/transactions/migrations/0029_rename_so_candinate_middle_name_schedulee_so_candidate_middle_name.py diff --git a/django-backend/fecfiler/transactions/fixtures/test_election_aggregation_data.json b/django-backend/fecfiler/transactions/fixtures/test_election_aggregation_data.json index 1eb16eb9df..f434b830b7 100644 --- a/django-backend/fecfiler/transactions/fixtures/test_election_aggregation_data.json +++ b/django-backend/fecfiler/transactions/fixtures/test_election_aggregation_data.json @@ -194,7 +194,7 @@ "so_candidate_id_number": "S6AK00162", "so_candidate_last_name": "Testerson", "so_candidate_first_name": "Phil", - "so_candinate_middle_name": null, + "so_candidate_middle_name": null, "so_candidate_prefix": null, "so_candidate_suffix": null, "so_candidate_office": "S", @@ -237,7 +237,7 @@ "so_candidate_id_number": "S6AK00162", "so_candidate_last_name": "Testerson", "so_candidate_first_name": "Phil", - "so_candinate_middle_name": null, + "so_candidate_middle_name": null, "so_candidate_prefix": null, "so_candidate_suffix": null, "so_candidate_office": "S", @@ -280,7 +280,7 @@ "so_candidate_id_number": "S6AK00162", "so_candidate_last_name": "Testerson", "so_candidate_first_name": "Phil", - "so_candinate_middle_name": null, + "so_candidate_middle_name": null, "so_candidate_prefix": null, "so_candidate_suffix": null, "so_candidate_office": "S", diff --git a/django-backend/fecfiler/transactions/migrations/0029_rename_so_candinate_middle_name_schedulee_so_candidate_middle_name.py b/django-backend/fecfiler/transactions/migrations/0029_rename_so_candinate_middle_name_schedulee_so_candidate_middle_name.py new file mode 100644 index 0000000000..219b5d790a --- /dev/null +++ b/django-backend/fecfiler/transactions/migrations/0029_rename_so_candinate_middle_name_schedulee_so_candidate_middle_name.py @@ -0,0 +1,18 @@ +# Generated by Django 4.1.3 on 2023-10-24 15:17 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('transactions', '0028_alter_transaction_report'), + ] + + operations = [ + migrations.RenameField( + model_name='schedulee', + old_name='so_candinate_middle_name', + new_name='so_candidate_middle_name', + ), + ] diff --git a/django-backend/fecfiler/transactions/schedule_e/models.py b/django-backend/fecfiler/transactions/schedule_e/models.py index 011c5adfd0..1603d6eaaf 100644 --- a/django-backend/fecfiler/transactions/schedule_e/models.py +++ b/django-backend/fecfiler/transactions/schedule_e/models.py @@ -38,7 +38,7 @@ class ScheduleE(models.Model): so_candidate_id_number = models.TextField(null=True, blank=True) so_candidate_last_name = models.TextField(null=True, blank=True) so_candidate_first_name = models.TextField(null=True, blank=True) - so_candinate_middle_name = models.TextField(null=True, blank=True) + so_candidate_middle_name = models.TextField(null=True, blank=True) so_candidate_prefix = models.TextField(null=True, blank=True) so_candidate_suffix = models.TextField(null=True, blank=True) so_candidate_office = models.TextField(null=True, blank=True) diff --git a/django-backend/fecfiler/transactions/schedule_e/serializers.py b/django-backend/fecfiler/transactions/schedule_e/serializers.py index 024b0d557a..0ca89c5244 100644 --- a/django-backend/fecfiler/transactions/schedule_e/serializers.py +++ b/django-backend/fecfiler/transactions/schedule_e/serializers.py @@ -72,7 +72,7 @@ def update(self, instance, validated_data: dict): so_candidate_id_number = CharField(required=False, allow_null=True) so_candidate_last_name = CharField(required=False, allow_null=True) so_candidate_first_name = CharField(required=False, allow_null=True) - so_candinate_middle_name = CharField(required=False, allow_null=True) + so_candidate_middle_name = CharField(required=False, allow_null=True) so_candidate_prefix = CharField(required=False, allow_null=True) so_candidate_suffix = CharField(required=False, allow_null=True) so_candidate_office = CharField(required=False, allow_null=True) From 06737be973c6bc3fd3faab7c6b2f913b8176ccf9 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Tue, 24 Oct 2023 11:26:24 -0400 Subject: [PATCH 36/70] Updates validator commit hash --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 767ecdcd61..6753be18c0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ django-cors-headers==3.13.0 django-storages==1.13.1 djangorestframework==3.14.0 drf-spectacular==0.24.2 -git+https://github.com/fecgov/fecfile-validate@57fd1af27991bca418df51b43b2999fef7ef4119#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@4040143cb7c730a1278ea026c3de2d4ebb5112b5#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.30 gunicorn==20.1.0 Jinja2==3.1.2 From 868e7ca88f589e324644e5ea6dbfcfd8701976a3 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Tue, 24 Oct 2023 11:33:44 -0400 Subject: [PATCH 37/70] Adds a migration to change existing ttis --- ...eipt_from_person_to_receipt_from_entity.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 django-backend/fecfiler/transactions/migrations/0029_update_receipt_from_person_to_receipt_from_entity.py diff --git a/django-backend/fecfiler/transactions/migrations/0029_update_receipt_from_person_to_receipt_from_entity.py b/django-backend/fecfiler/transactions/migrations/0029_update_receipt_from_person_to_receipt_from_entity.py new file mode 100644 index 0000000000..3ea99c734f --- /dev/null +++ b/django-backend/fecfiler/transactions/migrations/0029_update_receipt_from_person_to_receipt_from_entity.py @@ -0,0 +1,42 @@ +# Generated by Django 4.1.3 on 2023-06-05 20:15 + +from django.db import migrations + + +ttis_to_change = { + "UNREGISTERED_RECEIPT_FROM_PERSON": "RECEIPT_FROM_UNREGISTERED_ENTITY", + "UNREGISTERED_RECEIPT_FROM_PERSON_RETURN": "RECEIPT_FROM_UNREGISTERED_ENTITY_RETURN", +} + + +def update_ttis(apps, schema_editor): # noqa + Transaction = apps.get_model("transactions", "Transaction") # noqa + for transaction in Transaction.objects.all(): + keys = list(ttis_to_change.keys()) + if transaction.transaction_type_identifier in keys: + new_tti = ttis_to_change[transaction.transaction_type_identifier] + transaction.transaction_type_identifier = new_tti + transaction.save() + + +def reverse_ttis(apps, schema_editor): # noqa + reverse_ttis_to_change = {} + for key in ttis_to_change: + value = ttis_to_change[key] + reverse_ttis_to_change[value] = key + + Transaction = apps.get_model("transactions", "Transaction") # noqa + for transaction in Transaction.objects.all(): + keys = list(reverse_ttis_to_change.keys()) + if transaction.transaction_type_identifier in keys: + new_tti = reverse_ttis_to_change[transaction.transaction_type_identifier] + transaction.transaction_type_identifier = new_tti + transaction.save() + + +class Migration(migrations.Migration): + dependencies = [ + ("transactions", "0028_alter_transaction_report"), + ] + + operations = [migrations.RunPython(update_ttis, reverse_code=reverse_ttis)] From 0156f50b3457124ea0a846ad8614f387284926cb Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Tue, 24 Oct 2023 13:05:23 -0400 Subject: [PATCH 38/70] added bulk create/update --- .../transactions/schedule_c/serializers.py | 45 ++++++++++++------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/django-backend/fecfiler/transactions/schedule_c/serializers.py b/django-backend/fecfiler/transactions/schedule_c/serializers.py index 6c105a20c9..b452e93a81 100644 --- a/django-backend/fecfiler/transactions/schedule_c/serializers.py +++ b/django-backend/fecfiler/transactions/schedule_c/serializers.py @@ -39,7 +39,7 @@ def create(self, validated_data: dict): schedule_c_transaction = super().create(transaction_data) if not schedule_c_transaction.memo_code: - self.create_future_reports(schedule_c_transaction) + self.create_in_future_reports(schedule_c_transaction) return schedule_c_transaction def update(self, instance, validated_data: dict): @@ -49,7 +49,8 @@ def update(self, instance, validated_data: dict): setattr(instance.schedule_c, attr, value) instance.schedule_c.save() schedule_c_transaction = super().update(instance, validated_data) - self.update_future_reports(schedule_c_transaction) + if not schedule_c_transaction.memo_code: + self.update_in_future_reports(schedule_c_transaction, validated_data) return schedule_c_transaction def get_future_in_progress_reports(self, report: Report): @@ -61,29 +62,43 @@ def get_future_in_progress_reports(self, report: Report): coverage_through_date__gte=report.coverage_through_date, ) - def create_future_reports(self, schedule_c_transaction: Transaction): - report = schedule_c_transaction.report + def create_in_future_reports(self, transaction: Transaction): + report = transaction.report future_reports = self.get_future_in_progress_reports(report) + schedule_c_copies_to_insert = [] + transaction_copies_to_insert = [] for report in future_reports: - schedule_c_copy = copy.deepcopy(schedule_c_transaction.schedule_c) + schedule_c_copy = copy.deepcopy(transaction.schedule_c) schedule_c_copy.id = None - schedule_c_copy.save() - schedule_c_transaction_copy = copy.deepcopy(schedule_c_transaction) - schedule_c_transaction_copy.id = None - schedule_c_transaction_copy.report = report - schedule_c_transaction_copy.schedule_c = schedule_c_copy - schedule_c_transaction_copy.save() + schedule_c_copies_to_insert.append(schedule_c_copy) + transaction_copy = copy.deepcopy(transaction) + transaction_copy.id = None + transaction_copy.report = report + transaction_copy.schedule_c = schedule_c_copy + transaction_copies_to_insert.append(transaction_copy) + ScheduleC.objects.bulk_create(schedule_c_copies_to_insert) + Transaction.objects.bulk_create(transaction_copies_to_insert) - def update_future_reports(self, schedule_c_transaction: Transaction): - report = schedule_c_transaction.report + def update_in_future_reports(self, transaction: Transaction, validated_data: dict): + report = transaction.report future_reports = self.get_future_in_progress_reports(report) transactions_to_update = Transaction.objects.filter( - transaction_id=schedule_c_transaction.transaction_id, + transaction_id=transaction.transaction_id, report_id__in=models.Subquery( future_reports.values('id') ) ) - transactions_to_update.update(**schedule_c_transaction) + schedule_cs_to_update = ScheduleC.objects.filter( + transaction__schedule_c_id__in=models.Subquery( + transactions_to_update.values('schedule_c_id') + ) + ) + schedule_c_data = get_model_data(validated_data, ScheduleC) + del schedule_c_data['id'] + schedule_cs_to_update.update(**schedule_c_data) + transaction_data = get_model_data(validated_data, Transaction) + del transaction_data['id'] + transactions_to_update.update(**transaction_data) receipt_line_number = CharField(required=False, allow_null=True) lender_organization_name = CharField(required=False, allow_null=True) From 313648e138c2e43f26994d0cff40c0d01f3f9fa3 Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Tue, 24 Oct 2023 17:03:24 -0400 Subject: [PATCH 39/70] 1405 finished dev --- django-backend/fecfiler/reports/models.py | 49 ++++++++++--------- .../transactions/schedule_c/serializers.py | 44 ++++++++++++----- 2 files changed, 57 insertions(+), 36 deletions(-) diff --git a/django-backend/fecfiler/reports/models.py b/django-backend/fecfiler/reports/models.py index 4ffc8733a2..edbe23e718 100644 --- a/django-backend/fecfiler/reports/models.py +++ b/django-backend/fecfiler/reports/models.py @@ -95,29 +95,32 @@ def pull_forward_loans(self): ) for loan in loans_to_pull_forward: - # Save children as they are lost from the loan object - # when the loan is saved - loan_children = copy.deepcopy(loan.children) - - loan.schedule_c_id = self.save_copy(loan.schedule_c) - loan.memo_text_id = self.save_copy(loan.memo_text) - loan.report_id = self.id - loan.report = self - # The loan_id should point to the original loan transaction - # even if the loan is pulled forward multiple times. - loan.loan_id = loan.loan_id if loan.loan_id else loan.id - self.save_copy(loan) - for child in loan_children: - # If child is a guarantor transaction, copy it - # and link it to the new loan - if child.schedule_c2_id: - child.schedule_c2_id = self.save_copy(child.schedule_c2) - child.memo_text_id = self.save_copy(child.memo_text) - child.report_id = self.id - child.report = self - child.parent_transaction_id = loan.id - child.parent_transaction = loan - self.save_copy(child) + self.pull_forward_loan(loan) + + def pull_forward_loan(self, loan): + # Save children as they are lost from the loan object + # when the loan is saved + loan_children = copy.deepcopy(loan.children) + + loan.schedule_c_id = self.save_copy(loan.schedule_c) + loan.memo_text_id = self.save_copy(loan.memo_text) + loan.report_id = self.id + loan.report = self + # The loan_id should point to the original loan transaction + # even if the loan is pulled forward multiple times. + loan.loan_id = loan.loan_id if loan.loan_id else loan.id + self.save_copy(loan) + for child in loan_children: + # If child is a guarantor transaction, copy it + # and link it to the new loan + if child.schedule_c2_id: + child.schedule_c2_id = self.save_copy(child.schedule_c2) + child.memo_text_id = self.save_copy(child.memo_text) + child.report_id = self.id + child.report = self + child.parent_transaction_id = loan.id + child.parent_transaction = loan + self.save_copy(child) def pull_forward_debts(self): previous_report = self.get_previous_report() diff --git a/django-backend/fecfiler/transactions/schedule_c/serializers.py b/django-backend/fecfiler/transactions/schedule_c/serializers.py index b452e93a81..8768233720 100644 --- a/django-backend/fecfiler/transactions/schedule_c/serializers.py +++ b/django-backend/fecfiler/transactions/schedule_c/serializers.py @@ -3,7 +3,9 @@ from django.db import transaction, models from django.db.models import Q from fecfiler.transactions.schedule_c.models import ScheduleC +from fecfiler.transactions.schedule_c2.models import ScheduleC2 from fecfiler.transactions.models import Transaction +from fecfiler.memo_text.models import MemoText from fecfiler.reports.models import Report from fecfiler.transactions.serializers import TransactionSerializerBase from fecfiler.shared.utilities import get_model_data @@ -63,21 +65,11 @@ def get_future_in_progress_reports(self, report: Report): ) def create_in_future_reports(self, transaction: Transaction): - report = transaction.report + report: Report = transaction.report future_reports = self.get_future_in_progress_reports(report) - schedule_c_copies_to_insert = [] - transaction_copies_to_insert = [] + transaction_copy = copy.deepcopy(transaction) for report in future_reports: - schedule_c_copy = copy.deepcopy(transaction.schedule_c) - schedule_c_copy.id = None - schedule_c_copies_to_insert.append(schedule_c_copy) - transaction_copy = copy.deepcopy(transaction) - transaction_copy.id = None - transaction_copy.report = report - transaction_copy.schedule_c = schedule_c_copy - transaction_copies_to_insert.append(transaction_copy) - ScheduleC.objects.bulk_create(schedule_c_copies_to_insert) - Transaction.objects.bulk_create(transaction_copies_to_insert) + report.pull_forward_loan(transaction_copy) def update_in_future_reports(self, transaction: Transaction, validated_data: dict): report = transaction.report @@ -93,12 +85,38 @@ def update_in_future_reports(self, transaction: Transaction, validated_data: dic transactions_to_update.values('schedule_c_id') ) ) + memo_text_to_update = MemoText.objects.filter( + transaction__memo_text_id__in=models.Subquery( + transactions_to_update.values('memo_text_id') + ) + ) schedule_c_data = get_model_data(validated_data, ScheduleC) del schedule_c_data['id'] schedule_cs_to_update.update(**schedule_c_data) transaction_data = get_model_data(validated_data, Transaction) del transaction_data['id'] transactions_to_update.update(**transaction_data) + memo_text_data = get_model_data(validated_data, MemoText) + del memo_text_data['id'] + memo_text_to_update.update(**memo_text_data) + + loan_children = validated_data.pop("children", []) + for child in loan_children: + if child.schedule_c2_id: + schedule_c2_data = get_model_data(child, ScheduleC2) + ScheduleC2.objects.filter( + id=child.schedule_c2_id, + transaction__parent_transaction_id__in=models.Subquery( + transactions_to_update.values('id') + ) + ).update(**schedule_c2_data) + child_memo_text_data = get_model_data(child, MemoText) + MemoText.objects.filter( + id=child.memo_text_id, + transaction__parent_transaction_id__in=models.Subquery( + transactions_to_update.values('id') + ) + ).update(**child_memo_text_data) receipt_line_number = CharField(required=False, allow_null=True) lender_organization_name = CharField(required=False, allow_null=True) From f1d0731af6b780ee6a46e4133c288e315c3e0f53 Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Wed, 25 Oct 2023 14:13:26 -0400 Subject: [PATCH 40/70] 1405 create working --- django-backend/fecfiler/reports/models.py | 17 ++++++++++------- .../transactions/schedule_c/serializers.py | 15 ++------------- .../transactions/schedule_c2/serializers.py | 19 ++++++++++++++++++- .../fecfiler/transactions/serializers.py | 9 +++++++++ 4 files changed, 39 insertions(+), 21 deletions(-) diff --git a/django-backend/fecfiler/reports/models.py b/django-backend/fecfiler/reports/models.py index edbe23e718..48f872e865 100644 --- a/django-backend/fecfiler/reports/models.py +++ b/django-backend/fecfiler/reports/models.py @@ -114,13 +114,16 @@ def pull_forward_loan(self, loan): # If child is a guarantor transaction, copy it # and link it to the new loan if child.schedule_c2_id: - child.schedule_c2_id = self.save_copy(child.schedule_c2) - child.memo_text_id = self.save_copy(child.memo_text) - child.report_id = self.id - child.report = self - child.parent_transaction_id = loan.id - child.parent_transaction = loan - self.save_copy(child) + self.pull_forward_loan_guarantor(child, loan) + + def pull_forward_loan_guarantor(self, loan_guarantor, loan): + loan_guarantor.schedule_c2_id = self.save_copy(loan_guarantor.schedule_c2) + loan_guarantor.memo_text_id = self.save_copy(loan_guarantor.memo_text) + loan_guarantor.report_id = self.id + loan_guarantor.report = self + loan_guarantor.parent_transaction_id = loan.id + loan_guarantor.parent_transaction = loan + self.save_copy(loan_guarantor) def pull_forward_debts(self): previous_report = self.get_previous_report() diff --git a/django-backend/fecfiler/transactions/schedule_c/serializers.py b/django-backend/fecfiler/transactions/schedule_c/serializers.py index 8768233720..5b3a50d334 100644 --- a/django-backend/fecfiler/transactions/schedule_c/serializers.py +++ b/django-backend/fecfiler/transactions/schedule_c/serializers.py @@ -55,25 +55,14 @@ def update(self, instance, validated_data: dict): self.update_in_future_reports(schedule_c_transaction, validated_data) return schedule_c_transaction - def get_future_in_progress_reports(self, report: Report): - print(str(report.__dict__)) - return Report.objects.get_queryset().filter( - ~Q(id=report.id), - committee_account=report.committee_account_id, - upload_submission__isnull=True, - coverage_through_date__gte=report.coverage_through_date, - ) - def create_in_future_reports(self, transaction: Transaction): - report: Report = transaction.report - future_reports = self.get_future_in_progress_reports(report) + future_reports = super().get_future_in_progress_reports(transaction.report) transaction_copy = copy.deepcopy(transaction) for report in future_reports: report.pull_forward_loan(transaction_copy) def update_in_future_reports(self, transaction: Transaction, validated_data: dict): - report = transaction.report - future_reports = self.get_future_in_progress_reports(report) + future_reports = super().get_future_in_progress_reports(transaction.report) transactions_to_update = Transaction.objects.filter( transaction_id=transaction.transaction_id, report_id__in=models.Subquery( diff --git a/django-backend/fecfiler/transactions/schedule_c2/serializers.py b/django-backend/fecfiler/transactions/schedule_c2/serializers.py index 2044cb7404..dfc97688ac 100644 --- a/django-backend/fecfiler/transactions/schedule_c2/serializers.py +++ b/django-backend/fecfiler/transactions/schedule_c2/serializers.py @@ -1,10 +1,12 @@ import logging from django.db import transaction +from fecfiler.transactions.models import Transaction from fecfiler.transactions.schedule_c2.models import ScheduleC2 from fecfiler.transactions.serializers import TransactionSerializerBase from fecfiler.shared.utilities import get_model_data from rest_framework.fields import DecimalField, CharField +import copy logger = logging.getLogger(__name__) @@ -36,6 +38,8 @@ def create(self, validated_data: dict): del transaction_data[key] schedule_c2_transaction = super().create(transaction_data) + if not schedule_c2_transaction.parent_transaction.memo_code: + self.create_in_future_reports(schedule_c2_transaction) return schedule_c2_transaction def update(self, instance, validated_data: dict): @@ -44,7 +48,20 @@ def update(self, instance, validated_data: dict): if attr != "id": setattr(instance.schedule_c2, attr, value) instance.schedule_c2.save() - return super().update(instance, validated_data) + schedule_c2_transaction = super().update(instance, validated_data) + if not schedule_c2_transaction.parent_transaction.memo_code: + self.update_in_future_reports(schedule_c2_transaction, validated_data) + return schedule_c2_transaction + + def create_in_future_reports(self, transaction: Transaction): + future_reports = self.get_future_in_progress_reports(transaction.report) + transaction_copy = copy.deepcopy(transaction) + for report in future_reports: + loan = Transaction.objects.get( + report_id=report.id, + loan_id=transaction_copy.parent_transaction.id + ) + report.pull_forward_loan_guarantor(transaction_copy, loan) guarantor_last_name = CharField(required=False, allow_null=True) guarantor_first_name = CharField(required=False, allow_null=True) diff --git a/django-backend/fecfiler/transactions/serializers.py b/django-backend/fecfiler/transactions/serializers.py index a740e73bb9..1a406cfa31 100644 --- a/django-backend/fecfiler/transactions/serializers.py +++ b/django-backend/fecfiler/transactions/serializers.py @@ -15,6 +15,7 @@ ModelSerializer, DecimalField, ) +from fecfiler.reports.models import Report from fecfiler.contacts.models import Contact from fecfiler.transactions.models import Transaction from fecfiler.transactions.schedule_a.models import ScheduleA @@ -249,6 +250,14 @@ def to_internal_value(self, data): internal_value["_form_type"] = internal_value["form_type"] return internal_value + def get_future_in_progress_reports(self, report: Report): + return Report.objects.get_queryset().filter( + ~Q(id=report.id), + committee_account=report.committee_account_id, + upload_submission__isnull=True, + coverage_through_date__gte=report.coverage_through_date, + ) + def propagate_contacts(self, transaction): contact_1 = Contact.objects.get(id=transaction.contact_1_id) self.propagate_contact(transaction, contact_1) From 5bf173235ca2ca896ea1dffb13d11e570625640f Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Wed, 25 Oct 2023 14:48:08 -0400 Subject: [PATCH 41/70] Update validate commit hash --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6753be18c0..dc81906f80 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ django-cors-headers==3.13.0 django-storages==1.13.1 djangorestframework==3.14.0 drf-spectacular==0.24.2 -git+https://github.com/fecgov/fecfile-validate@4040143cb7c730a1278ea026c3de2d4ebb5112b5#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@59c6e2e7c24e63839765f2fc134fc46d7bbc0fe3#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.30 gunicorn==20.1.0 Jinja2==3.1.2 From aa0abb49d9a9394f96b5c39440067b6d86915149 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Wed, 25 Oct 2023 14:55:22 -0400 Subject: [PATCH 42/70] Replaced old transaction type id with new one in schedule_a managers --- django-backend/fecfiler/transactions/schedule_a/managers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django-backend/fecfiler/transactions/schedule_a/managers.py b/django-backend/fecfiler/transactions/schedule_a/managers.py index e7487b49ad..b6d12af541 100644 --- a/django-backend/fecfiler/transactions/schedule_a/managers.py +++ b/django-backend/fecfiler/transactions/schedule_a/managers.py @@ -40,7 +40,7 @@ "INDIVIDUAL_NATIONAL_PARTY_CONVENTION_ACCOUNT", "PARTY_NATIONAL_PARTY_CONVENTION_ACCOUNT", "TRIBAL_NATIONAL_PARTY_RECOUNT_ACCOUNT", - "UNREGISTERED_RECEIPT_FROM_PERSON", + "RECEIPT_FROM_UNREGISTERED_ENTITY", "PARTNERSHIP_RECEIPT", "PARTNERSHIP_NATIONAL_PARTY_CONVENTION_JF_TRANSFER_MEMO", "PARTNERSHIP_ATTRIBUTION_NATIONAL_PARTY_CONVENTION_JF_TRANSFER_MEMO", From 5a8bd9214058f9ce925a1b634b2e2baa54c36e46 Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Wed, 25 Oct 2023 16:25:57 -0400 Subject: [PATCH 43/70] 1405 update c2 --- .../transactions/schedule_c/serializers.py | 40 +++++-------------- .../transactions/schedule_c2/serializers.py | 34 +++++++++++++++- 2 files changed, 44 insertions(+), 30 deletions(-) diff --git a/django-backend/fecfiler/transactions/schedule_c/serializers.py b/django-backend/fecfiler/transactions/schedule_c/serializers.py index 5b3a50d334..d1073bc22f 100644 --- a/django-backend/fecfiler/transactions/schedule_c/serializers.py +++ b/django-backend/fecfiler/transactions/schedule_c/serializers.py @@ -1,12 +1,9 @@ import logging from django.db import transaction, models -from django.db.models import Q from fecfiler.transactions.schedule_c.models import ScheduleC -from fecfiler.transactions.schedule_c2.models import ScheduleC2 from fecfiler.transactions.models import Transaction from fecfiler.memo_text.models import MemoText -from fecfiler.reports.models import Report from fecfiler.transactions.serializers import TransactionSerializerBase from fecfiler.shared.utilities import get_model_data from rest_framework.fields import DecimalField, CharField, DateField, BooleanField @@ -63,50 +60,35 @@ def create_in_future_reports(self, transaction: Transaction): def update_in_future_reports(self, transaction: Transaction, validated_data: dict): future_reports = super().get_future_in_progress_reports(transaction.report) + + transaction_data = get_model_data(validated_data, Transaction) + del transaction_data['id'] transactions_to_update = Transaction.objects.filter( transaction_id=transaction.transaction_id, report_id__in=models.Subquery( future_reports.values('id') ) ) + transactions_to_update.update(**transaction_data) + + schedule_c_data = get_model_data(validated_data, ScheduleC) + del schedule_c_data['id'] schedule_cs_to_update = ScheduleC.objects.filter( transaction__schedule_c_id__in=models.Subquery( transactions_to_update.values('schedule_c_id') ) ) + schedule_cs_to_update.update(**schedule_c_data) + + memo_text_data = get_model_data(validated_data, MemoText) + del memo_text_data['id'] memo_text_to_update = MemoText.objects.filter( transaction__memo_text_id__in=models.Subquery( transactions_to_update.values('memo_text_id') ) ) - schedule_c_data = get_model_data(validated_data, ScheduleC) - del schedule_c_data['id'] - schedule_cs_to_update.update(**schedule_c_data) - transaction_data = get_model_data(validated_data, Transaction) - del transaction_data['id'] - transactions_to_update.update(**transaction_data) - memo_text_data = get_model_data(validated_data, MemoText) - del memo_text_data['id'] memo_text_to_update.update(**memo_text_data) - loan_children = validated_data.pop("children", []) - for child in loan_children: - if child.schedule_c2_id: - schedule_c2_data = get_model_data(child, ScheduleC2) - ScheduleC2.objects.filter( - id=child.schedule_c2_id, - transaction__parent_transaction_id__in=models.Subquery( - transactions_to_update.values('id') - ) - ).update(**schedule_c2_data) - child_memo_text_data = get_model_data(child, MemoText) - MemoText.objects.filter( - id=child.memo_text_id, - transaction__parent_transaction_id__in=models.Subquery( - transactions_to_update.values('id') - ) - ).update(**child_memo_text_data) - receipt_line_number = CharField(required=False, allow_null=True) lender_organization_name = CharField(required=False, allow_null=True) lender_last_name = CharField(required=False, allow_null=True) diff --git a/django-backend/fecfiler/transactions/schedule_c2/serializers.py b/django-backend/fecfiler/transactions/schedule_c2/serializers.py index dfc97688ac..6f06d992d7 100644 --- a/django-backend/fecfiler/transactions/schedule_c2/serializers.py +++ b/django-backend/fecfiler/transactions/schedule_c2/serializers.py @@ -1,9 +1,10 @@ import logging -from django.db import transaction +from django.db import transaction, models from fecfiler.transactions.models import Transaction from fecfiler.transactions.schedule_c2.models import ScheduleC2 from fecfiler.transactions.serializers import TransactionSerializerBase +from fecfiler.memo_text.models import MemoText from fecfiler.shared.utilities import get_model_data from rest_framework.fields import DecimalField, CharField import copy @@ -63,6 +64,37 @@ def create_in_future_reports(self, transaction: Transaction): ) report.pull_forward_loan_guarantor(transaction_copy, loan) + def update_in_future_reports(self, transaction: Transaction, validated_data: dict): + future_reports = super().get_future_in_progress_reports(transaction.report) + + transaction_data = get_model_data(validated_data, Transaction) + del transaction_data['id'] + transactions_to_update = Transaction.objects.filter( + transaction_id=transaction.transaction_id, + report_id__in=models.Subquery( + future_reports.values('id') + ) + ) + transactions_to_update.update(**transaction_data) + + schedule_c2_data = get_model_data(validated_data, ScheduleC2) + del schedule_c2_data['id'] + schedule_c2s_to_update = ScheduleC2.objects.filter( + transaction__schedule_c2_id__in=models.Subquery( + transactions_to_update.values('schedule_c2_id') + ) + ) + schedule_c2s_to_update.update(**schedule_c2_data) + + memo_text_data = get_model_data(validated_data, MemoText) + del memo_text_data['id'] + memo_text_to_update = MemoText.objects.filter( + transaction__memo_text_id__in=models.Subquery( + transactions_to_update.values('memo_text_id') + ) + ) + memo_text_to_update.update(**memo_text_data) + guarantor_last_name = CharField(required=False, allow_null=True) guarantor_first_name = CharField(required=False, allow_null=True) guarantor_middle_name = CharField(required=False, allow_null=True) From 59845e7ac7b3d3a94bdbba3f2205fa958571bd72 Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Thu, 26 Oct 2023 01:00:00 -0400 Subject: [PATCH 44/70] 1405 fix memo text updates --- .../transactions/schedule_c/serializers.py | 31 +++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/django-backend/fecfiler/transactions/schedule_c/serializers.py b/django-backend/fecfiler/transactions/schedule_c/serializers.py index d1073bc22f..0588fe3c2a 100644 --- a/django-backend/fecfiler/transactions/schedule_c/serializers.py +++ b/django-backend/fecfiler/transactions/schedule_c/serializers.py @@ -60,7 +60,7 @@ def create_in_future_reports(self, transaction: Transaction): def update_in_future_reports(self, transaction: Transaction, validated_data: dict): future_reports = super().get_future_in_progress_reports(transaction.report) - + transaction_data = get_model_data(validated_data, Transaction) del transaction_data['id'] transactions_to_update = Transaction.objects.filter( @@ -80,14 +80,27 @@ def update_in_future_reports(self, transaction: Transaction, validated_data: dic ) schedule_cs_to_update.update(**schedule_c_data) - memo_text_data = get_model_data(validated_data, MemoText) - del memo_text_data['id'] - memo_text_to_update = MemoText.objects.filter( - transaction__memo_text_id__in=models.Subquery( - transactions_to_update.values('memo_text_id') - ) - ) - memo_text_to_update.update(**memo_text_data) + memo_text_id = validated_data.get("memo_text_id") + if not memo_text_id: + transactions_to_update.update(**{"memo_text_id": None}) + else: + MemoText.objects.filter( + transaction__memo_text_id__in=models.Subquery( + transactions_to_update.values('memo_text_id') + ) + ).update(**{"text4000": transaction.memo_text.text4000}) + + for transaction_to_update in transactions_to_update.filter(memo_text_id__isnull=True).all(): + new_memo_text = MemoText.objects.create({ + "rec_type": "TEXT", + "transaction_id_number": transaction_to_update.transaction_id, + "text4000": transaction.memo_text.text4000, + "report_id": transaction.report.id, + "committee_account_id": transaction_to_update.committee_account.id, + "transaction_uuid": transaction_to_update.id, + "is_report_level_memo": False, + }) + transaction_to_update.update(**{"memo_data": new_memo_text}) receipt_line_number = CharField(required=False, allow_null=True) lender_organization_name = CharField(required=False, allow_null=True) From 93a9934d294dd5ee2237fdf712a162957028e661 Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Thu, 26 Oct 2023 10:57:42 -0400 Subject: [PATCH 45/70] 1405 local testing fixes --- .../fecfiler/transactions/schedule_c/serializers.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/django-backend/fecfiler/transactions/schedule_c/serializers.py b/django-backend/fecfiler/transactions/schedule_c/serializers.py index 0588fe3c2a..5761cd1836 100644 --- a/django-backend/fecfiler/transactions/schedule_c/serializers.py +++ b/django-backend/fecfiler/transactions/schedule_c/serializers.py @@ -82,6 +82,12 @@ def update_in_future_reports(self, transaction: Transaction, validated_data: dic memo_text_id = validated_data.get("memo_text_id") if not memo_text_id: + for memo_text in MemoText.objects.filter( + transaction__memo_text_id__in=models.Subquery( + transactions_to_update.values('memo_text_id') + ) + ): + memo_text.delete() transactions_to_update.update(**{"memo_text_id": None}) else: MemoText.objects.filter( @@ -90,8 +96,8 @@ def update_in_future_reports(self, transaction: Transaction, validated_data: dic ) ).update(**{"text4000": transaction.memo_text.text4000}) - for transaction_to_update in transactions_to_update.filter(memo_text_id__isnull=True).all(): - new_memo_text = MemoText.objects.create({ + for transaction_to_update in transactions_to_update.filter(memo_text_id__isnull=True): + new_memo_text = MemoText.objects.create(**{ "rec_type": "TEXT", "transaction_id_number": transaction_to_update.transaction_id, "text4000": transaction.memo_text.text4000, @@ -100,7 +106,7 @@ def update_in_future_reports(self, transaction: Transaction, validated_data: dic "transaction_uuid": transaction_to_update.id, "is_report_level_memo": False, }) - transaction_to_update.update(**{"memo_data": new_memo_text}) + Transaction.objects.filter(id=transaction_to_update.id).update(**{"memo_text": new_memo_text}) receipt_line_number = CharField(required=False, allow_null=True) lender_organization_name = CharField(required=False, allow_null=True) From 781195b34aee5d99406c5580bd16fe74db586c7f Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Thu, 26 Oct 2023 11:50:37 -0400 Subject: [PATCH 46/70] 1405 lint --- django-backend/fecfiler/reports/models.py | 2 +- .../fecfiler/transactions/schedule_c/serializers.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/django-backend/fecfiler/reports/models.py b/django-backend/fecfiler/reports/models.py index 48f872e865..59998a9d38 100644 --- a/django-backend/fecfiler/reports/models.py +++ b/django-backend/fecfiler/reports/models.py @@ -115,7 +115,7 @@ def pull_forward_loan(self, loan): # and link it to the new loan if child.schedule_c2_id: self.pull_forward_loan_guarantor(child, loan) - + def pull_forward_loan_guarantor(self, loan_guarantor, loan): loan_guarantor.schedule_c2_id = self.save_copy(loan_guarantor.schedule_c2) loan_guarantor.memo_text_id = self.save_copy(loan_guarantor.memo_text) diff --git a/django-backend/fecfiler/transactions/schedule_c/serializers.py b/django-backend/fecfiler/transactions/schedule_c/serializers.py index 5761cd1836..47a9037474 100644 --- a/django-backend/fecfiler/transactions/schedule_c/serializers.py +++ b/django-backend/fecfiler/transactions/schedule_c/serializers.py @@ -96,7 +96,9 @@ def update_in_future_reports(self, transaction: Transaction, validated_data: dic ) ).update(**{"text4000": transaction.memo_text.text4000}) - for transaction_to_update in transactions_to_update.filter(memo_text_id__isnull=True): + for transaction_to_update in transactions_to_update.filter( + memo_text_id__isnull=True + ): new_memo_text = MemoText.objects.create(**{ "rec_type": "TEXT", "transaction_id_number": transaction_to_update.transaction_id, @@ -106,7 +108,9 @@ def update_in_future_reports(self, transaction: Transaction, validated_data: dic "transaction_uuid": transaction_to_update.id, "is_report_level_memo": False, }) - Transaction.objects.filter(id=transaction_to_update.id).update(**{"memo_text": new_memo_text}) + Transaction.objects.filter(id=transaction_to_update.id).update( + **{"memo_text": new_memo_text} + ) receipt_line_number = CharField(required=False, allow_null=True) lender_organization_name = CharField(required=False, allow_null=True) From fe6676196242e2304539ea6a803ef9ff3d3ccce8 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Thu, 26 Oct 2023 15:16:05 -0400 Subject: [PATCH 47/70] Refactor calendar_ytd annotation to match schema property name --- .../test_election_aggregation_data.json | 3 -- .../fecfiler/transactions/managers.py | 51 +++++++++---------- .../0031_remove_schedulee_calendar_ytd.py | 17 +++++++ .../transactions/schedule_e/managers.py | 1 + .../transactions/schedule_e/models.py | 3 -- .../transactions/schedule_e/serializers.py | 2 +- .../schedule_e/tests/test_serializers.py | 2 +- .../fecfiler/transactions/serializers.py | 6 ++- .../transactions/tests/test_manager.py | 4 +- .../dot_fec/schema_fields/SchE.json | 4 +- requirements.txt | 2 +- 11 files changed, 54 insertions(+), 41 deletions(-) create mode 100644 django-backend/fecfiler/transactions/migrations/0031_remove_schedulee_calendar_ytd.py diff --git a/django-backend/fecfiler/transactions/fixtures/test_election_aggregation_data.json b/django-backend/fecfiler/transactions/fixtures/test_election_aggregation_data.json index f434b830b7..e6a31a6688 100644 --- a/django-backend/fecfiler/transactions/fixtures/test_election_aggregation_data.json +++ b/django-backend/fecfiler/transactions/fixtures/test_election_aggregation_data.json @@ -186,7 +186,6 @@ "dissemination_date": null, "expenditure_amount": "42.00", "disbursement_date": "2023-10-19", - "calendar_ytd": "42.00", "expenditure_purpose_descrip": "Reimbursement: See Below", "category_code": null, "payee_cmtte_fec_id_number": null, @@ -229,7 +228,6 @@ "dissemination_date": null, "expenditure_amount": "520.00", "disbursement_date": "2023-11-16", - "calendar_ytd": "520.00", "expenditure_purpose_descrip": "yes", "category_code": null, "payee_cmtte_fec_id_number": null, @@ -272,7 +270,6 @@ "dissemination_date": "2023-10-16", "expenditure_amount": "16.00", "disbursement_date": "2023-10-31", - "calendar_ytd": "16.00", "expenditure_purpose_descrip": "nooo", "category_code": null, "payee_cmtte_fec_id_number": null, diff --git a/django-backend/fecfiler/transactions/managers.py b/django-backend/fecfiler/transactions/managers.py index a0594b294c..eb5059b5f1 100644 --- a/django-backend/fecfiler/transactions/managers.py +++ b/django-backend/fecfiler/transactions/managers.py @@ -21,7 +21,7 @@ BooleanField, TextField, DecimalField, - ExpressionWrapper + ExpressionWrapper, ) from decimal import Decimal from enum import Enum @@ -51,7 +51,7 @@ def get_queryset(self): "schedule_b__expenditure_date", "schedule_c__loan_incurred_date", "schedule_e__disbursement_date", - "schedule_e__dissemination_date" + "schedule_e__dissemination_date", ), amount=Coalesce( "schedule_a__contribution_amount", @@ -68,31 +68,31 @@ def get_queryset(self): primary_contact_clause = Q(contact_1_id=OuterRef("contact_1_id")) election_clause = ( - Q( - schedule_e__isnull=False - ) & Q( - schedule_e__election_code=OuterRef( - "schedule_e__election_code" - ) - ) & Q( + Q(schedule_e__isnull=False) + & Q(schedule_e__election_code=OuterRef("schedule_e__election_code")) + & Q( schedule_e__so_candidate_office=OuterRef( "schedule_e__so_candidate_office" ) - ) & Q( + ) + & Q( Q( schedule_e__so_candidate_state=OuterRef( "schedule_e__so_candidate_state" ) - ) | ( + ) + | ( Q(schedule_e__so_candidate_state__isnull=True) & Q(outer_candidate_state__isnull=True) ) - ) & ( + ) + & ( Q( schedule_e__so_candidate_district=OuterRef( "schedule_e__so_candidate_district" ) - ) | ( + ) + | ( Q(schedule_e__so_candidate_district__isnull=True) & Q(outer_candidate_district__isnull=True) ) @@ -117,25 +117,20 @@ def get_queryset(self): .annotate(aggregate=Sum("effective_amount")) .values("aggregate") ) - calendar_ytd_clause = ( + calendar_ytd_per_election_office_clause = ( queryset.alias( # Needed to get around null-matching bug with Q() outer_candidate_district=ExpressionWrapper( - OuterRef('schedule_e__so_candidate_district'), - output_field=TextField() + OuterRef("schedule_e__so_candidate_district"), + output_field=TextField(), ), outer_candidate_state=ExpressionWrapper( - OuterRef('schedule_e__so_candidate_state'), - output_field=TextField() - ) - ).filter( - election_clause, - year_clause, - date_clause, - group_clause, + OuterRef("schedule_e__so_candidate_state"), output_field=TextField() + ), ) + .filter(election_clause, year_clause, date_clause, group_clause,) .values("committee_account_id") - .annotate(calendar_ytd=Sum("effective_amount")) - .values("calendar_ytd") + .annotate(calendar_ytd_per_election_office=Sum("effective_amount")) + .values("calendar_ytd_per_election_office") ) loan_payment_to_date_clause = ( @@ -186,7 +181,9 @@ def get_queryset(self): return ( queryset.annotate( aggregate=Coalesce(Subquery(aggregate_clause), Value(Decimal(0))), - calendar_ytd=Coalesce(Subquery(calendar_ytd_clause), Value(Decimal(0))), + calendar_ytd_per_election_office=Coalesce( + Subquery(calendar_ytd_per_election_office_clause), Value(Decimal(0)) + ), loan_payment_to_date=Case( When( schedule_c__isnull=False, diff --git a/django-backend/fecfiler/transactions/migrations/0031_remove_schedulee_calendar_ytd.py b/django-backend/fecfiler/transactions/migrations/0031_remove_schedulee_calendar_ytd.py new file mode 100644 index 0000000000..d2fd0c4bb8 --- /dev/null +++ b/django-backend/fecfiler/transactions/migrations/0031_remove_schedulee_calendar_ytd.py @@ -0,0 +1,17 @@ +# Generated by Django 4.1.3 on 2023-10-26 19:14 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('transactions', '0030_update_receipt_from_person_to_receipt_from_entity'), + ] + + operations = [ + migrations.RemoveField( + model_name='schedulee', + name='calendar_ytd', + ), + ] diff --git a/django-backend/fecfiler/transactions/schedule_e/managers.py b/django-backend/fecfiler/transactions/schedule_e/managers.py index 9ee47be00a..96e7d70b79 100644 --- a/django-backend/fecfiler/transactions/schedule_e/managers.py +++ b/django-backend/fecfiler/transactions/schedule_e/managers.py @@ -7,4 +7,5 @@ "INDEPENDENT_EXPENDITURE_CREDIT_CARD_PAYMENT_MEMO", "INDEPENDENT_EXPENDITURE_PAYMENT_TO_PAYROLL_MEMO", "INDEPENDENT_EXPENDITURE_STAFF_REIMBURSEMENT_MEMO", + "MULTISTATE_INDPENDENT_EXPENDITURE", ] diff --git a/django-backend/fecfiler/transactions/schedule_e/models.py b/django-backend/fecfiler/transactions/schedule_e/models.py index 1603d6eaaf..fb2bc08e9f 100644 --- a/django-backend/fecfiler/transactions/schedule_e/models.py +++ b/django-backend/fecfiler/transactions/schedule_e/models.py @@ -28,9 +28,6 @@ class ScheduleE(models.Model): null=True, blank=True, max_digits=11, decimal_places=2 ) disbursement_date = models.DateField(null=True, blank=True) - calendar_ytd = models.DecimalField( - null=True, blank=True, max_digits=11, decimal_places=2 - ) expenditure_purpose_descrip = models.TextField(null=True, blank=True) category_code = models.TextField(null=True, blank=True) payee_cmtte_fec_id_number = models.TextField(null=True, blank=True) diff --git a/django-backend/fecfiler/transactions/schedule_e/serializers.py b/django-backend/fecfiler/transactions/schedule_e/serializers.py index 0ca89c5244..55131d8d20 100644 --- a/django-backend/fecfiler/transactions/schedule_e/serializers.py +++ b/django-backend/fecfiler/transactions/schedule_e/serializers.py @@ -62,7 +62,7 @@ def update(self, instance, validated_data: dict): required=False, allow_null=True, max_digits=11, decimal_places=2 ) disbursement_date = DateField(required=False, allow_null=True) - calendar_ytd = DecimalField( + calendar_ytd_per_election_office = DecimalField( required=False, allow_null=True, max_digits=11, decimal_places=2 ) expenditure_purpose_descrip = CharField(required=False, allow_null=True) diff --git a/django-backend/fecfiler/transactions/schedule_e/tests/test_serializers.py b/django-backend/fecfiler/transactions/schedule_e/tests/test_serializers.py index 613c26ea7f..768c1ccdec 100644 --- a/django-backend/fecfiler/transactions/schedule_e/tests/test_serializers.py +++ b/django-backend/fecfiler/transactions/schedule_e/tests/test_serializers.py @@ -47,7 +47,7 @@ def setUp(self): "dissemination_date": "2022-02-09", "expenditure_amount": 1500, "disbursement_date": "2022-02-09", - "calendar_ytd": 3000, + "calendar_ytd_per_election_office": 3000, "so_candidate_last_name": "Testerson", "so_candidate_first_name": "William", "completing_last_name": "Testerson", diff --git a/django-backend/fecfiler/transactions/serializers.py b/django-backend/fecfiler/transactions/serializers.py index 28fa582874..bcc5e1e342 100644 --- a/django-backend/fecfiler/transactions/serializers.py +++ b/django-backend/fecfiler/transactions/serializers.py @@ -129,7 +129,9 @@ class TransactionSerializerBase( date = DateField(read_only=True) amount = DecimalField(max_digits=11, decimal_places=2, read_only=True) aggregate = DecimalField(max_digits=11, decimal_places=2, read_only=True) - calendar_ytd = DecimalField(max_digits=11, decimal_places=2, read_only=True) + calendar_ytd_per_election_office = DecimalField( + max_digits=11, decimal_places=2, read_only=True + ) balance = DecimalField(max_digits=11, decimal_places=2, read_only=True) loan_payment_to_date = DecimalField(max_digits=11, decimal_places=2, read_only=True) loan_balance = DecimalField(max_digits=11, decimal_places=2, read_only=True) @@ -313,7 +315,7 @@ def get_fields(): "date", "amount", "aggregate", - "calendar_ytd", + "calendar_ytd_per_election_office", "loan_payment_to_date", "balance", "loan_balance", diff --git a/django-backend/fecfiler/transactions/tests/test_manager.py b/django-backend/fecfiler/transactions/tests/test_manager.py index 826470e91d..20992b12ac 100644 --- a/django-backend/fecfiler/transactions/tests/test_manager.py +++ b/django-backend/fecfiler/transactions/tests/test_manager.py @@ -43,7 +43,9 @@ def test_refund_aggregation(self): def test_election_aggregation(self): transaction = Transaction.objects.get(id="c4ba684a-607f-4f5d-bfb4-0fa1776d4e35") - self.assertEqual(transaction.calendar_ytd, Decimal("578.00")) + self.assertEqual( + transaction.calendar_ytd_per_election_office, Decimal("578.00") + ) def test_debt_repayment(self): repayment = Transaction.objects.get(id="dbdbdbdb-62f7-4a11-ac8e-27ea2afa9491") diff --git a/django-backend/fecfiler/web_services/dot_fec/schema_fields/SchE.json b/django-backend/fecfiler/web_services/dot_fec/schema_fields/SchE.json index f6de153e0e..af172135d2 100644 --- a/django-backend/fecfiler/web_services/dot_fec/schema_fields/SchE.json +++ b/django-backend/fecfiler/web_services/dot_fec/schema_fields/SchE.json @@ -57,8 +57,8 @@ "serializer": "DATE", "path": "schedule_e.disbursement_date" }, - "calendar_ytd": { - "path": "schedule_e.calendar_ytd" + "calendar_ytd_per_election_office": { + "path": "schedule_e.calendar_ytd_per_election_office" }, "expenditure_purpose_descrip": { "path": "schedule_e.expenditure_purpose_descrip" diff --git a/requirements.txt b/requirements.txt index dc81906f80..112097ddef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ django-cors-headers==3.13.0 django-storages==1.13.1 djangorestframework==3.14.0 drf-spectacular==0.24.2 -git+https://github.com/fecgov/fecfile-validate@59c6e2e7c24e63839765f2fc134fc46d7bbc0fe3#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@ec49101de759863016ff6cd5afd642aad87949f0#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.30 gunicorn==20.1.0 Jinja2==3.1.2 From cd2f937e1d2e010f9414cee60632f9b05258fd70 Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Thu, 26 Oct 2023 16:18:41 -0400 Subject: [PATCH 48/70] 1405 added tests --- .../transactions/schedule_c/serializers.py | 8 ++ .../schedule_c/tests/test_serializers.py | 63 +++++++++ .../transactions/schedule_c2/serializers.py | 7 +- .../schedule_c2/tests/test_serializers.py | 125 +++++++++++++++++- 4 files changed, 197 insertions(+), 6 deletions(-) diff --git a/django-backend/fecfiler/transactions/schedule_c/serializers.py b/django-backend/fecfiler/transactions/schedule_c/serializers.py index 47a9037474..892139d14a 100644 --- a/django-backend/fecfiler/transactions/schedule_c/serializers.py +++ b/django-backend/fecfiler/transactions/schedule_c/serializers.py @@ -80,6 +80,14 @@ def update_in_future_reports(self, transaction: Transaction, validated_data: dic ) schedule_cs_to_update.update(**schedule_c_data) + self.update_memo_text_in_future_reports( + transaction, validated_data, transactions_to_update) + + def update_memo_text_in_future_reports( + self, + transaction: Transaction, + validated_data: dict, + transactions_to_update): memo_text_id = validated_data.get("memo_text_id") if not memo_text_id: for memo_text in MemoText.objects.filter( diff --git a/django-backend/fecfiler/transactions/schedule_c/tests/test_serializers.py b/django-backend/fecfiler/transactions/schedule_c/tests/test_serializers.py index 44dd7cf540..1a1ca829d4 100644 --- a/django-backend/fecfiler/transactions/schedule_c/tests/test_serializers.py +++ b/django-backend/fecfiler/transactions/schedule_c/tests/test_serializers.py @@ -3,6 +3,7 @@ from rest_framework.request import HttpRequest, Request from fecfiler.transactions.schedule_c.serializers import ScheduleCTransactionSerializer +from fecfiler.transactions.models import Transaction class ScheduleCTransactionSerializerTestCase(TestCase): @@ -10,6 +11,7 @@ class ScheduleCTransactionSerializerTestCase(TestCase): "test_committee_accounts", "test_f3x_reports", "test_contacts", + "test_transaction_serializer", "test_memo_text", ] @@ -28,6 +30,22 @@ def setUp(self): "updated": "2022-02-09T00:00:00.000Z", "committee_account_id": "735db943-9446-462a-9be0-c820baadb622", } + + self.updated_contact = { + "id": "00000000-6486-4062-944f-aa0c4cbe4074", + "type": "IND", + "last_name": "contact_1", + "first_name": "Updated", + "street_1": "Street", + "city": "City", + "state": "CA", + "zip": "12345678", + "country": "Country", + "created": "2022-02-09T00:00:00.000Z", + "updated": "2022-02-09T00:00:00.000Z", + "committee_account_id": "735db943-9446-462a-9be0-c820baadb622", + } + self.valid_schedule_c_transaction = { "form_type": "SC/10", "transaction_type_identifier": "SCHEDULE_C", @@ -35,6 +53,7 @@ def setUp(self): "entity_type": "IND", "lender_organization_name": "John Smith Co", "lender_first_name": "John", + "lender_middle_name": "test_lmn1", "lender_last_name": "Smith", "lender_state": "AK", "lender_city": "Homer", @@ -45,6 +64,25 @@ def setUp(self): "schema_name": "SchC", } + self.updated_contact_schedule_c_transaction = { + "id": "35c18f45-9e4f-4593-b004-73126152abae", + "form_type": "SC/10", + "transaction_type_identifier": "SCHEDULE_C", + "transaction_id": "ABCDEF0123456789", + "entity_type": "IND", + "lender_organization_name": "John Smith Co", + "lender_first_name": "John", + "lender_middle_name": "updated_mn", + "lender_last_name": "Smith", + "lender_state": "AK", + "lender_city": "Homer", + "lender_zip": "1234", + "lender_street_1": "1 Homer Spit Road", + "report_id": "b6d60d2d-d926-4e89-ad4b-c47d152a66ae", + "contact_1": self.updated_contact, + "schema_name": "SchC", + } + self.mock_request = Request(HttpRequest()) user = Account() user.cmtee_id = "C00277616" @@ -74,3 +112,28 @@ def test_no_lender_first_name(self): ) self.assertFalse(missing_type_serializer.is_valid()) self.assertIsNotNone(missing_type_serializer.errors["lender_first_name"]) + + def test_create_and_update_with_loan_pulled_forward(self): + # create + transaction = self.valid_schedule_c_transaction.copy() + valid_serializer = ScheduleCTransactionSerializer( + data=transaction, + context={"request": self.mock_request}, + ) + self.assertTrue(valid_serializer.is_valid(raise_exception=True)) + valid_serializer.create(valid_serializer.to_internal_value(transaction)) + created_instance = Transaction.objects.filter( + schedule_c__lender_middle_name="test_lmn1" + ) + self.assertEqual(created_instance.count(), 3) + + # update + updated_transaction = self.updated_contact_schedule_c_transaction.copy() + updated_transaction["lender_middle_name"] = 'test_lmn1_updated1' + valid_serializer.update( + created_instance[0], valid_serializer.to_internal_value(updated_transaction) + ) + updated_instance = Transaction.objects.filter( + schedule_c__lender_middle_name="test_lmn1_updated1" + ) + self.assertEqual(updated_instance.count(), 3) diff --git a/django-backend/fecfiler/transactions/schedule_c2/serializers.py b/django-backend/fecfiler/transactions/schedule_c2/serializers.py index 6f06d992d7..3feebdce49 100644 --- a/django-backend/fecfiler/transactions/schedule_c2/serializers.py +++ b/django-backend/fecfiler/transactions/schedule_c2/serializers.py @@ -55,14 +55,13 @@ def update(self, instance, validated_data: dict): return schedule_c2_transaction def create_in_future_reports(self, transaction: Transaction): - future_reports = self.get_future_in_progress_reports(transaction.report) - transaction_copy = copy.deepcopy(transaction) + future_reports = super().get_future_in_progress_reports(transaction.report) for report in future_reports: loan = Transaction.objects.get( report_id=report.id, - loan_id=transaction_copy.parent_transaction.id + loan_id=transaction.parent_transaction.id ) - report.pull_forward_loan_guarantor(transaction_copy, loan) + report.pull_forward_loan_guarantor(copy.deepcopy(transaction), loan) def update_in_future_reports(self, transaction: Transaction, validated_data: dict): future_reports = super().get_future_in_progress_reports(transaction.report) diff --git a/django-backend/fecfiler/transactions/schedule_c2/tests/test_serializers.py b/django-backend/fecfiler/transactions/schedule_c2/tests/test_serializers.py index 904c49c33d..111f167c73 100644 --- a/django-backend/fecfiler/transactions/schedule_c2/tests/test_serializers.py +++ b/django-backend/fecfiler/transactions/schedule_c2/tests/test_serializers.py @@ -1,15 +1,27 @@ from django.test import TestCase from fecfiler.authentication.models import Account from rest_framework.request import HttpRequest, Request +from fecfiler.transactions.models import Transaction from fecfiler.transactions.schedule_c2.serializers import ( ScheduleC2TransactionSerializer, ) +from fecfiler.transactions.schedule_c.serializers import ( + ScheduleCTransactionSerializer, +) class ScheduleC2TransactionSerializerTestCase(TestCase): + fixtures = [ + "test_committee_accounts", + "test_f3x_reports", + "test_contacts", + "test_transaction_serializer", + "test_memo_text", + ] + def setUp(self): - self.new_contact = { + self.new_contact1 = { "id": "9bb5c8b2-31f3-488f-84e1-a63b0133a000", "type": "IND", "last_name": "Smith", @@ -23,6 +35,54 @@ def setUp(self): "updated": "2022-02-09T00:00:00.000Z", "committee_account_id": "735db943-9446-462a-9be0-c820baadb622", } + self.new_contact2 = { + "id": "05ab716f-c207-4dc1-8d9c-bb0fef47084a", + "type": "IND", + "last_name": "Smith2", + "first_name": "John2", + "street_1": "Street2", + "city": "City2", + "state": "CA", + "zip": "12345678", + "country": "Country", + "created": "2022-02-09T00:00:00.000Z", + "updated": "2022-02-09T00:00:00.000Z", + "committee_account_id": "735db943-9446-462a-9be0-c820baadb622", + } + self.updated_contact = { + "id": "00000000-6486-4062-944f-aa0c4cbe4074", + "type": "IND", + "last_name": "contact_1", + "first_name": "Updated", + "street_1": "Street", + "city": "City", + "state": "CA", + "zip": "12345678", + "country": "Country", + "created": "2022-02-09T00:00:00.000Z", + "updated": "2022-02-09T00:00:00.000Z", + "committee_account_id": "735db943-9446-462a-9be0-c820baadb622", + } + + self.valid_schedule_c_transaction = { + "id": "a879cf45-e74a-4252-a7bf-abf379a6e88c", + "form_type": "SC/10", + "transaction_type_identifier": "SCHEDULE_C", + "transaction_id": "ABCDEF0123456789", + "entity_type": "IND", + "lender_organization_name": "John Smith Co", + "lender_first_name": "John", + "lender_middle_name": "test_lmn1", + "lender_last_name": "Smith", + "lender_state": "AK", + "lender_city": "Homer", + "lender_zip": "1234", + "lender_street_1": "1 Homer Spit Road", + "report_id": "b6d60d2d-d926-4e89-ad4b-c47d152a66ae", + "contact_1": self.new_contact1, + "schema_name": "SchC", + } + self.valid_schedule_c2_transaction = { "form_type": "SC2/9", "transaction_type_identifier": "SCHEDULE_C1", @@ -30,14 +90,36 @@ def setUp(self): "back_reference_tran_id_number": "BBCDEF0123456789", "guarantor_last_name": "Smythe", "guarantor_first_name": "Fred", + "guarantor_middle_name": "test_gmn1", "guarantor_state": "AK", "guarantor_city": "Homer", "guarantor_zip": "1234", "guarantor_street_1": "1 Homer Spit Road", "report_id": "b6d60d2d-d926-4e89-ad4b-c47d152a66ae", - "contact": self.new_contact, + "contact_1": self.new_contact2, "schema_name": "SchC2", "loan_amount": "1000.00", + "guaranteed_amount": "5.00", + } + + self.updated_valid_schedule_c2_transaction = { + "id": "249fd59f-6a97-49f8-8c7f-239279bb7bea", + "form_type": "SC2/9", + "transaction_type_identifier": "SCHEDULE_C1", + "transaction_id": "ABCDEF0123456789", + "back_reference_tran_id_number": "BBCDEF0123456789", + "guarantor_last_name": "Smythe", + "guarantor_first_name": "Fred", + "guarantor_middle_name": "test_gmn1", + "guarantor_state": "AK", + "guarantor_city": "Homer", + "guarantor_zip": "1234", + "guarantor_street_1": "1 Homer Spit Road", + "report_id": "b6d60d2d-d926-4e89-ad4b-c47d152a66ae", + "contact_1": self.updated_contact, + "schema_name": "SchC2", + "loan_amount": "1000.00", + "guaranteed_amount": "5.00", } self.mock_request = Request(HttpRequest()) @@ -69,3 +151,42 @@ def test_no_guarantor_last_name(self): ) self.assertFalse(missing_type_serializer.is_valid()) self.assertIsNotNone(missing_type_serializer.errors["guarantor_last_name"]) + + def test_create_and_update_with_loan_guarantor_pulled_forward(self): + # create schedule c that gets pulled forward + transaction = self.valid_schedule_c_transaction.copy() + valid_serializer = ScheduleCTransactionSerializer( + data=transaction, + context={"request": self.mock_request}, + ) + self.assertTrue(valid_serializer.is_valid(raise_exception=True)) + valid_serializer.create(valid_serializer.to_internal_value(transaction)) + created_instance = Transaction.objects.filter( + schedule_c__lender_middle_name="test_lmn1" + ) + self.assertEqual(created_instance.count(), 3) + + # test c2 create + self.valid_schedule_c2_transaction["parent_transaction_id"] = created_instance[0].id + transaction = self.valid_schedule_c2_transaction.copy() + valid_serializer = ScheduleC2TransactionSerializer( + data=transaction, + context={"request": self.mock_request}, + ) + self.assertTrue(valid_serializer.is_valid(raise_exception=True)) + valid_serializer.create(valid_serializer.to_internal_value(transaction)) + created_instance = Transaction.objects.filter( + schedule_c2__guarantor_middle_name="test_gmn1" + ) + self.assertEqual(created_instance.count(), 3) + + # test c2 update + updated_transaction = self.updated_valid_schedule_c2_transaction.copy() + updated_transaction["guaranteed_amount"] = "10.00" + valid_serializer.update( + created_instance[0], valid_serializer.to_internal_value(updated_transaction) + ) + updated_instance = Transaction.objects.filter( + schedule_c2__guaranteed_amount="10.00" + ) + self.assertEqual(updated_instance.count(), 4) From 38691c2c3158b2499bedd5b7144b2acf8669a457 Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Thu, 26 Oct 2023 16:22:10 -0400 Subject: [PATCH 49/70] 1405 fix tests --- .../fecfiler/transactions/schedule_c/tests/test_serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django-backend/fecfiler/transactions/schedule_c/tests/test_serializers.py b/django-backend/fecfiler/transactions/schedule_c/tests/test_serializers.py index 1a1ca829d4..4289abddd9 100644 --- a/django-backend/fecfiler/transactions/schedule_c/tests/test_serializers.py +++ b/django-backend/fecfiler/transactions/schedule_c/tests/test_serializers.py @@ -136,4 +136,4 @@ def test_create_and_update_with_loan_pulled_forward(self): updated_instance = Transaction.objects.filter( schedule_c__lender_middle_name="test_lmn1_updated1" ) - self.assertEqual(updated_instance.count(), 3) + self.assertEqual(updated_instance.count(), 4) From 96bca253f6541f83a96126eb1b55c006c4a90369 Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Thu, 26 Oct 2023 16:26:49 -0400 Subject: [PATCH 50/70] 1405 lint --- .../transactions/schedule_c2/tests/test_serializers.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/django-backend/fecfiler/transactions/schedule_c2/tests/test_serializers.py b/django-backend/fecfiler/transactions/schedule_c2/tests/test_serializers.py index 111f167c73..c23a1fd489 100644 --- a/django-backend/fecfiler/transactions/schedule_c2/tests/test_serializers.py +++ b/django-backend/fecfiler/transactions/schedule_c2/tests/test_serializers.py @@ -165,9 +165,11 @@ def test_create_and_update_with_loan_guarantor_pulled_forward(self): schedule_c__lender_middle_name="test_lmn1" ) self.assertEqual(created_instance.count(), 3) - + # test c2 create - self.valid_schedule_c2_transaction["parent_transaction_id"] = created_instance[0].id + self.valid_schedule_c2_transaction[ + "parent_transaction_id" + ] = created_instance[0].id transaction = self.valid_schedule_c2_transaction.copy() valid_serializer = ScheduleC2TransactionSerializer( data=transaction, From 4f15ebe625fcd310f74799bdeba118ce25513f36 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Thu, 26 Oct 2023 16:55:09 -0400 Subject: [PATCH 51/70] Update validate commit hash --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 112097ddef..603a09d502 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ django-cors-headers==3.13.0 django-storages==1.13.1 djangorestframework==3.14.0 drf-spectacular==0.24.2 -git+https://github.com/fecgov/fecfile-validate@ec49101de759863016ff6cd5afd642aad87949f0#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@5eef24798e1e0b38205def5ea1a6d2a548719b5b#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.30 gunicorn==20.1.0 Jinja2==3.1.2 From 944f0cd87ac0876da4f83187e0959cbd6a64858a Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Mon, 30 Oct 2023 14:11:26 -0400 Subject: [PATCH 52/70] 1406 pull debts forward into future reports --- django-backend/fecfiler/reports/models.py | 19 +++--- .../transactions/schedule_d/serializers.py | 37 +++++++++++- .../schedule_d/tests/test_serializers.py | 59 +++++++++++++++++++ 3 files changed, 105 insertions(+), 10 deletions(-) diff --git a/django-backend/fecfiler/reports/models.py b/django-backend/fecfiler/reports/models.py index 59998a9d38..187957d65b 100644 --- a/django-backend/fecfiler/reports/models.py +++ b/django-backend/fecfiler/reports/models.py @@ -136,14 +136,17 @@ def pull_forward_debts(self): ) for debt in debts_to_pull_forward: - debt.schedule_d.incurred_amount = Decimal(0) - debt.schedule_d_id = self.save_copy(debt.schedule_d) - debt.report_id = self.id - debt.report = self - # The debt_id should point to the original debt transaction - # even if the debt is pulled forward multiple times. - debt.debt_id = debt.debt_id if debt.debt_id else debt.id - self.save_copy(debt) + self.pull_forward_debt(debt) + + def pull_forward_debt(self, debt): + debt.schedule_d.incurred_amount = Decimal(0) + debt.schedule_d_id = self.save_copy(debt.schedule_d) + debt.report_id = self.id + debt.report = self + # The debt_id should point to the original debt transaction + # even if the debt is pulled forward multiple times. + debt.debt_id = debt.debt_id if debt.debt_id else debt.id + self.save_copy(debt) def get_previous_report(self): return ( diff --git a/django-backend/fecfiler/transactions/schedule_d/serializers.py b/django-backend/fecfiler/transactions/schedule_d/serializers.py index ddadc8090b..54b3d8180f 100644 --- a/django-backend/fecfiler/transactions/schedule_d/serializers.py +++ b/django-backend/fecfiler/transactions/schedule_d/serializers.py @@ -1,10 +1,12 @@ import logging -from django.db import transaction +from django.db import transaction, models from fecfiler.transactions.schedule_d.models import ScheduleD +from fecfiler.transactions.models import Transaction from fecfiler.transactions.serializers import TransactionSerializerBase from fecfiler.shared.utilities import get_model_data from rest_framework.fields import DecimalField, CharField +import copy logger = logging.getLogger(__name__) @@ -36,6 +38,7 @@ def create(self, validated_data: dict): del transaction_data[key] schedule_d_transaction = super().create(transaction_data) + self.create_in_future_reports(schedule_d_transaction) return schedule_d_transaction def update(self, instance, validated_data: dict): @@ -44,7 +47,37 @@ def update(self, instance, validated_data: dict): if attr != "id": setattr(instance.schedule_d, attr, value) instance.schedule_d.save() - return super().update(instance, validated_data) + schedule_d_transaction = super().update(instance, validated_data) + self.update_in_future_reports(schedule_d_transaction, validated_data) + return schedule_d_transaction + + def create_in_future_reports(self, transaction: Transaction): + future_reports = super().get_future_in_progress_reports(transaction.report) + transaction_copy = copy.deepcopy(transaction) + for report in future_reports: + report.pull_forward_debt(transaction_copy) + + def update_in_future_reports(self, transaction: Transaction, validated_data: dict): + future_reports = super().get_future_in_progress_reports(transaction.report) + + transaction_data = get_model_data(validated_data, Transaction) + del transaction_data['id'] + transactions_to_update = Transaction.objects.filter( + transaction_id=transaction.transaction_id, + report_id__in=models.Subquery( + future_reports.values('id') + ) + ) + transactions_to_update.update(**transaction_data) + + schedule_d_data = get_model_data(validated_data, ScheduleD) + del schedule_d_data['id'] + schedule_ds_to_update = ScheduleD.objects.filter( + transaction__schedule_d_id__in=models.Subquery( + transactions_to_update.values('schedule_d_id') + ) + ) + schedule_ds_to_update.update(**schedule_d_data) receipt_line_number = CharField(required=False, allow_null=True) receipt_line_number = CharField(required=False, allow_null=True) diff --git a/django-backend/fecfiler/transactions/schedule_d/tests/test_serializers.py b/django-backend/fecfiler/transactions/schedule_d/tests/test_serializers.py index 2d2b723810..425738b9a3 100644 --- a/django-backend/fecfiler/transactions/schedule_d/tests/test_serializers.py +++ b/django-backend/fecfiler/transactions/schedule_d/tests/test_serializers.py @@ -3,6 +3,7 @@ from rest_framework.request import HttpRequest, Request from fecfiler.transactions.schedule_d.serializers import ScheduleDTransactionSerializer +from fecfiler.transactions.models import Transaction class ScheduleDTransactionSerializerTestCase(TestCase): @@ -28,6 +29,20 @@ def setUp(self): "updated": "2022-02-09T00:00:00.000Z", "committee_account_id": "735db943-9446-462a-9be0-c820baadb622", } + self.updated_contact = { + "id": "00000000-6486-4062-944f-aa0c4cbe4074", + "type": "IND", + "last_name": "contact_1", + "first_name": "Updated", + "street_1": "Street", + "city": "City", + "state": "CA", + "zip": "12345678", + "country": "Country", + "created": "2022-02-09T00:00:00.000Z", + "updated": "2022-02-09T00:00:00.000Z", + "committee_account_id": "735db943-9446-462a-9be0-c820baadb622", + } self.valid_schedule_d_transaction = { "form_type": "SD10", "transaction_type_identifier": "SCHEDULE_D", @@ -35,6 +50,7 @@ def setUp(self): "entity_type": "IND", "creditor_organization_name": "John Smith Co", "creditor_first_name": "John", + "creditor_middle_name": "test_mn1", "creditor_last_name": "Smith", "creditor_state": "AK", "creditor_city": "Homer", @@ -44,6 +60,24 @@ def setUp(self): "contact_1": self.new_contact, "schema_name": "SchD", } + self.updated_contact_schedule_d_transaction = { + "id": "dc688ac7-c379-4c34-be8a-05439b375ce4", + "form_type": "SD10", + "transaction_type_identifier": "SCHEDULE_D", + "transaction_id": "ABCDEF0123456789", + "entity_type": "IND", + "creditor_organization_name": "John Smith Co", + "creditor_first_name": "John", + "creditor_middle_name": "updated_mn", + "creditor_last_name": "Smith", + "creditor_state": "AK", + "creditor_city": "Homer", + "creditor_zip": "1234", + "creditor_street_1": "1 Homer Spit Road", + "report_id": "b6d60d2d-d926-4e89-ad4b-c47d152a66ae", + "contact_1": self.updated_contact, + "schema_name": "SchD", + } self.mock_request = Request(HttpRequest()) user = Account() @@ -74,3 +108,28 @@ def test_no_creditor_first_name(self): ) self.assertFalse(missing_type_serializer.is_valid()) self.assertIsNotNone(missing_type_serializer.errors["creditor_first_name"]) + + def test_create_and_update_with_debt_pulled_forward(self): + # create + transaction = self.valid_schedule_d_transaction.copy() + valid_serializer = ScheduleDTransactionSerializer( + data=transaction, + context={"request": self.mock_request}, + ) + self.assertTrue(valid_serializer.is_valid(raise_exception=True)) + valid_serializer.create(valid_serializer.to_internal_value(transaction)) + created_instance = Transaction.objects.filter( + schedule_d__creditor_middle_name="test_mn1" + ) + self.assertEqual(created_instance.count(), 3) + + # update + updated_transaction = self.updated_contact_schedule_d_transaction.copy() + updated_transaction["creditor_middle_name"] = 'test_mn1_updated1' + valid_serializer.update( + created_instance[0], valid_serializer.to_internal_value(updated_transaction) + ) + updated_instance = Transaction.objects.filter( + schedule_d__creditor_middle_name="test_mn1_updated1" + ) + self.assertEqual(updated_instance.count(), 4) From baaa22a2e3b0d9eaa97439dd902bddcee80b37e0 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Tue, 31 Oct 2023 18:10:25 -0400 Subject: [PATCH 53/70] Add office filter to candidate lookups --- django-backend/fecfiler/contacts/views.py | 29 +++++++++++++---------- django-backend/fecfiler/settings/base.py | 2 +- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/django-backend/fecfiler/contacts/views.py b/django-backend/fecfiler/contacts/views.py index 1ceea21fd0..9ed18f073f 100644 --- a/django-backend/fecfiler/contacts/views.py +++ b/django-backend/fecfiler/contacts/views.py @@ -78,15 +78,18 @@ def candidate_lookup(self, request): return HttpResponseBadRequest() max_fecfile_results, max_fec_results = self.get_max_results(request) - - params = urlencode({"q": q, "api_key": FEC_API_KEY}) + office = request.GET.get("office", "") + params = {"q": q, "api_key": FEC_API_KEY} + if office: + params["office"] = office + params = urlencode(params) json_results = requests.get( FEC_API_CANDIDATE_LOOKUP_ENDPOINT, params=params ).json() tokens = list(filter(None, re.split("[^\\w+]", q))) term = (".*" + ".* .*".join(tokens) + ".*").lower() - fecfile_candidates = list( + query_set = ( self.get_queryset() .annotate( full_name_fwd=Lower(NAME_CLAUSE), @@ -102,17 +105,20 @@ def candidate_lookup(self, request): .values() .order_by("-candidate_id") ) + if office: + query_set = query_set.filter(candidate_office=office) + fecfile_candidates = list(query_set) fec_api_candidates = json_results.get("results", []) fec_api_candidates = [ fac for fac in fec_api_candidates - if not any(fac["id"] == ffc["candidate_id"] for ffc in fecfile_candidates) + if not any( + fac["candidate_id"] == ffc["candidate_id"] for ffc in fecfile_candidates + ) ] - fec_api_candidates = fec_api_candidates[:max_fec_results] - fecfile_candidates = fecfile_candidates[:max_fecfile_results] return_value = { - "fec_api_candidates": fec_api_candidates, - "fecfile_candidates": fecfile_candidates, + "fec_api_candidates": fec_api_candidates[:max_fec_results], + "fecfile_candidates": fecfile_candidates[:max_fecfile_results], } return JsonResponse(return_value) @@ -240,9 +246,7 @@ def get_max_results(self, request): class DeletedContactsViewSet( - CommitteeOwnedViewSet, - mixins.ListModelMixin, - GenericViewSet, + CommitteeOwnedViewSet, mixins.ListModelMixin, GenericViewSet, ): serializer_class = ContactSerializer @@ -273,8 +277,7 @@ def restore(self, request): contacts = self.queryset.filter(id__in=ids_to_restore) if len(ids_to_restore) != contacts.count(): return Response( - "Contact Ids are invalid", - status=status.HTTP_400_BAD_REQUEST, + "Contact Ids are invalid", status=status.HTTP_400_BAD_REQUEST, ) for contact in contacts: contact.deleted = None diff --git a/django-backend/fecfiler/settings/base.py b/django-backend/fecfiler/settings/base.py index d30efffb6a..95f997d850 100644 --- a/django-backend/fecfiler/settings/base.py +++ b/django-backend/fecfiler/settings/base.py @@ -260,4 +260,4 @@ FEC_API_COMMITTEE_LOOKUP_IDS_OVERRIDE = env.get_credential( "FEC_API_COMMITTEE_LOOKUP_IDS_OVERRIDE" ) -FEC_API_CANDIDATE_LOOKUP_ENDPOINT = str(FEC_API) + "names/candidates/" +FEC_API_CANDIDATE_LOOKUP_ENDPOINT = str(FEC_API) + "candidates/" From 266de0ea3a58f849d6c070f53278be9962902691 Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Wed, 1 Nov 2023 15:00:14 -0400 Subject: [PATCH 54/70] 1406 only create if balance_at_close != 0 check --- .../fecfiler/transactions/schedule_d/serializers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/django-backend/fecfiler/transactions/schedule_d/serializers.py b/django-backend/fecfiler/transactions/schedule_d/serializers.py index 54b3d8180f..f891d8b45f 100644 --- a/django-backend/fecfiler/transactions/schedule_d/serializers.py +++ b/django-backend/fecfiler/transactions/schedule_d/serializers.py @@ -38,7 +38,8 @@ def create(self, validated_data: dict): del transaction_data[key] schedule_d_transaction = super().create(transaction_data) - self.create_in_future_reports(schedule_d_transaction) + if not schedule_d_transaction.balance_at_close: + self.create_in_future_reports(schedule_d_transaction) return schedule_d_transaction def update(self, instance, validated_data: dict): @@ -62,6 +63,7 @@ def update_in_future_reports(self, transaction: Transaction, validated_data: dic transaction_data = get_model_data(validated_data, Transaction) del transaction_data['id'] + del transaction_data['incurred_amount'] transactions_to_update = Transaction.objects.filter( transaction_id=transaction.transaction_id, report_id__in=models.Subquery( From 0d742a695073acc08fff47fff177a094638fe5b8 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Wed, 1 Nov 2023 16:16:12 -0400 Subject: [PATCH 55/70] Update validate commit hash --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 603a09d502..74990aebfb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ django-cors-headers==3.13.0 django-storages==1.13.1 djangorestframework==3.14.0 drf-spectacular==0.24.2 -git+https://github.com/fecgov/fecfile-validate@5eef24798e1e0b38205def5ea1a6d2a548719b5b#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@a0b39d5ead6e177d5696d26ee1b6643831d59088#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.30 gunicorn==20.1.0 Jinja2==3.1.2 From 9cc8fa34889395047733eb16fd27790f5a132901 Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Thu, 2 Nov 2023 12:11:12 -0400 Subject: [PATCH 56/70] 1406 add balance_at_close check for debt pull fwd --- .../fecfiler/transactions/schedule_d/serializers.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/django-backend/fecfiler/transactions/schedule_d/serializers.py b/django-backend/fecfiler/transactions/schedule_d/serializers.py index f891d8b45f..fb61277f49 100644 --- a/django-backend/fecfiler/transactions/schedule_d/serializers.py +++ b/django-backend/fecfiler/transactions/schedule_d/serializers.py @@ -1,6 +1,8 @@ +from decimal import Decimal import logging from django.db import transaction, models +from django.db.models import Q from fecfiler.transactions.schedule_d.models import ScheduleD from fecfiler.transactions.models import Transaction from fecfiler.transactions.serializers import TransactionSerializerBase @@ -38,7 +40,10 @@ def create(self, validated_data: dict): del transaction_data[key] schedule_d_transaction = super().create(transaction_data) - if not schedule_d_transaction.balance_at_close: + if Transaction.objects.filter( + ~Q(balance_at_close=Decimal(0)) | Q(balance_at_close__isnull=True), + id=schedule_d_transaction.id, + ).count(): self.create_in_future_reports(schedule_d_transaction) return schedule_d_transaction @@ -63,7 +68,6 @@ def update_in_future_reports(self, transaction: Transaction, validated_data: dic transaction_data = get_model_data(validated_data, Transaction) del transaction_data['id'] - del transaction_data['incurred_amount'] transactions_to_update = Transaction.objects.filter( transaction_id=transaction.transaction_id, report_id__in=models.Subquery( @@ -74,6 +78,8 @@ def update_in_future_reports(self, transaction: Transaction, validated_data: dic schedule_d_data = get_model_data(validated_data, ScheduleD) del schedule_d_data['id'] + if "incurred_amount" in schedule_d_data: + del schedule_d_data['incurred_amount'] schedule_ds_to_update = ScheduleD.objects.filter( transaction__schedule_d_id__in=models.Subquery( transactions_to_update.values('schedule_d_id') From 6aecd92a4b0083ded64ba2b868456d8d753d0ddd Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Thu, 2 Nov 2023 12:23:54 -0400 Subject: [PATCH 57/70] 1406 update test case --- .../fecfiler/transactions/schedule_d/tests/test_serializers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/django-backend/fecfiler/transactions/schedule_d/tests/test_serializers.py b/django-backend/fecfiler/transactions/schedule_d/tests/test_serializers.py index 425738b9a3..a14438589a 100644 --- a/django-backend/fecfiler/transactions/schedule_d/tests/test_serializers.py +++ b/django-backend/fecfiler/transactions/schedule_d/tests/test_serializers.py @@ -77,6 +77,7 @@ def setUp(self): "report_id": "b6d60d2d-d926-4e89-ad4b-c47d152a66ae", "contact_1": self.updated_contact, "schema_name": "SchD", + "incurred_amount": 4, } self.mock_request = Request(HttpRequest()) From 67625c13a3ee77993628e8997ae90f90902f3f11 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Thu, 2 Nov 2023 15:31:14 -0400 Subject: [PATCH 58/70] Feature/1461 Set up back-end to handle reattribution/redesignation --- ...eattribution_redesignation_tag_and_more.py | 23 +++++++++++++++++++ .../transactions/schedule_a/models.py | 2 ++ .../transactions/schedule_b/models.py | 2 ++ requirements.txt | 2 +- 4 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 django-backend/fecfiler/transactions/migrations/0031_schedulea_reattribution_redesignation_tag_and_more.py diff --git a/django-backend/fecfiler/transactions/migrations/0031_schedulea_reattribution_redesignation_tag_and_more.py b/django-backend/fecfiler/transactions/migrations/0031_schedulea_reattribution_redesignation_tag_and_more.py new file mode 100644 index 0000000000..e1891e345e --- /dev/null +++ b/django-backend/fecfiler/transactions/migrations/0031_schedulea_reattribution_redesignation_tag_and_more.py @@ -0,0 +1,23 @@ +# Generated by Django 4.1.3 on 2023-11-02 19:29 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('transactions', '0030_update_receipt_from_person_to_receipt_from_entity'), + ] + + operations = [ + migrations.AddField( + model_name='schedulea', + name='reattribution_redesignation_tag', + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name='scheduleb', + name='reattribution_redesignation_tag', + field=models.TextField(blank=True, null=True), + ), + ] diff --git a/django-backend/fecfiler/transactions/schedule_a/models.py b/django-backend/fecfiler/transactions/schedule_a/models.py index fc4d4d255b..5225fa8cbf 100644 --- a/django-backend/fecfiler/transactions/schedule_a/models.py +++ b/django-backend/fecfiler/transactions/schedule_a/models.py @@ -56,6 +56,8 @@ class ScheduleA(models.Model): null=True, blank=True ) + reattribution_redesignation_tag = models.TextField(null=True, blank=True) + class Meta: app_label = "transactions" diff --git a/django-backend/fecfiler/transactions/schedule_b/models.py b/django-backend/fecfiler/transactions/schedule_b/models.py index 0d21315e51..60f26554ca 100644 --- a/django-backend/fecfiler/transactions/schedule_b/models.py +++ b/django-backend/fecfiler/transactions/schedule_b/models.py @@ -57,6 +57,8 @@ class ScheduleB(models.Model): null=True, blank=True ) + reattribution_redesignation_tag = models.TextField(null=True, blank=True) + def get_date(self): return self.expenditure_date diff --git a/requirements.txt b/requirements.txt index dc81906f80..24b4e3dcde 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ django-cors-headers==3.13.0 django-storages==1.13.1 djangorestframework==3.14.0 drf-spectacular==0.24.2 -git+https://github.com/fecgov/fecfile-validate@59c6e2e7c24e63839765f2fc134fc46d7bbc0fe3#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@60651610ea3a8b20edd2102a1fd21445ae4dba7e#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.30 gunicorn==20.1.0 Jinja2==3.1.2 From 20f387736bed950a67ea9b909f45c4dcd94d5732 Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Thu, 2 Nov 2023 15:53:22 -0400 Subject: [PATCH 59/70] 1198 amendment backdoor --- .../web_services/dot_fec/dot_fec_submitter.py | 4 +++- .../web_services/dot_fec/test_dot_fec_submitter.py | 13 +++++++++++++ django-backend/fecfiler/web_services/serializers.py | 1 + django-backend/fecfiler/web_services/tasks.py | 7 ++++++- django-backend/fecfiler/web_services/views.py | 7 ++++++- 5 files changed, 29 insertions(+), 3 deletions(-) diff --git a/django-backend/fecfiler/web_services/dot_fec/dot_fec_submitter.py b/django-backend/fecfiler/web_services/dot_fec/dot_fec_submitter.py index a964fcc46a..72d01bdb12 100644 --- a/django-backend/fecfiler/web_services/dot_fec/dot_fec_submitter.py +++ b/django-backend/fecfiler/web_services/dot_fec/dot_fec_submitter.py @@ -21,7 +21,7 @@ def __init__(self, api): if api: self.fec_soap_client = Client(f"{api}/webload/services/upload?wsdl") - def get_submission_json(self, dot_fec_record, e_filing_password): + def get_submission_json(self, dot_fec_record, e_filing_password, backdoor_code=None): json_obj = { "committee_id": FILE_AS_TEST_COMMITTEE or dot_fec_record.report.committee_account.committee_id, @@ -36,6 +36,8 @@ def get_submission_json(self, dot_fec_record, e_filing_password): } if dot_fec_record.report.report_id: json_obj["amendment_id"] = dot_fec_record.report.report_id + if backdoor_code: + json_obj["amendment_id"] += backdoor_code return json.dumps(json_obj) def submit(self, dot_fec_bytes, json_payload, fec_report_id=None): diff --git a/django-backend/fecfiler/web_services/dot_fec/test_dot_fec_submitter.py b/django-backend/fecfiler/web_services/dot_fec/test_dot_fec_submitter.py index e1df71e893..24314ff078 100644 --- a/django-backend/fecfiler/web_services/dot_fec/test_dot_fec_submitter.py +++ b/django-backend/fecfiler/web_services/dot_fec/test_dot_fec_submitter.py @@ -1,4 +1,5 @@ import json +from uuid import uuid4 as uuid from django.test import TestCase from .dot_fec_submitter import DotFECSubmitter from fecfiler.web_services.models import DotFEC @@ -30,3 +31,15 @@ def test_get_submission_json(self): self.assertEqual(json_obj["email_1"], self.f3x.confirmation_email_1) self.assertEqual(json_obj["email_2"], self.f3x.confirmation_email_2) self.assertFalse(json_obj["wait"]) + + def test_get_submission_json_for_amendment(self): + submitter = DotFECSubmitter(None) + self.dot_fec_record.report.report_id = str(uuid()) + json_str = submitter.get_submission_json( + self.dot_fec_record, "test_json_password", "test_backdoor_code" + ) + json_obj = json.loads(json_str) + self.assertEqual( + json_obj["amendment_id"], + self.dot_fec_record.report.report_id + "test_backdoor_code" + ) diff --git a/django-backend/fecfiler/web_services/serializers.py b/django-backend/fecfiler/web_services/serializers.py index 09fffd9c3b..bbc5585ff0 100644 --- a/django-backend/fecfiler/web_services/serializers.py +++ b/django-backend/fecfiler/web_services/serializers.py @@ -21,6 +21,7 @@ def validate(self, data): class SubmissionRequestSerializer(ReportIdSerializer): password = serializers.CharField(allow_blank=False) + backdoor_code = serializers.CharField(required=False, allow_null=True) class UploadSubmissionSerializer(serializers.ModelSerializer): diff --git a/django-backend/fecfiler/web_services/tasks.py b/django-backend/fecfiler/web_services/tasks.py index bf72a57593..4c680fa3a7 100644 --- a/django-backend/fecfiler/web_services/tasks.py +++ b/django-backend/fecfiler/web_services/tasks.py @@ -59,6 +59,7 @@ def submit_to_fec( dot_fec_id, submission_record_id, e_filing_password, + backdoor_code=None, api=None, force_read_from_disk=False, ): @@ -84,7 +85,11 @@ def submit_to_fec( """Submit to FEC""" submitter = DotFECSubmitter(api) logger.info(f"Uploading {file_name} to FEC") - submission_json = submitter.get_submission_json(dot_fec_record, e_filing_password) + submission_json = submitter.get_submission_json( + dot_fec_record, + e_filing_password, + backdoor_code + ) submission_response_string = submitter.submit( dot_fec_bytes, submission_json, dot_fec_record.report.report_id or None ) diff --git a/django-backend/fecfiler/web_services/views.py b/django-backend/fecfiler/web_services/views.py index 48121fd74c..b4e6546884 100644 --- a/django-backend/fecfiler/web_services/views.py +++ b/django-backend/fecfiler/web_services/views.py @@ -81,6 +81,7 @@ def submit_to_fec(self, request): """Retrieve parameters""" report_id = serializer.validated_data["report_id"] e_filing_password = serializer.validated_data["password"] + backdoor_code = serializer.validated_data.get("backdoor_code", None) """Start tracking submission""" upload_submission = UploadSubmission.objects.initiate_submission(report_id) @@ -89,7 +90,11 @@ def submit_to_fec(self, request): task = ( create_dot_fec.s(report_id, upload_submission_id=upload_submission.id) | submit_to_fec.s( - upload_submission.id, e_filing_password, FEC_FILING_API, False + upload_submission.id, + e_filing_password, + backdoor_code, + FEC_FILING_API, + True ) ).apply_async(retry=False) From 1dc0f1bc3b796e4fdfe29861e76c158c4f9706d2 Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Thu, 2 Nov 2023 16:18:00 -0400 Subject: [PATCH 60/70] 1198 fix unit test --- django-backend/fecfiler/web_services/tasks.py | 2 +- django-backend/fecfiler/web_services/views.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/django-backend/fecfiler/web_services/tasks.py b/django-backend/fecfiler/web_services/tasks.py index 4c680fa3a7..7b4a682193 100644 --- a/django-backend/fecfiler/web_services/tasks.py +++ b/django-backend/fecfiler/web_services/tasks.py @@ -59,9 +59,9 @@ def submit_to_fec( dot_fec_id, submission_record_id, e_filing_password, - backdoor_code=None, api=None, force_read_from_disk=False, + backdoor_code=None, ): logger.info(f"FEC API: {FEC_FILING_API}") logger.info(f"api submitter: {api}") diff --git a/django-backend/fecfiler/web_services/views.py b/django-backend/fecfiler/web_services/views.py index b4e6546884..c341e07dac 100644 --- a/django-backend/fecfiler/web_services/views.py +++ b/django-backend/fecfiler/web_services/views.py @@ -92,9 +92,9 @@ def submit_to_fec(self, request): | submit_to_fec.s( upload_submission.id, e_filing_password, - backdoor_code, FEC_FILING_API, - True + True, + backdoor_code ) ).apply_async(retry=False) From b7352c2d2f483f0e4753682196772d6b5ff7f5a6 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Fri, 3 Nov 2023 16:35:12 -0400 Subject: [PATCH 61/70] Add reatt/redist to serializers --- django-backend/fecfiler/transactions/schedule_a/serializers.py | 1 + django-backend/fecfiler/transactions/schedule_b/serializers.py | 1 + 2 files changed, 2 insertions(+) diff --git a/django-backend/fecfiler/transactions/schedule_a/serializers.py b/django-backend/fecfiler/transactions/schedule_a/serializers.py index b4e94a4987..75dfbe5866 100644 --- a/django-backend/fecfiler/transactions/schedule_a/serializers.py +++ b/django-backend/fecfiler/transactions/schedule_a/serializers.py @@ -87,6 +87,7 @@ class Meta(TransactionSerializerBase.Meta): reference_to_si_or_sl_system_code_that_identifies_the_account = CharField( required=False, allow_null=True ) + reattribution_redesignation_tag = CharField(required=False, allow_null=True) class ScheduleATransactionSerializerTier2(ScheduleATransactionSerializerBase): diff --git a/django-backend/fecfiler/transactions/schedule_b/serializers.py b/django-backend/fecfiler/transactions/schedule_b/serializers.py index 26b8a3134f..27de3a375f 100644 --- a/django-backend/fecfiler/transactions/schedule_b/serializers.py +++ b/django-backend/fecfiler/transactions/schedule_b/serializers.py @@ -86,6 +86,7 @@ class Meta(TransactionSerializerBase.Meta): reference_to_si_or_sl_system_code_that_identifies_the_account = CharField( required=False, allow_null=True ) + reattribution_redesignation_tag = CharField(required=False, allow_null=True) class ScheduleBTransactionSerializer(ScheduleBTransactionSerializerBase): From e5016b8a809f19d6d717d7d1e89d6b71c2e59491 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Mon, 6 Nov 2023 11:37:49 -0500 Subject: [PATCH 62/70] Update validate commit hash --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 74990aebfb..0388861b27 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ django-cors-headers==3.13.0 django-storages==1.13.1 djangorestframework==3.14.0 drf-spectacular==0.24.2 -git+https://github.com/fecgov/fecfile-validate@a0b39d5ead6e177d5696d26ee1b6643831d59088#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@c8d4fa4d39c28925dd522aba8c0dd55048d35d3e#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.30 gunicorn==20.1.0 Jinja2==3.1.2 From 73db049dfe71a22b7293336b04f5b7551ffebb27 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Mon, 6 Nov 2023 11:40:32 -0500 Subject: [PATCH 63/70] Add exception to safety check list --- .safety.dependency.ignore | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.safety.dependency.ignore b/.safety.dependency.ignore index 176ef18107..ef7c6ed211 100644 --- a/.safety.dependency.ignore +++ b/.safety.dependency.ignore @@ -8,11 +8,12 @@ # Example: # 40104 2022-01-15 # -52945 2023-12-01 # django -53315 2023-12-01 # django -59293 2023-12-01 # django -55264 2023-12-01 # django -60350 2023-12-01 # django -60841 2023-12-01 # django -60789 2023-12-01 # django -60956 2023-12-01 # django +52945 2024-01-01 # django +53315 2024-01-01 # django +59293 2024-01-01 # django +55264 2024-01-01 # django +60350 2024-01-01 # django +60841 2024-01-01 # django +60789 2024-01-01 # django +60956 2024-01-01 # django +61586 2024-01-01 # django From 5b18622f2f07a663b8ff7c8ad32907f2eac99d9d Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Mon, 6 Nov 2023 11:50:26 -0500 Subject: [PATCH 64/70] Fix migration file conflict --- ... 0032_schedulea_reattribution_redesignation_tag_and_more.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename django-backend/fecfiler/transactions/migrations/{0031_schedulea_reattribution_redesignation_tag_and_more.py => 0032_schedulea_reattribution_redesignation_tag_and_more.py} (86%) diff --git a/django-backend/fecfiler/transactions/migrations/0031_schedulea_reattribution_redesignation_tag_and_more.py b/django-backend/fecfiler/transactions/migrations/0032_schedulea_reattribution_redesignation_tag_and_more.py similarity index 86% rename from django-backend/fecfiler/transactions/migrations/0031_schedulea_reattribution_redesignation_tag_and_more.py rename to django-backend/fecfiler/transactions/migrations/0032_schedulea_reattribution_redesignation_tag_and_more.py index e1891e345e..744667647d 100644 --- a/django-backend/fecfiler/transactions/migrations/0031_schedulea_reattribution_redesignation_tag_and_more.py +++ b/django-backend/fecfiler/transactions/migrations/0032_schedulea_reattribution_redesignation_tag_and_more.py @@ -6,7 +6,7 @@ class Migration(migrations.Migration): dependencies = [ - ('transactions', '0030_update_receipt_from_person_to_receipt_from_entity'), + ('transactions', '0031_remove_schedulee_calendar_ytd'), ] operations = [ From b67d519ffe9cc8dbbf700c2e6b9149bfb710b759 Mon Sep 17 00:00:00 2001 From: toddlees Date: Mon, 6 Nov 2023 12:36:24 -0500 Subject: [PATCH 65/70] handle no children --- django-backend/fecfiler/transactions/views.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/django-backend/fecfiler/transactions/views.py b/django-backend/fecfiler/transactions/views.py index 6694d552c5..a345c36dfa 100644 --- a/django-backend/fecfiler/transactions/views.py +++ b/django-backend/fecfiler/transactions/views.py @@ -39,7 +39,7 @@ def save_transaction(request): just its specific schedule fields and validation rules """ transaction_data = request.data - children_data = request.data["children"] + children_data = request.data.get("children", []) transaction_data["children"] = [] serializers = dict( @@ -259,7 +259,10 @@ def previous_entity(self, request): error_msg = ( "Please provide " + ",".join(missing_params) + " in query params" ) - return Response(error_msg, status=status.HTTP_400_BAD_REQUEST,) + return Response( + error_msg, + status=status.HTTP_400_BAD_REQUEST, + ) date = datetime.fromisoformat(date) From c64d38956807809ac7ce7a78724bf096a7d10be2 Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Mon, 6 Nov 2023 14:52:38 -0500 Subject: [PATCH 66/70] 1198 filing fix + logging --- .../fecfiler/web_services/dot_fec/dot_fec_submitter.py | 1 + django-backend/fecfiler/web_services/views.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/django-backend/fecfiler/web_services/dot_fec/dot_fec_submitter.py b/django-backend/fecfiler/web_services/dot_fec/dot_fec_submitter.py index 72d01bdb12..c969f5fc05 100644 --- a/django-backend/fecfiler/web_services/dot_fec/dot_fec_submitter.py +++ b/django-backend/fecfiler/web_services/dot_fec/dot_fec_submitter.py @@ -43,6 +43,7 @@ def get_submission_json(self, dot_fec_record, e_filing_password, backdoor_code=N def submit(self, dot_fec_bytes, json_payload, fec_report_id=None): response = "" if self.fec_soap_client: + logger.info(f"FEC Upload Service JSON: {json_payload}") response = self.fec_soap_client.service.upload(json_payload, dot_fec_bytes) else: """return an accepted message without reaching out to api""" diff --git a/django-backend/fecfiler/web_services/views.py b/django-backend/fecfiler/web_services/views.py index c341e07dac..cd23a6b45a 100644 --- a/django-backend/fecfiler/web_services/views.py +++ b/django-backend/fecfiler/web_services/views.py @@ -93,7 +93,7 @@ def submit_to_fec(self, request): upload_submission.id, e_filing_password, FEC_FILING_API, - True, + False, backdoor_code ) ).apply_async(retry=False) From 18ed2fe33864df4ed41d8e45f8bc01ca4e73014d Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Mon, 6 Nov 2023 16:33:57 -0500 Subject: [PATCH 67/70] logging --- .../web_services/dot_fec/dot_fec_submitter.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/django-backend/fecfiler/web_services/dot_fec/dot_fec_submitter.py b/django-backend/fecfiler/web_services/dot_fec/dot_fec_submitter.py index c969f5fc05..ff9d868aaa 100644 --- a/django-backend/fecfiler/web_services/dot_fec/dot_fec_submitter.py +++ b/django-backend/fecfiler/web_services/dot_fec/dot_fec_submitter.py @@ -1,3 +1,4 @@ +import copy import json from uuid import uuid4 as uuid from zeep import Client @@ -38,12 +39,20 @@ def get_submission_json(self, dot_fec_record, e_filing_password, backdoor_code=N json_obj["amendment_id"] = dot_fec_record.report.report_id if backdoor_code: json_obj["amendment_id"] += backdoor_code + self.log_submission_json(json_obj, dot_fec_record, backdoor_code) return json.dumps(json_obj) + def log_submission_json(self, json_obj, dot_fec_record, backdoor_code): + copy_json_obj = copy.deepcopy(json_obj) + copy_json_obj.pop("password", None) + copy_json_obj.pop("api_key", None) + if "amendment_id" in copy_json_obj and backdoor_code: + copy_json_obj["amendment_id"] = dot_fec_record.report.report_id + "xxxxx" + logger.info(f"submission json: {json.dumps(copy_json_obj)}") + def submit(self, dot_fec_bytes, json_payload, fec_report_id=None): response = "" if self.fec_soap_client: - logger.info(f"FEC Upload Service JSON: {json_payload}") response = self.fec_soap_client.service.upload(json_payload, dot_fec_bytes) else: """return an accepted message without reaching out to api""" From 185edcfa19a71684e1b47fc7b0979887474f8f18 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Mon, 13 Nov 2023 11:12:18 -0500 Subject: [PATCH 68/70] Change calculation of DSP line 6c --- .../fecfiler/web_services/summary/summary.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/django-backend/fecfiler/web_services/summary/summary.py b/django-backend/fecfiler/web_services/summary/summary.py index f2aef1ad4a..d75e1ced49 100644 --- a/django-backend/fecfiler/web_services/summary/summary.py +++ b/django-backend/fecfiler/web_services/summary/summary.py @@ -48,18 +48,12 @@ def calculate_summary_column_a(self): temp_sc10=self.get_line("SC/10"), temp_sd10=self.get_line("SD/10") ) - summary["line_6c"] = ( - summary["line_11c"] + summary["line_12"] + summary["line_13"] - + summary["line_14"] + summary["line_15"] + summary["line_16"] - + summary["line_17"] + summary.get("line_18c", Decimal("0.00")) - ) summary["line_9"] = summary["temp_sc9"] + summary["temp_sd9"] summary["line_10"] = summary["temp_sc10"] + summary["temp_sd10"] summary["line_11aiii"] = summary["line_11ai"] + summary["line_11aii"] summary["line_11d"] = ( summary["line_11aiii"] + summary["line_11b"] + summary["line_11c"] ) - summary["line_19"] = summary["line_6c"] summary["line_28d"] = ( summary["line_28a"] + summary["line_28b"] + summary["line_28c"] ) @@ -67,6 +61,12 @@ def calculate_summary_column_a(self): summary["line_34"] = summary["line_28d"] summary["line_35"] = summary["line_33"] - summary["line_34"] summary["line_37"] = summary["line_15"] + summary["line_6c"] = ( + summary["line_11d"] + summary["line_12"] + summary["line_13"] + + summary["line_14"] + summary["line_15"] + summary["line_16"] + + summary["line_17"] + summary.get("line_18c", Decimal("0.00")) + ) + summary["line_19"] = summary["line_6c"] # Remove temporary aggregations to clean up the summary for key in list(summary.keys()): @@ -107,16 +107,10 @@ def calculate_summary_column_b(self): line_29=self.get_line("SB29"), line_30b=self.get_line("SB30B"), ) - summary["line_6c"] = ( - summary["line_11c"] + summary["line_12"] + summary["line_13"] - + summary["line_14"] + summary["line_15"] + summary["line_16"] - + summary["line_17"] + summary.get("line_18c", Decimal("0.00")) - ) summary["line_11aiii"] = summary["line_11ai"] + summary["line_11aii"] summary["line_11d"] = ( summary["line_11aiii"] + summary["line_11b"] + summary["line_11c"] ) - summary["line_19"] = summary["line_6c"] summary["line_28d"] = ( summary["line_28a"] + summary["line_28b"] + summary["line_28c"] ) @@ -124,6 +118,12 @@ def calculate_summary_column_b(self): summary["line_34"] = summary["line_28d"] summary["line_35"] = summary["line_33"] - summary["line_34"] summary["line_37"] = summary["line_15"] + summary["line_6c"] = ( + summary["line_11d"] + summary["line_12"] + summary["line_13"] + + summary["line_14"] + summary["line_15"] + summary["line_16"] + + summary["line_17"] + summary.get("line_18c", Decimal("0.00")) + ) + summary["line_19"] = summary["line_6c"] return summary From 6a8fcbb60d2ff0f21106dcdea43baf7be3b5c841 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Mon, 13 Nov 2023 11:18:32 -0500 Subject: [PATCH 69/70] Fix unit test for summary calculation --- .../fecfiler/web_services/summary/test_summary.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/django-backend/fecfiler/web_services/summary/test_summary.py b/django-backend/fecfiler/web_services/summary/test_summary.py index 39e2fbfb78..2ee4184504 100644 --- a/django-backend/fecfiler/web_services/summary/test_summary.py +++ b/django-backend/fecfiler/web_services/summary/test_summary.py @@ -20,7 +20,7 @@ def test_calculate_summary_column_a(self): f3x = Report.objects.get(id="b6d60d2d-d926-4e89-ad4b-c47d152a66ae") summary_service = SummaryService(f3x) summary = summary_service.calculate_summary() - self.assertEqual(summary["a"]["line_6c"], Decimal("7636.73")) + self.assertEqual(summary["a"]["line_6c"], Decimal("18085.17")) self.assertEqual(summary["a"]["line_9"], Decimal("225.00")) self.assertEqual(summary["a"]["line_10"], Decimal("250.00")) self.assertEqual(summary["a"]["line_11ai"], Decimal("10000.23")) @@ -41,7 +41,7 @@ def test_calculate_summary_column_a(self): self.assertEqual(summary["a"]["line_15"], summary["a"]["line_37"]) self.assertEqual(summary["a"]["line_16"], Decimal("16.00")) self.assertEqual(summary["a"]["line_17"], Decimal("1000.00")) - self.assertEqual(summary["a"]["line_19"], Decimal("7636.73")) + self.assertEqual(summary["a"]["line_19"], Decimal("18085.17")) self.assertEqual(summary["a"]["line_21b"], Decimal("150.00")) self.assertEqual(summary["a"]["line_22"], Decimal("22.00")) self.assertEqual(summary["a"]["line_23"], Decimal("14.00")) @@ -80,7 +80,7 @@ def test_calculate_summary_column_b(self): t = Transaction.objects.get(id="aaaaaaaa-4d75-46f0-bce2-111000000001") self.assertEqual(t.itemized, False) - self.assertEqual(summary["b"]["line_6c"], Decimal("8336.73")) + self.assertEqual(summary["b"]["line_6c"], Decimal("18985.17")) self.assertEqual(summary["b"]["line_11ai"], Decimal("10000.23")) self.assertEqual(summary["b"]["line_11aii"], Decimal("103.77")) self.assertEqual(summary["b"]["line_11aiii"], Decimal(10104.00)) @@ -96,7 +96,7 @@ def test_calculate_summary_column_b(self): self.assertEqual(summary["b"]["line_15"], Decimal("2225.79")) self.assertEqual(summary["b"]["line_16"], Decimal("116.00")) self.assertEqual(summary["b"]["line_17"], Decimal("1100.00")) - self.assertEqual(summary["b"]["line_19"], Decimal("8336.73")) + self.assertEqual(summary["b"]["line_19"], Decimal("18985.17")) self.assertEqual(summary["b"]["line_21b"], Decimal("250.00")) self.assertEqual(summary["b"]["line_23"], Decimal("64.00")) self.assertEqual(summary["b"]["line_26"], Decimal("61.00")) From 8b16e327de126a9f13429ad6859816df68e24708 Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Tue, 14 Nov 2023 15:06:53 -0500 Subject: [PATCH 70/70] 604 add stubbed committee data for UAT testers --- .../committee_accounts.json | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/django-backend/fecfiler/openfec/test_efo_mock_data/committee_accounts.json b/django-backend/fecfiler/openfec/test_efo_mock_data/committee_accounts.json index d323067834..7f652d7824 100644 --- a/django-backend/fecfiler/openfec/test_efo_mock_data/committee_accounts.json +++ b/django-backend/fecfiler/openfec/test_efo_mock_data/committee_accounts.json @@ -484,5 +484,59 @@ "custodian_zip": "37208", "custodian_name_title": "President", "custodian_phone": "8175212197" + }, + { + "committee_id": "C00100933", + "name": "Sarah for Congress", + "committee_type_full": "House", + "street_1": "1050 First Street, NE ", + "city": "Washington", + "state": "District of Columbia", + "zip": "20002", + "email": "fakeemail@testaccount.com", + "treasurer_name_1": "Ryan", + "treasurer_name_2": "Lanz", + "treasurer_street_1": "1 Apple Park Way", + "treasurer_city": "Cupertino", + "treasurer_state": "CA", + "treasurer_zip": "95014", + "treasurer_phone": "8175212197", + "custodian_name_full": "Clark, Paul", + "custodian_name_1": "Paul", + "custodian_name_2": "Clark", + "custodian_street_1": "1 Apple Park Way", + "custodian_city": "Cupertino", + "custodian_state": "CA", + "custodian_zip": "95014", + "custodian_name_title": "Custodian of record", + "filing_frequency": "A", + "committee_type": "H" + }, + { + "committee_id": "C00100941", + "name": "Michael for Congress", + "committee_type_full": "House", + "street_1": "1050 First Street, NE ", + "city": "Washington", + "state": "District of Columbia", + "zip": "20002", + "email": "fakeemail@testaccount.com", + "treasurer_name_1": "Ryan", + "treasurer_name_2": "Lanz", + "treasurer_street_1": "1 Apple Park Way", + "treasurer_city": "Cupertino", + "treasurer_state": "CA", + "treasurer_zip": "95014", + "treasurer_phone": "8175212197", + "custodian_name_full": "Colucci, Marlene", + "custodian_name_1": "Marlene", + "custodian_name_2": "Colucci", + "custodian_street_1": "1 Apple Park Way", + "custodian_city": "Cupertino", + "custodian_state": "CA", + "custodian_zip": "95014", + "custodian_name_title": "Custodian of record", + "filing_frequency": "A", + "committee_type": "H" } ] \ No newline at end of file