From 385aeeae07cb72f0b374a9f59e81d6123bc601da Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Fri, 17 Nov 2023 14:31:10 -0500 Subject: [PATCH 001/109] Add reattribution/redesignation fkey --- .../0034_add_reattr_redes_foreign_key.py | 24 +++++++++++++++++++ .../fecfiler/transactions/models.py | 7 ++++++ .../fecfiler/transactions/serializers.py | 10 ++++++++ django-backend/fecfiler/transactions/views.py | 1 + 4 files changed, 42 insertions(+) create mode 100644 django-backend/fecfiler/transactions/migrations/0034_add_reattr_redes_foreign_key.py diff --git a/django-backend/fecfiler/transactions/migrations/0034_add_reattr_redes_foreign_key.py b/django-backend/fecfiler/transactions/migrations/0034_add_reattr_redes_foreign_key.py new file mode 100644 index 0000000000..79599bf404 --- /dev/null +++ b/django-backend/fecfiler/transactions/migrations/0034_add_reattr_redes_foreign_key.py @@ -0,0 +1,24 @@ +# Generated by Django 4.1.3 on 2023-11-17 20:42 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("transactions", "0033_remove_receipt_line_number_from_loans_by_committee"), + ] + + operations = [ + migrations.AddField( + model_name="transaction", + name="reatt_redes", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="reatt_redes_associations", + to="transactions.transaction", + ), + ), + ] diff --git a/django-backend/fecfiler/transactions/models.py b/django-backend/fecfiler/transactions/models.py index aafa9becbd..06d5f5bfb9 100644 --- a/django-backend/fecfiler/transactions/models.py +++ b/django-backend/fecfiler/transactions/models.py @@ -45,6 +45,13 @@ class Transaction(SoftDeleteModel, CommitteeOwnedModel, ReportMixin): blank=True, related_name="loan_associations", ) + reatt_redes = models.ForeignKey( # reattribution/redesignation + "self", + on_delete=models.CASCADE, + null=True, + blank=True, + related_name="reatt_redes_associations", + ) # The _form_type value in the db may or may not be correct based on whether # the transaction is itemized or not. For some transactions, the form_type diff --git a/django-backend/fecfiler/transactions/serializers.py b/django-backend/fecfiler/transactions/serializers.py index f5da36b710..8d1c8a974f 100644 --- a/django-backend/fecfiler/transactions/serializers.py +++ b/django-backend/fecfiler/transactions/serializers.py @@ -4,6 +4,7 @@ from fecfiler.contacts.serializers import LinkedContactSerializerMixin from fecfiler.memo_text.serializers import LinkedMemoTextSerializerMixin from fecfiler.validation.serializers import FecSchemaValidatorSerializerMixin +from fecfiler.reports.serializers import ReportSerializer from rest_framework.exceptions import ValidationError from rest_framework.serializers import empty from collections import OrderedDict @@ -131,6 +132,8 @@ class TransactionSerializer( schedule_d = ScheduleDSerializer(required=False) schedule_e = ScheduleESerializer(required=False) + report = ReportSerializer(read_only=True) + back_reference_tran_id_number = CharField( required=False, allow_null=True, read_only=True ) @@ -168,12 +171,14 @@ def get_fields(): "transaction", "debt_associations", "loan_associations", + "reatt_redes_associations", # reattribution/redesignation "_form_type", ] ] + [ "parent_transaction_id", "debt_id", "loan_id", + "reatt_redes_id", "report_id", "contact_1_id", "contact_2_id", @@ -280,6 +285,11 @@ def to_representation(self, instance): representation["debt"] = TransactionSerializer().to_representation( instance.debt ) + # represent original reattribution/redesignation transaction + if instance.reatt_redes: + representation["reatt_redes"] = TransactionSerializer().to_representation( + instance.reatt_redes + ) return representation diff --git a/django-backend/fecfiler/transactions/views.py b/django-backend/fecfiler/transactions/views.py index 3f9e31d233..b65e25ee05 100644 --- a/django-backend/fecfiler/transactions/views.py +++ b/django-backend/fecfiler/transactions/views.py @@ -239,6 +239,7 @@ def save_transaction(self, transaction_data, request): ) transaction_data["debt"] = transaction_data.get("debt_id", None) transaction_data["loan"] = transaction_data.get("loan_id", None) + transaction_data["reatt_redes"] = transaction_data.get("reatt_redes_id", None) if transaction_data.get("form_type"): transaction_data["_form_type"] = transaction_data["form_type"] From 2a35c5d0cf6ff555522d5472f1cd6964b60193cd Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Fri, 17 Nov 2023 14:35:44 -0500 Subject: [PATCH 002/109] Fix linting error reported by SonarCloud --- django-backend/fecfiler/transactions/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django-backend/fecfiler/transactions/serializers.py b/django-backend/fecfiler/transactions/serializers.py index 8d1c8a974f..17d4a7cd81 100644 --- a/django-backend/fecfiler/transactions/serializers.py +++ b/django-backend/fecfiler/transactions/serializers.py @@ -171,7 +171,7 @@ def get_fields(): "transaction", "debt_associations", "loan_associations", - "reatt_redes_associations", # reattribution/redesignation + "reatt_redes_associations", # reattribution/redesignation "_form_type", ] ] + [ From faf4b81df34fef52076a96ddc0b31fb72ee4022a Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Mon, 20 Nov 2023 10:24:41 -0500 Subject: [PATCH 003/109] WIP --- django-backend/fecfiler/reports/models.py | 7 +- .../fecfiler/web_services/summary/summary.py | 13 +- .../fecfiler/web_services/summary/tasks.py | 374 ++++++++++-------- 3 files changed, 222 insertions(+), 172 deletions(-) diff --git a/django-backend/fecfiler/reports/models.py b/django-backend/fecfiler/reports/models.py index 70e42d424a..fb8b7cdcc7 100644 --- a/django-backend/fecfiler/reports/models.py +++ b/django-backend/fecfiler/reports/models.py @@ -42,6 +42,12 @@ class Report(SoftDeleteModel, CommitteeOwnedModel): treasurer_suffix = models.TextField(null=True, blank=True) date_signed = models.DateField(null=True, blank=True) calculation_status = models.CharField(max_length=255, null=True, blank=True) + calculation_token = models.UUIDField( + null=True, + blank=True, + editable=True, + unique=True + ) confirmation_email_1 = models.EmailField( max_length=44, null=True, blank=True, default=None @@ -195,7 +201,6 @@ def save(self, *args, **kwargs): report_year = report_date.year reports_to_flag_for_recalculation = Report.objects.filter( - ~models.Q(upload_submission__fec_status=models.Value("ACCEPTED")), committee_account=committee, coverage_from_date__year=report_year, coverage_from_date__gte=report_date, diff --git a/django-backend/fecfiler/web_services/summary/summary.py b/django-backend/fecfiler/web_services/summary/summary.py index 094423d7e4..aac5843314 100644 --- a/django-backend/fecfiler/web_services/summary/summary.py +++ b/django-backend/fecfiler/web_services/summary/summary.py @@ -1,5 +1,6 @@ from decimal import Decimal from fecfiler.transactions.models import Transaction +from fecfiler.reports.models import Report from django.db.models import Q, Sum from django.db.models.functions import Coalesce import logging @@ -10,6 +11,12 @@ class SummaryService: def __init__(self, report) -> None: self.report = report + self.previous_f3x_report = Report.objects.filter( + ~Q(id=report.id), + form_3x__isnull=False, + coverage_from_date__year=report.coverage_from_date.year, + coverage_through_date__lt=report.coverage_from_date + ).order_by("-coverage_through_date").first().form_3x def calculate_summary(self): summary = { @@ -49,8 +56,7 @@ def calculate_summary_column_a(self): temp_sc10=self.get_line("SC/10", field="loan_balance"), temp_sd10=self.get_line("SD10", field="balance_at_close") ) - summary["line_6a"] = Decimal(0) # Stubbed out until a future ticket - summary["line_6b"] = Decimal(0) # Stubbed out until a future ticket + summary["line_6b"] = self.previous_f3x_report.L8_cash_on_hand_close_period summary["line_9"] = ( summary["temp_sc9"] + summary["temp_sd9"] @@ -196,8 +202,7 @@ def calculate_summary_column_b(self): line_29=self.get_line("SB29"), line_30b=self.get_line("SB30B"), ) - summary["line_6a"] = Decimal(0) # Stubbed out until a future ticket - summary["line_6b"] = Decimal(0) # Stubbed out until a future ticket + summary["line_6a"] = self.previous_f3x_report.L8_cash_on_hand_at_close_ytd summary["line_11aiii"] = ( summary["line_11ai"] + summary["line_11aii"] diff --git a/django-backend/fecfiler/web_services/summary/tasks.py b/django-backend/fecfiler/web_services/summary/tasks.py index f0928a487c..f91f726a7a 100644 --- a/django-backend/fecfiler/web_services/summary/tasks.py +++ b/django-backend/fecfiler/web_services/summary/tasks.py @@ -2,6 +2,7 @@ from celery import shared_task from fecfiler.reports.models import Report from .summary import SummaryService +import uuid import logging @@ -19,175 +20,214 @@ def __str__(self): return str(self.value) +def get_reports_to_calculate(primary_report): + report_year = primary_report.coverage_from_date.year + reports_in_year = Report.objects.filter( + coverage_from_date__year=report_year, + coverage_through_date__lte=primary_report.coverage_through_date + ).order_by("-coverage_through_date") + + reports_to_recalculate = [] + for report in reports_in_year: + if report.calculation_status == None: + reports_to_recalculate.insert(0, report) + else: + break + + return reports_to_recalculate + + @shared_task def calculate_summary(report_id): try: - report = Report.objects.get(id=report_id) + primary_report = Report.objects.get(id=report_id) except Exception: return None - report.calculation_status = CalculationState.CALCULATING - report.save() - - summary_service = SummaryService(report) - summary = summary_service.calculate_summary() - a = summary["a"] - b = summary["b"] - - # line 6a - report.form_3x.L6a_cash_on_hand_jan_1_ytd = b.get("line_6a", 0) - report.form_3x.L6a_year_for_above_ytd = b.get("line_6a", 0) - # line 6b - report.form_3x.L6b_cash_on_hand_beginning_period = a.get("line_6b", 0) - # line 6c - report.form_3x.L6c_total_receipts_period = a.get("line_6c", 0) - report.form_3x.L6c_total_receipts_ytd = b.get("line_6c", 0) - # line 6d - report.form_3x.L6d_subtotal_period = a.get("line_6d", 0) - report.form_3x.L6d_subtotal_ytd = b.get("line_6d", 0) - # line 7 - report.form_3x.L7_total_disbursements_period = a.get("line_7", 0) - report.form_3x.L7_total_disbursements_ytd = b.get("line_7", 0) - # line 8 - report.form_3x.L8_cash_on_hand_at_close_period = a.get("line_8", 0) - report.form_3x.L8_cash_on_hand_close_ytd = b.get("line_8", 0) - # line 9 - report.form_3x.L9_debts_to_period = a.get("line_9", 0) - # line 10 - report.form_3x.L10_debts_by_period = a.get("line_10", 0) - # line 11ai - report.form_3x.L11ai_itemized_period = a.get("line_11ai", 0) - report.form_3x.L11ai_itemized_ytd = b.get("line_11ai", 0) - # line 11aii - report.form_3x.L11aii_unitemized_period = a.get("line_11aii", 0) - report.form_3x.L11aii_unitemized_ytd = b.get("line_11aii", 0) - # line 11aiii - report.form_3x.L11aiii_total_period = a.get("line_11aiii", 0) - report.form_3x.L11aiii_total_ytd = b.get("line_11aiii", 0) - # line 11b - report.form_3x.L11b_political_party_committees_period = a.get("line_11b", 0) - report.form_3x.L11b_political_party_committees_ytd = b.get("line_11b", 0) - # line 11c - report.form_3x.L11c_other_political_committees_pacs_period = a.get("line_11c", 0) - report.form_3x.L11c_other_political_committees_pacs_ytd = b.get("line_11c", 0) - # line 11d - report.form_3x.L11d_total_contributions_period = a.get("line_11d", 0) - report.form_3x.L11d_total_contributions_ytd = b.get("line_11d", 0) - # line 12 - report.form_3x.L12_transfers_from_affiliated_other_party_cmtes_period = a.get("line_12", 0) # noqa: E501 - report.form_3x.L12_transfers_from_affiliated_other_party_cmtes_ytd = b.get("line_12", 0) # noqa: E501 - # line 13 - report.form_3x.L13_all_loans_received_period = a.get("line_13", 0) - report.form_3x.L13_all_loans_received_ytd = b.get("line_13", 0) - # line 14 - report.form_3x.L14_loan_repayments_received_period = a.get("line_14", 0) - report.form_3x.L14_loan_repayments_received_ytd = b.get("line_14", 0) - # line 15 - report.form_3x.L15_offsets_to_operating_expenditures_refunds_period = a.get("line_15", 0) # noqa: E501 - report.form_3x.L15_offsets_to_operating_expenditures_refunds_ytd = b.get("line_15", 0) - # line 16 - report.form_3x.L16_refunds_of_federal_contributions_period = a.get("line_16", 0) - report.form_3x.L16_refunds_of_federal_contributions_ytd = b.get("line_16", 0) - # line 17 - report.form_3x.L17_other_federal_receipts_dividends_period = a.get("line_17", 0) - report.form_3x.L17_other_federal_receipts_dividends_ytd = b.get("line_17", 0) - # line 18a - report.form_3x.L18a_transfers_from_nonfederal_account_h3_period = a.get("line_18a", 0) - report.form_3x.L18a_transfers_from_nonfederal_account_h3_ytd = b.get("line_18a", 0) - # line 18b - report.form_3x.L18b_transfers_from_nonfederal_levin_h5_period = a.get("line_18b", 0) - report.form_3x.L18b_transfers_from_nonfederal_levin_h5_ytd = b.get("line_18b", 0) - # line 18c - report.form_3x.L18c_total_nonfederal_transfers_18a_18b_period = a.get("line_18c", 0) - report.form_3x.L18c_total_nonfederal_transfers_18a_18b_ytd = b.get("line_18c", 0) - # line 19 - report.form_3x.L19_total_receipts_period = a.get("line_19", 0) - report.form_3x.L19_total_receipts_ytd = b.get("line_19", 0) - # line 20 - report.form_3x.L20_total_federal_receipts_period = a.get("line_20", 0) - report.form_3x.L20_total_federal_receipts_ytd = b.get("line_20", 0) - # line 21ai - report.form_3x.L21ai_federal_share_period = a.get("line_21ai", 0) - report.form_3x.L21ai_federal_share_ytd = b.get("line_21ai", 0) - # line 21aii - report.form_3x.L21aii_nonfederal_share_period = a.get("line_21aii", 0) - report.form_3x.L21aii_nonfederal_share_ytd = b.get("line_21aii", 0) - # line 21b - report.form_3x.L21b_other_federal_operating_expenditures_period = a.get("line_21b", 0) - report.form_3x.L21b_other_federal_operating_expenditures_ytd = b.get("line_21b", 0) - # line 21c - report.form_3x.L21c_total_operating_expenditures_ytd = b.get("line_21c", 0) - report.form_3x.L21c_total_operating_expenditures_period = a.get("line_21c", 0) - # line 22 - report.form_3x.L22_transfers_to_affiliated_other_party_cmtes_period = a.get("line_22", 0) # noqa: E501 - report.form_3x.L22_transfers_to_affiliated_other_party_cmtes_ytd = b.get("line_22", 0) - # line 23 - report.form_3x.L23_contributions_to_federal_candidates_cmtes_period = a.get("line_23", 0) # noqa: E501 - report.form_3x.L23_contributions_to_federal_candidates_cmtes_ytd = b.get("line_23", 0) - # line 24 - report.form_3x.L24_independent_expenditures_period = a.get("line_24", 0) - report.form_3x.L24_independent_expenditures_ytd = b.get("line_24", 0) - # line 25 - report.form_3x.L25_coordinated_expend_made_by_party_cmtes_period = a.get("line_25", 0) - report.form_3x.L25_coordinated_expend_made_by_party_cmtes_ytd = b.get("line_25", 0) - # line 26 - report.form_3x.L26_loan_repayments_period = a.get("line_26", 0) - report.form_3x.L26_loan_repayments_made_ytd = b.get("line_26", 0) - # line 27 - report.form_3x.L27_loans_made_period = a.get("line_27", 0) - report.form_3x.L27_loans_made_ytd = b.get("line_27", 0) - # line 28a - report.form_3x.L28a_individuals_persons_period = a.get("line_28a", 0) - report.form_3x.L28a_individuals_persons_ytd = b.get("line_28a", 0) - # line 28b - report.form_3x.L28b_political_party_committees_period = a.get("line_28b", 0) - report.form_3x.L28b_political_party_committees_ytd = b.get("line_28b", 0) - # line 28c - report.form_3x.L28c_other_political_committees_period = a.get("line_28c", 0) - report.form_3x.L28c_other_political_committees_ytd = b.get("line_28c", 0) - # line 28d - report.form_3x.L28d_total_contributions_refunds_period = a.get("line_28d", 0) - report.form_3x.L28d_total_contributions_refunds_ytd = b.get("line_28d", 0) - # line 29 - report.form_3x.L29_other_disbursements_period = a.get("line_29", 0) - report.form_3x.L29_other_disbursements_ytd = b.get("line_29", 0) - # line 30ai - report.form_3x.L30ai_shared_federal_activity_h6_fed_share_period = a.get("line_30ai", 0) # noqa: E501 - report.form_3x.L30ai_shared_federal_activity_h6_fed_share_ytd = b.get("line_30ai", 0) - # line 30aii - report.form_3x.L30aii_shared_federal_activity_h6_nonfed_period = a.get("line_30aii", 0) # noqa: E501 - report.form_3x.L30aii_shared_federal_activity_h6_nonfed_ytd = b.get("line_30aii", 0) - # line 30b - report.form_3x.L30b_nonallocable_fed_election_activity_period = a.get("line_30b", 0) - report.form_3x.L30b_nonallocable_fed_election_activity_ytd = b.get("line_30b", 0) - # line 30c - report.form_3x.L30c_total_federal_election_activity_period = a.get("line_30c", 0) - report.form_3x.L30c_total_federal_election_activity_ytd = b.get("line_30c", 0) - # line 31 - report.form_3x.L31_total_disbursements_period = a.get("line_31", 0) - report.form_3x.L31_total_disbursements_ytd = b.get("line_31", 0) - # line 32 - report.form_3x.L32_total_federal_disbursements_period = a.get("line_32", 0) - report.form_3x.L32_total_federal_disbursements_ytd = b.get("line_32", 0) - # line 33 - report.form_3x.L33_total_contributions_period = a.get("line_33", 0) - report.form_3x.L33_total_contributions_ytd = b.get("line_33", 0) - # line 34 - report.form_3x.L34_total_contribution_refunds_period = a.get("line_34", 0) - report.form_3x.L34_total_contribution_refunds_ytd = b.get("line_34", 0) - # line 35 - report.form_3x.L35_net_contributions_period = a.get("line_35", 0) - report.form_3x.L35_net_contributions_ytd = b.get("line_35", 0) - # line 36 - report.form_3x.L36_total_federal_operating_expenditures_period = a.get("line_36", 0) - report.form_3x.L36_total_federal_operating_expenditures_ytd = b.get("line_36", 0) - # line 37 - report.form_3x.L37_offsets_to_operating_expenditures_period = a.get("line_37", 0) - report.form_3x.L37_offsets_to_operating_expenditures_ytd = b.get("line_37", 0) - # line 38 - report.form_3x.L38_net_operating_expenditures_period = a.get("line_38", 0) - report.form_3x.L38_net_operating_expenditures_ytd = b.get("line_38", 0) - - report.form_3x.save() - report.calculation_status = CalculationState.SUCCEEDED - report.save() - return report.id + + reports_to_recalculate = get_reports_to_calculate(primary_report) + calculation_token = uuid.uuid4() + reports_to_recalculate.update( + calculation_token=calculation_token, + calculation_status=CalculationState.CALCULATING + ) + + for report in reports_to_recalculate: + summary_service = SummaryService(report) + summary = summary_service.calculate_summary() + a = summary["a"] + b = summary["b"] + + # line 6a + report.form_3x.L6a_cash_on_hand_jan_1_ytd = b.get("line_6a", 0) + report.form_3x.L6a_year_for_above_ytd = b.get("line_6a", 0) + # line 6b + report.form_3x.L6b_cash_on_hand_beginning_period = a.get("line_6b", 0) + # line 6c + report.form_3x.L6c_total_receipts_period = a.get("line_6c", 0) + report.form_3x.L6c_total_receipts_ytd = b.get("line_6c", 0) + # line 6d + report.form_3x.L6d_subtotal_period = a.get("line_6d", 0) + report.form_3x.L6d_subtotal_ytd = b.get("line_6d", 0) + # line 7 + report.form_3x.L7_total_disbursements_period = a.get("line_7", 0) + report.form_3x.L7_total_disbursements_ytd = b.get("line_7", 0) + # line 8 + report.form_3x.L8_cash_on_hand_at_close_period = a.get("line_8", 0) + report.form_3x.L8_cash_on_hand_close_ytd = b.get("line_8", 0) + # line 9 + report.form_3x.L9_debts_to_period = a.get("line_9", 0) + # line 10 + report.form_3x.L10_debts_by_period = a.get("line_10", 0) + # line 11ai + report.form_3x.L11ai_itemized_period = a.get("line_11ai", 0) + report.form_3x.L11ai_itemized_ytd = b.get("line_11ai", 0) + # line 11aii + report.form_3x.L11aii_unitemized_period = a.get("line_11aii", 0) + report.form_3x.L11aii_unitemized_ytd = b.get("line_11aii", 0) + # line 11aiii + report.form_3x.L11aiii_total_period = a.get("line_11aiii", 0) + report.form_3x.L11aiii_total_ytd = b.get("line_11aiii", 0) + # line 11b + report.form_3x.L11b_political_party_committees_period = a.get("line_11b", 0) + report.form_3x.L11b_political_party_committees_ytd = b.get("line_11b", 0) + # line 11c + report.form_3x.L11c_other_political_committees_pacs_period = a.get("line_11c", 0) + report.form_3x.L11c_other_political_committees_pacs_ytd = b.get("line_11c", 0) + # line 11d + report.form_3x.L11d_total_contributions_period = a.get("line_11d", 0) + report.form_3x.L11d_total_contributions_ytd = b.get("line_11d", 0) + # line 12 + report.form_3x.L12_transfers_from_affiliated_other_party_cmtes_period = a.get("line_12", 0) # noqa: E501 + report.form_3x.L12_transfers_from_affiliated_other_party_cmtes_ytd = b.get("line_12", 0) # noqa: E501 + # line 13 + report.form_3x.L13_all_loans_received_period = a.get("line_13", 0) + report.form_3x.L13_all_loans_received_ytd = b.get("line_13", 0) + # line 14 + report.form_3x.L14_loan_repayments_received_period = a.get("line_14", 0) + report.form_3x.L14_loan_repayments_received_ytd = b.get("line_14", 0) + # line 15 + report.form_3x.L15_offsets_to_operating_expenditures_refunds_period = a.get("line_15", 0) # noqa: E501 + report.form_3x.L15_offsets_to_operating_expenditures_refunds_ytd = b.get("line_15", 0) + # line 16 + report.form_3x.L16_refunds_of_federal_contributions_period = a.get("line_16", 0) + report.form_3x.L16_refunds_of_federal_contributions_ytd = b.get("line_16", 0) + # line 17 + report.form_3x.L17_other_federal_receipts_dividends_period = a.get("line_17", 0) + report.form_3x.L17_other_federal_receipts_dividends_ytd = b.get("line_17", 0) + # line 18a + report.form_3x.L18a_transfers_from_nonfederal_account_h3_period = a.get("line_18a", 0) + report.form_3x.L18a_transfers_from_nonfederal_account_h3_ytd = b.get("line_18a", 0) + # line 18b + report.form_3x.L18b_transfers_from_nonfederal_levin_h5_period = a.get("line_18b", 0) + report.form_3x.L18b_transfers_from_nonfederal_levin_h5_ytd = b.get("line_18b", 0) + # line 18c + report.form_3x.L18c_total_nonfederal_transfers_18a_18b_period = a.get("line_18c", 0) + report.form_3x.L18c_total_nonfederal_transfers_18a_18b_ytd = b.get("line_18c", 0) + # line 19 + report.form_3x.L19_total_receipts_period = a.get("line_19", 0) + report.form_3x.L19_total_receipts_ytd = b.get("line_19", 0) + # line 20 + report.form_3x.L20_total_federal_receipts_period = a.get("line_20", 0) + report.form_3x.L20_total_federal_receipts_ytd = b.get("line_20", 0) + # line 21ai + report.form_3x.L21ai_federal_share_period = a.get("line_21ai", 0) + report.form_3x.L21ai_federal_share_ytd = b.get("line_21ai", 0) + # line 21aii + report.form_3x.L21aii_nonfederal_share_period = a.get("line_21aii", 0) + report.form_3x.L21aii_nonfederal_share_ytd = b.get("line_21aii", 0) + # line 21b + report.form_3x.L21b_other_federal_operating_expenditures_period = a.get("line_21b", 0) + report.form_3x.L21b_other_federal_operating_expenditures_ytd = b.get("line_21b", 0) + # line 21c + report.form_3x.L21c_total_operating_expenditures_ytd = b.get("line_21c", 0) + report.form_3x.L21c_total_operating_expenditures_period = a.get("line_21c", 0) + # line 22 + report.form_3x.L22_transfers_to_affiliated_other_party_cmtes_period = a.get("line_22", 0) # noqa: E501 + report.form_3x.L22_transfers_to_affiliated_other_party_cmtes_ytd = b.get("line_22", 0) + # line 23 + report.form_3x.L23_contributions_to_federal_candidates_cmtes_period = a.get("line_23", 0) # noqa: E501 + report.form_3x.L23_contributions_to_federal_candidates_cmtes_ytd = b.get("line_23", 0) + # line 24 + report.form_3x.L24_independent_expenditures_period = a.get("line_24", 0) + report.form_3x.L24_independent_expenditures_ytd = b.get("line_24", 0) + # line 25 + report.form_3x.L25_coordinated_expend_made_by_party_cmtes_period = a.get("line_25", 0) + report.form_3x.L25_coordinated_expend_made_by_party_cmtes_ytd = b.get("line_25", 0) + # line 26 + report.form_3x.L26_loan_repayments_period = a.get("line_26", 0) + report.form_3x.L26_loan_repayments_made_ytd = b.get("line_26", 0) + # line 27 + report.form_3x.L27_loans_made_period = a.get("line_27", 0) + report.form_3x.L27_loans_made_ytd = b.get("line_27", 0) + # line 28a + report.form_3x.L28a_individuals_persons_period = a.get("line_28a", 0) + report.form_3x.L28a_individuals_persons_ytd = b.get("line_28a", 0) + # line 28b + report.form_3x.L28b_political_party_committees_period = a.get("line_28b", 0) + report.form_3x.L28b_political_party_committees_ytd = b.get("line_28b", 0) + # line 28c + report.form_3x.L28c_other_political_committees_period = a.get("line_28c", 0) + report.form_3x.L28c_other_political_committees_ytd = b.get("line_28c", 0) + # line 28d + report.form_3x.L28d_total_contributions_refunds_period = a.get("line_28d", 0) + report.form_3x.L28d_total_contributions_refunds_ytd = b.get("line_28d", 0) + # line 29 + report.form_3x.L29_other_disbursements_period = a.get("line_29", 0) + report.form_3x.L29_other_disbursements_ytd = b.get("line_29", 0) + # line 30ai + report.form_3x.L30ai_shared_federal_activity_h6_fed_share_period = a.get("line_30ai", 0) # noqa: E501 + report.form_3x.L30ai_shared_federal_activity_h6_fed_share_ytd = b.get("line_30ai", 0) + # line 30aii + report.form_3x.L30aii_shared_federal_activity_h6_nonfed_period = a.get("line_30aii", 0) # noqa: E501 + report.form_3x.L30aii_shared_federal_activity_h6_nonfed_ytd = b.get("line_30aii", 0) + # line 30b + report.form_3x.L30b_nonallocable_fed_election_activity_period = a.get("line_30b", 0) + report.form_3x.L30b_nonallocable_fed_election_activity_ytd = b.get("line_30b", 0) + # line 30c + report.form_3x.L30c_total_federal_election_activity_period = a.get("line_30c", 0) + report.form_3x.L30c_total_federal_election_activity_ytd = b.get("line_30c", 0) + # line 31 + report.form_3x.L31_total_disbursements_period = a.get("line_31", 0) + report.form_3x.L31_total_disbursements_ytd = b.get("line_31", 0) + # line 32 + report.form_3x.L32_total_federal_disbursements_period = a.get("line_32", 0) + report.form_3x.L32_total_federal_disbursements_ytd = b.get("line_32", 0) + # line 33 + report.form_3x.L33_total_contributions_period = a.get("line_33", 0) + report.form_3x.L33_total_contributions_ytd = b.get("line_33", 0) + # line 34 + report.form_3x.L34_total_contribution_refunds_period = a.get("line_34", 0) + report.form_3x.L34_total_contribution_refunds_ytd = b.get("line_34", 0) + # line 35 + report.form_3x.L35_net_contributions_period = a.get("line_35", 0) + report.form_3x.L35_net_contributions_ytd = b.get("line_35", 0) + # line 36 + report.form_3x.L36_total_federal_operating_expenditures_period = a.get("line_36", 0) + report.form_3x.L36_total_federal_operating_expenditures_ytd = b.get("line_36", 0) + # line 37 + report.form_3x.L37_offsets_to_operating_expenditures_period = a.get("line_37", 0) + report.form_3x.L37_offsets_to_operating_expenditures_ytd = b.get("line_37", 0) + # line 38 + report.form_3x.L38_net_operating_expenditures_period = a.get("line_38", 0) + report.form_3x.L38_net_operating_expenditures_ytd = b.get("line_38", 0) + + report.form_3x.save() + + # Set the calculation status to SUCCEEDED *only* + # if the recalculation still belongs to this task + updated = bool( + Report.objects.filter( + id=report.id, + calculation_status=CalculationState.CALCULATING, + calculation_token=calculation_token + ).update( + calculation_status=CalculationState.SUCCEEDED, + calculation_token=None + ) + ) + + # If we failed to update, do not calculate any further reports + if not updated: + break + + return primary_report.id From 9df1186e54fe7013ab6551baf93a1449138c894c Mon Sep 17 00:00:00 2001 From: toddlees Date: Tue, 21 Nov 2023 17:44:10 -0500 Subject: [PATCH 004/109] add loan agreement key to serializer --- django-backend/fecfiler/transactions/serializers.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/django-backend/fecfiler/transactions/serializers.py b/django-backend/fecfiler/transactions/serializers.py index f5da36b710..3f5fa83d6a 100644 --- a/django-backend/fecfiler/transactions/serializers.py +++ b/django-backend/fecfiler/transactions/serializers.py @@ -224,6 +224,12 @@ def to_representation(self, instance): if not representation.get(property): representation[property] = schedule_b[property] if schedule_c: + loan_agreement = instance.children.filter( + transaction_type_identifier="C1_LOAN_AGREEMENT" + ).first() + representation["loan_agreement"] = ( + loan_agreement.id if loan_agreement else None + ) for property in schedule_c: if not representation.get(property): representation[property] = schedule_c[property] From 547bb712b8e6a71e3ce70d00f147b5fd8a7c4028 Mon Sep 17 00:00:00 2001 From: toddlees Date: Tue, 21 Nov 2023 18:24:06 -0500 Subject: [PATCH 005/109] serialize name --- django-backend/fecfiler/transactions/serializers.py | 2 ++ django-backend/fecfiler/transactions/views.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/django-backend/fecfiler/transactions/serializers.py b/django-backend/fecfiler/transactions/serializers.py index 3f5fa83d6a..fc4c3c982a 100644 --- a/django-backend/fecfiler/transactions/serializers.py +++ b/django-backend/fecfiler/transactions/serializers.py @@ -139,6 +139,7 @@ class TransactionSerializer( ) form_type = CharField(required=False, allow_null=True) itemized = BooleanField(read_only=True) + name = CharField(read_only=True) date = DateField(read_only=True) amount = DecimalField(max_digits=11, decimal_places=2, read_only=True) aggregate = DecimalField(max_digits=11, decimal_places=2, read_only=True) @@ -185,6 +186,7 @@ def get_fields(): "itemized", "fields_to_validate", "schema_name", + "name", "date", "amount", "aggregate", diff --git a/django-backend/fecfiler/transactions/views.py b/django-backend/fecfiler/transactions/views.py index b6ec48e95a..ecf552ba86 100644 --- a/django-backend/fecfiler/transactions/views.py +++ b/django-backend/fecfiler/transactions/views.py @@ -83,7 +83,7 @@ def get_queryset(self): queryset = ( super() .get_queryset() - .alias( + .annotate( name=DISPLAY_NAME_CLAUSE, ) ) From 51980b431de46eccf336fa33b05db8ba0b69ccec Mon Sep 17 00:00:00 2001 From: toddlees Date: Tue, 21 Nov 2023 19:37:49 -0500 Subject: [PATCH 006/109] save child with id --- django-backend/fecfiler/transactions/views.py | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/django-backend/fecfiler/transactions/views.py b/django-backend/fecfiler/transactions/views.py index ecf552ba86..90908cec6b 100644 --- a/django-backend/fecfiler/transactions/views.py +++ b/django-backend/fecfiler/transactions/views.py @@ -267,15 +267,21 @@ def save_transaction(self, transaction_data, request): ) for child_transaction_data in children: - child_transaction_data["parent_transaction_id"] = transaction_instance.id - child_transaction_data.pop("parent_transaction", None) - if child_transaction_data.get("use_parent_contact", None): + if type(child_transaction_data) is str: + child_transaction = self.get_queryset().get(id=child_transaction_data) + child_transaction.parent_transaction_id = transaction_instance.id + else: child_transaction_data[ - "contact_1_id" - ] = transaction_instance.contact_1_id - del child_transaction_data["contact_1"] - - self.save_transaction(child_transaction_data, request) + "parent_transaction_id" + ] = transaction_instance.id + child_transaction_data.pop("parent_transaction", None) + if child_transaction_data.get("use_parent_contact", None): + child_transaction_data[ + "contact_1_id" + ] = transaction_instance.contact_1_id + del child_transaction_data["contact_1"] + + self.save_transaction(child_transaction_data, request) return self.queryset.get(id=transaction_instance.id) From 68589890e86f7a1fb5c2572c79695db6191d0898 Mon Sep 17 00:00:00 2001 From: toddlees Date: Wed, 22 Nov 2023 12:25:04 -0500 Subject: [PATCH 007/109] fix status mapping to get a better idea of what's going on --- django-backend/fecfiler/reports/views.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/django-backend/fecfiler/reports/views.py b/django-backend/fecfiler/reports/views.py index ae5ac7a3af..24da316908 100644 --- a/django-backend/fecfiler/reports/views.py +++ b/django-backend/fecfiler/reports/views.py @@ -45,24 +45,17 @@ def get_status_mapping(): """returns Django Case that determines report status based on upload submission""" - in_progress = Q(upload_submission__fec_status=None) | Q(upload_submission=None) - submitted = Q( - upload_submission__fecfile_task_state__in=[ - FECSubmissionState.INITIALIZING, - FECSubmissionState.CREATING_FILE, - FECSubmissionState.SUBMITTING, - ] - ) | Q(upload_submission__fec_status=FECStatus.PROCESSING) + upload_exists = Q(upload_submission__isnull=False) success = Q(upload_submission__fec_status=FECStatus.ACCEPTED) failed = Q(upload_submission__fecfile_task_state=FECSubmissionState.FAILED) | Q( upload_submission__fec_status=FECStatus.REJECTED ) return Case( - When(in_progress, then=Value("In progress")), - When(submitted, then=Value("Submission pending")), When(success, then=Value("Submission success")), When(failed, then=Value("Submission failure")), + When(upload_exists, then=Value("Submission pending")), + default=Value("In progress"), ) From c237aa576d0ce77264438c63e57a463654d21e20 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Wed, 22 Nov 2023 13:04:58 -0500 Subject: [PATCH 008/109] More WIP on unit tests --- .../0005_report_calculation_token.py | 18 +++++++++++++++ .../fecfiler/web_services/summary/summary.py | 16 +++++++++---- .../fecfiler/web_services/summary/tasks.py | 23 +++++++++---------- 3 files changed, 41 insertions(+), 16 deletions(-) create mode 100644 django-backend/fecfiler/reports/migrations/0005_report_calculation_token.py diff --git a/django-backend/fecfiler/reports/migrations/0005_report_calculation_token.py b/django-backend/fecfiler/reports/migrations/0005_report_calculation_token.py new file mode 100644 index 0000000000..d911aaa0ea --- /dev/null +++ b/django-backend/fecfiler/reports/migrations/0005_report_calculation_token.py @@ -0,0 +1,18 @@ +# Generated by Django 4.1.3 on 2023-11-22 17:29 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('reports', '0004_form1m_report_form_1m'), + ] + + operations = [ + migrations.AddField( + model_name='report', + name='calculation_token', + field=models.UUIDField(blank=True, null=True, unique=True), + ), + ] diff --git a/django-backend/fecfiler/web_services/summary/summary.py b/django-backend/fecfiler/web_services/summary/summary.py index aac5843314..6507668cce 100644 --- a/django-backend/fecfiler/web_services/summary/summary.py +++ b/django-backend/fecfiler/web_services/summary/summary.py @@ -11,12 +11,12 @@ class SummaryService: def __init__(self, report) -> None: self.report = report - self.previous_f3x_report = Report.objects.filter( + self.previous_report = Report.objects.filter( ~Q(id=report.id), form_3x__isnull=False, coverage_from_date__year=report.coverage_from_date.year, coverage_through_date__lt=report.coverage_from_date - ).order_by("-coverage_through_date").first().form_3x + ).order_by("-coverage_through_date").first() def calculate_summary(self): summary = { @@ -56,7 +56,11 @@ def calculate_summary_column_a(self): temp_sc10=self.get_line("SC/10", field="loan_balance"), temp_sd10=self.get_line("SD10", field="balance_at_close") ) - summary["line_6b"] = self.previous_f3x_report.L8_cash_on_hand_close_period + + summary["line_6b"] = 0 + if self.previous_report and self.previous_report.form_3x: + summary["line_6b"] = self.previous_report.form_3x.L8_cash_on_hand_close_period + summary["line_9"] = ( summary["temp_sc9"] + summary["temp_sd9"] @@ -202,7 +206,11 @@ def calculate_summary_column_b(self): line_29=self.get_line("SB29"), line_30b=self.get_line("SB30B"), ) - summary["line_6a"] = self.previous_f3x_report.L8_cash_on_hand_at_close_ytd + + summary["line_6a"] = 0 + if self.previous_report and self.previous_report.form_3x: + summary["line_6a"] = self.previous_report.form_3x.L8_cash_on_hand_at_close_ytd + summary["line_11aiii"] = ( summary["line_11ai"] + summary["line_11aii"] diff --git a/django-backend/fecfiler/web_services/summary/tasks.py b/django-backend/fecfiler/web_services/summary/tasks.py index f91f726a7a..066d4202fc 100644 --- a/django-backend/fecfiler/web_services/summary/tasks.py +++ b/django-backend/fecfiler/web_services/summary/tasks.py @@ -22,19 +22,15 @@ def __str__(self): def get_reports_to_calculate(primary_report): report_year = primary_report.coverage_from_date.year - reports_in_year = Report.objects.filter( + reports_to_recalculate = Report.objects.filter( coverage_from_date__year=report_year, - coverage_through_date__lte=primary_report.coverage_through_date - ).order_by("-coverage_through_date") - - reports_to_recalculate = [] - for report in reports_in_year: - if report.calculation_status == None: - reports_to_recalculate.insert(0, report) - else: - break + coverage_through_date__lte=primary_report.coverage_through_date, + calculation_status__isnull=True + ).order_by("coverage_through_date") - return reports_to_recalculate + if len(reports_to_recalculate) > 0: + return reports_to_recalculate + return primary_report @shared_task @@ -215,10 +211,13 @@ def calculate_summary(report_id): # Set the calculation status to SUCCEEDED *only* # if the recalculation still belongs to this task + # + # In the future, we can look into filtering on + # calculation_status as well to prevent saving + # reports that have been invalidated updated = bool( Report.objects.filter( id=report.id, - calculation_status=CalculationState.CALCULATING, calculation_token=calculation_token ).update( calculation_status=CalculationState.SUCCEEDED, From 1c7cc32f8e88303dbd11224642fc2f1f2e154860 Mon Sep 17 00:00:00 2001 From: toddlees Date: Wed, 22 Nov 2023 13:44:22 -0500 Subject: [PATCH 009/109] get more info about state --- django-backend/fecfiler/reports/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django-backend/fecfiler/reports/views.py b/django-backend/fecfiler/reports/views.py index 24da316908..88771d94d6 100644 --- a/django-backend/fecfiler/reports/views.py +++ b/django-backend/fecfiler/reports/views.py @@ -9,7 +9,7 @@ from fecfiler.memo_text.models import MemoText from fecfiler.web_services.models import DotFEC, UploadSubmission, WebPrintSubmission from .serializers import ReportSerializer -from django.db.models import Case, Value, When, Q +from django.db.models import Case, Value, When, Q, F import logging logger = logging.getLogger(__name__) @@ -54,7 +54,7 @@ def get_status_mapping(): return Case( When(success, then=Value("Submission success")), When(failed, then=Value("Submission failure")), - When(upload_exists, then=Value("Submission pending")), + When(upload_exists, then=F("upload_submission__fecfile_task_state")), default=Value("In progress"), ) From 662812a30309a4927157cad91a506851d02bd516 Mon Sep 17 00:00:00 2001 From: toddlees Date: Wed, 22 Nov 2023 14:50:19 -0500 Subject: [PATCH 010/109] catch error during .fec --- django-backend/fecfiler/web_services/tasks.py | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/django-backend/fecfiler/web_services/tasks.py b/django-backend/fecfiler/web_services/tasks.py index 7b4a682193..48904898fd 100644 --- a/django-backend/fecfiler/web_services/tasks.py +++ b/django-backend/fecfiler/web_services/tasks.py @@ -33,15 +33,20 @@ def create_dot_fec( if webprint_submission_id: submission = WebPrintSubmission.objects.get(id=webprint_submission_id) submission.save_state(FECSubmissionState.CREATING_FILE) - file_content = compose_dot_fec(report_id, upload_submission_id) - file_name = f"{report_id}_{math.floor(datetime.now().timestamp())}.fec" - if not file_content or not file_name: - if submission: - submission.save_error("Creating .FEC failed") + try: + file_content = compose_dot_fec(report_id, upload_submission_id) + file_name = f"{report_id}_{math.floor(datetime.now().timestamp())}.fec" + + if not file_content or not file_name: + raise Exception("No file created") + store_file(file_content, file_name, force_write_to_disk) + dot_fec_record = DotFEC(report_id=report_id, file_name=file_name) + dot_fec_record.save() + + except Exception as e: + submission.save_error("Creating .FEC failed") return None - store_file(file_content, file_name, force_write_to_disk) - dot_fec_record = DotFEC(report_id=report_id, file_name=file_name) - dot_fec_record.save() + if upload_submission_id: UploadSubmission.objects.filter(id=upload_submission_id).update( dot_fec=dot_fec_record @@ -86,9 +91,7 @@ def submit_to_fec( submitter = DotFECSubmitter(api) logger.info(f"Uploading {file_name} to FEC") submission_json = submitter.get_submission_json( - dot_fec_record, - e_filing_password, - backdoor_code + dot_fec_record, e_filing_password, backdoor_code ) submission_response_string = submitter.submit( dot_fec_bytes, submission_json, dot_fec_record.report.report_id or None From 634d94bc76a41026ed41fbf1989a14dfae8eafab Mon Sep 17 00:00:00 2001 From: toddlees Date: Wed, 22 Nov 2023 16:19:19 -0500 Subject: [PATCH 011/109] save error state while submitting --- django-backend/fecfiler/web_services/tasks.py | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/django-backend/fecfiler/web_services/tasks.py b/django-backend/fecfiler/web_services/tasks.py index 48904898fd..2edb19f50d 100644 --- a/django-backend/fecfiler/web_services/tasks.py +++ b/django-backend/fecfiler/web_services/tasks.py @@ -88,26 +88,30 @@ def submit_to_fec( return """Submit to FEC""" - submitter = DotFECSubmitter(api) - logger.info(f"Uploading {file_name} to FEC") - submission_json = submitter.get_submission_json( - dot_fec_record, e_filing_password, backdoor_code - ) - submission_response_string = submitter.submit( - dot_fec_bytes, submission_json, dot_fec_record.report.report_id or None - ) - submission.save_fec_response(submission_response_string) - - """Poll FEC for status of submission""" - # TODO: add timeout? - while submission.fec_status not in FECStatus.get_terminal_statuses_strings(): - logger.info(f"Polling status for {submission.fec_submission_id}.") - logger.info( - f"Status: {submission.fec_status}, Message: {submission.fec_message}" + try: + submitter = DotFECSubmitter(api) + logger.info(f"Uploading {file_name} to FEC") + submission_json = submitter.get_submission_json( + dot_fec_record, e_filing_password, backdoor_code ) - time.sleep(2) - status_response_string = submitter.poll_status(submission.fec_submission_id) - submission.save_fec_response(status_response_string) + submission_response_string = submitter.submit( + dot_fec_bytes, submission_json, dot_fec_record.report.report_id or None + ) + submission.save_fec_response(submission_response_string) + + """Poll FEC for status of submission""" + # TODO: add timeout? + while submission.fec_status not in FECStatus.get_terminal_statuses_strings(): + logger.info(f"Polling status for {submission.fec_submission_id}.") + logger.info( + f"Status: {submission.fec_status}, Message: {submission.fec_message}" + ) + time.sleep(2) + status_response_string = submitter.poll_status(submission.fec_submission_id) + submission.save_fec_response(status_response_string) + except Exception: + submission.save_error("Failed submitting to FEC") + return new_state = ( FECSubmissionState.SUCCEEDED From 631313ce5894068ea50bd6eefb7b38d23a2fd1b0 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Wed, 22 Nov 2023 16:29:42 -0500 Subject: [PATCH 012/109] Create transaction/multisave endpoint --- Dockerfile-e2e | 8 ++++---- django-backend/fecfiler/transactions/views.py | 6 ++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Dockerfile-e2e b/Dockerfile-e2e index cc6d7c906a..0f83485d9b 100644 --- a/Dockerfile-e2e +++ b/Dockerfile-e2e @@ -1,15 +1,15 @@ FROM python:3.10 ENV PYTHONUNBUFFERED=1 -RUN mkdir /opt/nxg_fec_e2e -WORKDIR /opt/nxg_fec_e2e +RUN mkdir /opt/nxg_fec +WORKDIR /opt/nxg_fec ADD requirements.txt /opt -ADD django-backend /opt/nxg_fec_e2e/ +ADD django-backend /opt/nxg_fec/ RUN pip3 install -r /opt/requirements.txt RUN mv /etc/localtime /etc/localtime.backup && ln -s /usr/share/zoneinfo/EST5EDT /etc/localtime -RUN useradd nxgu --no-create-home --home /opt/nxg_fec_e2e && chown -R nxgu:nxgu /opt/nxg_fec_e2e +RUN useradd nxgu --no-create-home --home /opt/nxg_fec && chown -R nxgu:nxgu /opt/nxg_fec USER nxgu EXPOSE 8080 diff --git a/django-backend/fecfiler/transactions/views.py b/django-backend/fecfiler/transactions/views.py index b65e25ee05..9ff2340018 100644 --- a/django-backend/fecfiler/transactions/views.py +++ b/django-backend/fecfiler/transactions/views.py @@ -289,6 +289,12 @@ def save_transaction(self, transaction_data, request): return self.queryset.get(id=transaction_instance.id) + @action(detail=False, methods=["put"], url_path=r"multisave") + def save_transactions(self, request): + with db_transaction.atomic(): + saved_data = [self.save_transaction(data, request) for data in request.data] + return Response([TransactionSerializer().to_representation(data) for data in saved_data]) + def noop(transaction, is_existing): pass From 12717a0505353aa0aeb2dbe795d7c21249689f8c Mon Sep 17 00:00:00 2001 From: toddlees Date: Wed, 22 Nov 2023 17:49:06 -0500 Subject: [PATCH 013/109] print if api exists --- django-backend/fecfiler/web_services/tasks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/django-backend/fecfiler/web_services/tasks.py b/django-backend/fecfiler/web_services/tasks.py index 2edb19f50d..a946f73b73 100644 --- a/django-backend/fecfiler/web_services/tasks.py +++ b/django-backend/fecfiler/web_services/tasks.py @@ -71,6 +71,8 @@ def submit_to_fec( logger.info(f"FEC API: {FEC_FILING_API}") logger.info(f"api submitter: {api}") submission = UploadSubmission.objects.get(id=submission_record_id) + submission.save_error(f"api submitter: {api is not None}") + return submission.save_state(FECSubmissionState.SUBMITTING) """Get Password""" From 2299dd1678a72cebd2cf4895b0f73ea907bb262f Mon Sep 17 00:00:00 2001 From: toddlees Date: Wed, 22 Nov 2023 18:10:44 -0500 Subject: [PATCH 014/109] testing --- django-backend/fecfiler/reports/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django-backend/fecfiler/reports/views.py b/django-backend/fecfiler/reports/views.py index 88771d94d6..224697a5d0 100644 --- a/django-backend/fecfiler/reports/views.py +++ b/django-backend/fecfiler/reports/views.py @@ -52,7 +52,7 @@ def get_status_mapping(): ) return Case( - When(success, then=Value("Submission success")), + When(success, then=F("upload_submission__fecfile_error")), When(failed, then=Value("Submission failure")), When(upload_exists, then=F("upload_submission__fecfile_task_state")), default=Value("In progress"), From 023a5f977c27b3ae9599f50ca6160f00ddb58b5b Mon Sep 17 00:00:00 2001 From: toddlees Date: Wed, 22 Nov 2023 18:10:44 -0500 Subject: [PATCH 015/109] testing --- django-backend/fecfiler/reports/views.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/django-backend/fecfiler/reports/views.py b/django-backend/fecfiler/reports/views.py index 224697a5d0..cfa53951f8 100644 --- a/django-backend/fecfiler/reports/views.py +++ b/django-backend/fecfiler/reports/views.py @@ -52,9 +52,9 @@ def get_status_mapping(): ) return Case( - When(success, then=F("upload_submission__fecfile_error")), - When(failed, then=Value("Submission failure")), - When(upload_exists, then=F("upload_submission__fecfile_task_state")), + When(success, then=Value("Submission success")), + When(failed, then=F("upload_submission__fecfile_task_state")), + When(upload_exists, then=Value("Submission pending")), default=Value("In progress"), ) From efd87393d8d908aa6fd99cb8a291349e51255d6d Mon Sep 17 00:00:00 2001 From: toddlees Date: Wed, 22 Nov 2023 19:36:19 -0500 Subject: [PATCH 016/109] try again --- django-backend/fecfiler/reports/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django-backend/fecfiler/reports/views.py b/django-backend/fecfiler/reports/views.py index cfa53951f8..82c75e5ede 100644 --- a/django-backend/fecfiler/reports/views.py +++ b/django-backend/fecfiler/reports/views.py @@ -53,7 +53,7 @@ def get_status_mapping(): return Case( When(success, then=Value("Submission success")), - When(failed, then=F("upload_submission__fecfile_task_state")), + When(failed, then=F("upload_submission__fecfile_error")), When(upload_exists, then=Value("Submission pending")), default=Value("In progress"), ) From 3e1d28beb0b379a768a375cd97f3119ac1b83280 Mon Sep 17 00:00:00 2001 From: toddlees Date: Wed, 22 Nov 2023 19:52:21 -0500 Subject: [PATCH 017/109] set output_field --- django-backend/fecfiler/reports/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django-backend/fecfiler/reports/views.py b/django-backend/fecfiler/reports/views.py index 82c75e5ede..eeac365530 100644 --- a/django-backend/fecfiler/reports/views.py +++ b/django-backend/fecfiler/reports/views.py @@ -9,7 +9,7 @@ from fecfiler.memo_text.models import MemoText from fecfiler.web_services.models import DotFEC, UploadSubmission, WebPrintSubmission from .serializers import ReportSerializer -from django.db.models import Case, Value, When, Q, F +from django.db.models import Case, Value, When, Q, F, CharField import logging logger = logging.getLogger(__name__) @@ -56,6 +56,7 @@ def get_status_mapping(): When(failed, then=F("upload_submission__fecfile_error")), When(upload_exists, then=Value("Submission pending")), default=Value("In progress"), + output_field=CharField(), ) From 9fe2d48418919a8fe2713289d97922e118acd946 Mon Sep 17 00:00:00 2001 From: toddlees Date: Wed, 22 Nov 2023 20:32:34 -0500 Subject: [PATCH 018/109] test --- django-backend/fecfiler/web_services/tasks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/django-backend/fecfiler/web_services/tasks.py b/django-backend/fecfiler/web_services/tasks.py index a946f73b73..e2654b1434 100644 --- a/django-backend/fecfiler/web_services/tasks.py +++ b/django-backend/fecfiler/web_services/tasks.py @@ -71,8 +71,6 @@ def submit_to_fec( logger.info(f"FEC API: {FEC_FILING_API}") logger.info(f"api submitter: {api}") submission = UploadSubmission.objects.get(id=submission_record_id) - submission.save_error(f"api submitter: {api is not None}") - return submission.save_state(FECSubmissionState.SUBMITTING) """Get Password""" @@ -101,6 +99,9 @@ def submit_to_fec( ) submission.save_fec_response(submission_response_string) + submission.save_error("HIT") + return + """Poll FEC for status of submission""" # TODO: add timeout? while submission.fec_status not in FECStatus.get_terminal_statuses_strings(): From e7562275317e0cb30f4500cf709a224904187225 Mon Sep 17 00:00:00 2001 From: toddlees Date: Wed, 22 Nov 2023 21:09:00 -0500 Subject: [PATCH 019/109] now we're on to something --- django-backend/fecfiler/web_services/tasks.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/django-backend/fecfiler/web_services/tasks.py b/django-backend/fecfiler/web_services/tasks.py index e2654b1434..9ccab8bfeb 100644 --- a/django-backend/fecfiler/web_services/tasks.py +++ b/django-backend/fecfiler/web_services/tasks.py @@ -70,6 +70,10 @@ def submit_to_fec( ): logger.info(f"FEC API: {FEC_FILING_API}") logger.info(f"api submitter: {api}") + + if submission.fecfile_task_state == FECSubmissionState.FAILED: + return + submission = UploadSubmission.objects.get(id=submission_record_id) submission.save_state(FECSubmissionState.SUBMITTING) @@ -79,9 +83,9 @@ def submit_to_fec( return """Get .FEC file bytes""" - dot_fec_record = DotFEC.objects.get(id=dot_fec_id) - file_name = dot_fec_record.file_name try: + dot_fec_record = DotFEC.objects.get(id=dot_fec_id) + file_name = dot_fec_record.file_name dot_fec_bytes = get_file_bytes(file_name, force_read_from_disk) except Exception: submission.save_error("Could not retrieve .FEC bytes") @@ -99,9 +103,6 @@ def submit_to_fec( ) submission.save_fec_response(submission_response_string) - submission.save_error("HIT") - return - """Poll FEC for status of submission""" # TODO: add timeout? while submission.fec_status not in FECStatus.get_terminal_statuses_strings(): From 4f22e268d3d01131129c4aa750cff732ce96deef Mon Sep 17 00:00:00 2001 From: toddlees Date: Wed, 22 Nov 2023 21:11:04 -0500 Subject: [PATCH 020/109] more visibility --- django-backend/fecfiler/web_services/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django-backend/fecfiler/web_services/tasks.py b/django-backend/fecfiler/web_services/tasks.py index 9ccab8bfeb..335a2e1e3d 100644 --- a/django-backend/fecfiler/web_services/tasks.py +++ b/django-backend/fecfiler/web_services/tasks.py @@ -44,7 +44,7 @@ def create_dot_fec( dot_fec_record.save() except Exception as e: - submission.save_error("Creating .FEC failed") + submission.save_error("Creating .FEC failed {e}") return None if upload_submission_id: From 4cf214bbe89766c6a6861d9afe85dba00c9c3553 Mon Sep 17 00:00:00 2001 From: toddlees Date: Wed, 22 Nov 2023 21:46:01 -0500 Subject: [PATCH 021/109] left out f --- django-backend/fecfiler/web_services/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django-backend/fecfiler/web_services/tasks.py b/django-backend/fecfiler/web_services/tasks.py index 335a2e1e3d..13fdded928 100644 --- a/django-backend/fecfiler/web_services/tasks.py +++ b/django-backend/fecfiler/web_services/tasks.py @@ -44,7 +44,7 @@ def create_dot_fec( dot_fec_record.save() except Exception as e: - submission.save_error("Creating .FEC failed {e}") + submission.save_error(f"Creating .FEC failed {e}") return None if upload_submission_id: From 3f81d47d2ede3d4690aa0dde0649c1b696efafe7 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Fri, 24 Nov 2023 20:22:49 -0500 Subject: [PATCH 022/109] Add unit tests --- .../fecfiler/transactions/tests/test_views.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/django-backend/fecfiler/transactions/tests/test_views.py b/django-backend/fecfiler/transactions/tests/test_views.py index ea2be3720f..3e0acb498d 100644 --- a/django-backend/fecfiler/transactions/tests/test_views.py +++ b/django-backend/fecfiler/transactions/tests/test_views.py @@ -157,3 +157,24 @@ def test_inherited_election_aggregate(self): ) transaction = response.data self.assertEqual(transaction.get("calendar_ytd_per_election_office"), 58.00) + + def test_multisave_transactions(self): + + request = self.request([self.payloads["IN_KIND"] for i in range(3)]) + response = TransactionViewSet().save_transactions(request) + + txn1 = deepcopy(self.payloads["IN_KIND"]) + txn1["id"] = str(response.data[0]["id"]) + txn1["contributor_last_name"] = "one" + txn2 = deepcopy(self.payloads["IN_KIND"]) + txn2["id"] = str(response.data[1]["id"]) + txn2["contributor_last_name"] = "two" + txn3 = deepcopy(self.payloads["IN_KIND"]) + txn3["id"] = str(response.data[2]["id"]) + txn3["contributor_last_name"] = "three" + + request = self.request([txn1, txn2, txn3]) + response = TransactionViewSet().save_transactions(request) + self.assertEqual(len(response.data), 3) + # self.assertEqual("one", updated_transaction.schedule_a.contributor_last_name) + From d39a03573af0b65150af459034e488be0df52baa Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Fri, 24 Nov 2023 20:27:07 -0500 Subject: [PATCH 023/109] Update unit test --- django-backend/fecfiler/transactions/tests/test_views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django-backend/fecfiler/transactions/tests/test_views.py b/django-backend/fecfiler/transactions/tests/test_views.py index 3e0acb498d..aa16756a5e 100644 --- a/django-backend/fecfiler/transactions/tests/test_views.py +++ b/django-backend/fecfiler/transactions/tests/test_views.py @@ -160,7 +160,7 @@ def test_inherited_election_aggregate(self): def test_multisave_transactions(self): - request = self.request([self.payloads["IN_KIND"] for i in range(3)]) + request = self.request([self.payloads["IN_KIND"] for _ in range(3)]) response = TransactionViewSet().save_transactions(request) txn1 = deepcopy(self.payloads["IN_KIND"]) From 4ad315114be04d5142c0c786dff055d32c793ebe Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Fri, 24 Nov 2023 20:31:50 -0500 Subject: [PATCH 024/109] Fix lint issues raised by SonarCloud --- django-backend/fecfiler/transactions/tests/test_views.py | 1 - django-backend/fecfiler/transactions/views.py | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/django-backend/fecfiler/transactions/tests/test_views.py b/django-backend/fecfiler/transactions/tests/test_views.py index aa16756a5e..f22b991602 100644 --- a/django-backend/fecfiler/transactions/tests/test_views.py +++ b/django-backend/fecfiler/transactions/tests/test_views.py @@ -177,4 +177,3 @@ def test_multisave_transactions(self): response = TransactionViewSet().save_transactions(request) self.assertEqual(len(response.data), 3) # self.assertEqual("one", updated_transaction.schedule_a.contributor_last_name) - diff --git a/django-backend/fecfiler/transactions/views.py b/django-backend/fecfiler/transactions/views.py index 9ff2340018..6589f4fcfa 100644 --- a/django-backend/fecfiler/transactions/views.py +++ b/django-backend/fecfiler/transactions/views.py @@ -293,7 +293,9 @@ def save_transaction(self, transaction_data, request): def save_transactions(self, request): with db_transaction.atomic(): saved_data = [self.save_transaction(data, request) for data in request.data] - return Response([TransactionSerializer().to_representation(data) for data in saved_data]) + return Response( + [TransactionSerializer().to_representation(data) for data in saved_data] + ) def noop(transaction, is_existing): From 9f4d36ce049655354abadd4fe111ec1d901d1933 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Sat, 25 Nov 2023 12:07:33 -0500 Subject: [PATCH 025/109] Add reattributions/redesignation endpoint --- django-backend/fecfiler/transactions/views.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/django-backend/fecfiler/transactions/views.py b/django-backend/fecfiler/transactions/views.py index 0ad101a0c1..3461bb6546 100644 --- a/django-backend/fecfiler/transactions/views.py +++ b/django-backend/fecfiler/transactions/views.py @@ -288,6 +288,21 @@ def save_transactions(self, request): [TransactionSerializer().to_representation(data) for data in saved_data] ) + @action(detail=False, methods=["get"], url_path=r"reattributions-redesignations") + def transaction_reattributions_redesignations(self, request): + transaction_id = request.query_params.get("transaction_id", None) + from pprint import pprint + query = self.get_queryset().filter( + Q(reatt_redes_id=transaction_id), + ) + transactions = query.all() + + if len(transactions) > 0: + return Response(data=[self.get_serializer(t).data for t in transactions]) + + response = {"message": "Did not find 2 reattributions or 2 redesignations."} + return Response(response, status=status.HTTP_404_NOT_FOUND) + def noop(transaction, is_existing): pass From 0862b8e81c8f40a886a30805dbda6251b915af48 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Sat, 25 Nov 2023 12:51:24 -0500 Subject: [PATCH 026/109] Fix lint error reported by SonarCloud --- django-backend/fecfiler/transactions/views.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/django-backend/fecfiler/transactions/views.py b/django-backend/fecfiler/transactions/views.py index 3461bb6546..88e5f34605 100644 --- a/django-backend/fecfiler/transactions/views.py +++ b/django-backend/fecfiler/transactions/views.py @@ -291,7 +291,6 @@ def save_transactions(self, request): @action(detail=False, methods=["get"], url_path=r"reattributions-redesignations") def transaction_reattributions_redesignations(self, request): transaction_id = request.query_params.get("transaction_id", None) - from pprint import pprint query = self.get_queryset().filter( Q(reatt_redes_id=transaction_id), ) @@ -300,7 +299,7 @@ def transaction_reattributions_redesignations(self, request): if len(transactions) > 0: return Response(data=[self.get_serializer(t).data for t in transactions]) - response = {"message": "Did not find 2 reattributions or 2 redesignations."} + response = {"message": "Did not find reattributions or redesignations."} return Response(response, status=status.HTTP_404_NOT_FOUND) From 02a88315fa1561d256406673a2683f11ab51957b Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Mon, 27 Nov 2023 12:46:04 -0500 Subject: [PATCH 027/109] Old unit tests working again! --- .../migrations/0005_report_calculation_token.py | 2 +- django-backend/fecfiler/reports/models.py | 3 +-- .../fecfiler/web_services/summary/tasks.py | 12 ++++++++++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/django-backend/fecfiler/reports/migrations/0005_report_calculation_token.py b/django-backend/fecfiler/reports/migrations/0005_report_calculation_token.py index d911aaa0ea..3a72007b5f 100644 --- a/django-backend/fecfiler/reports/migrations/0005_report_calculation_token.py +++ b/django-backend/fecfiler/reports/migrations/0005_report_calculation_token.py @@ -13,6 +13,6 @@ class Migration(migrations.Migration): migrations.AddField( model_name='report', name='calculation_token', - field=models.UUIDField(blank=True, null=True, unique=True), + field=models.UUIDField(blank=True, null=True, default=None), ), ] diff --git a/django-backend/fecfiler/reports/models.py b/django-backend/fecfiler/reports/models.py index fb8b7cdcc7..1c51ed0da7 100644 --- a/django-backend/fecfiler/reports/models.py +++ b/django-backend/fecfiler/reports/models.py @@ -45,8 +45,7 @@ class Report(SoftDeleteModel, CommitteeOwnedModel): calculation_token = models.UUIDField( null=True, blank=True, - editable=True, - unique=True + default=None ) confirmation_email_1 = models.EmailField( diff --git a/django-backend/fecfiler/web_services/summary/tasks.py b/django-backend/fecfiler/web_services/summary/tasks.py index 066d4202fc..97e980457d 100644 --- a/django-backend/fecfiler/web_services/summary/tasks.py +++ b/django-backend/fecfiler/web_services/summary/tasks.py @@ -1,6 +1,7 @@ from enum import Enum from celery import shared_task from fecfiler.reports.models import Report +from django.db.models import Q from .summary import SummaryService import uuid @@ -23,9 +24,9 @@ def __str__(self): def get_reports_to_calculate(primary_report): report_year = primary_report.coverage_from_date.year reports_to_recalculate = Report.objects.filter( + ~Q(calculation_status=CalculationState.SUCCEEDED), coverage_from_date__year=report_year, - coverage_through_date__lte=primary_report.coverage_through_date, - calculation_status__isnull=True + coverage_through_date__lte=primary_report.coverage_through_date ).order_by("coverage_through_date") if len(reports_to_recalculate) > 0: @@ -42,12 +43,17 @@ def calculate_summary(report_id): reports_to_recalculate = get_reports_to_calculate(primary_report) calculation_token = uuid.uuid4() + print("Before update", len(reports_to_recalculate)) reports_to_recalculate.update( calculation_token=calculation_token, calculation_status=CalculationState.CALCULATING ) + print("After update", len(reports_to_recalculate)) + + print("\n\n\n","Recalculating", len(reports_to_recalculate), "Reports") for report in reports_to_recalculate: + print("Recalculating", report.id) summary_service = SummaryService(report) summary = summary_service.calculate_summary() a = summary["a"] @@ -207,6 +213,8 @@ def calculate_summary(report_id): report.form_3x.L38_net_operating_expenditures_period = a.get("line_38", 0) report.form_3x.L38_net_operating_expenditures_ytd = b.get("line_38", 0) + print("\n\n\n",a["line_15"], b["line_15"],"\n\n\n") + report.form_3x.save() # Set the calculation status to SUCCEEDED *only* From 8095d6c7f7a5712994462730ac16c94a462267a2 Mon Sep 17 00:00:00 2001 From: toddlees Date: Tue, 28 Nov 2023 08:35:06 -0500 Subject: [PATCH 028/109] show current state --- django-backend/fecfiler/reports/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django-backend/fecfiler/reports/views.py b/django-backend/fecfiler/reports/views.py index eeac365530..2e338be462 100644 --- a/django-backend/fecfiler/reports/views.py +++ b/django-backend/fecfiler/reports/views.py @@ -53,7 +53,7 @@ def get_status_mapping(): return Case( When(success, then=Value("Submission success")), - When(failed, then=F("upload_submission__fecfile_error")), + When(failed, then=F("upload_submission__fecfile_task_state")), When(upload_exists, then=Value("Submission pending")), default=Value("In progress"), output_field=CharField(), From 71a3c66cfb976898476912652fc66f1213492211 Mon Sep 17 00:00:00 2001 From: toddlees Date: Tue, 28 Nov 2023 08:52:17 -0500 Subject: [PATCH 029/109] show state --- django-backend/fecfiler/reports/views.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/django-backend/fecfiler/reports/views.py b/django-backend/fecfiler/reports/views.py index 2e338be462..41b260f3b7 100644 --- a/django-backend/fecfiler/reports/views.py +++ b/django-backend/fecfiler/reports/views.py @@ -53,8 +53,10 @@ def get_status_mapping(): return Case( When(success, then=Value("Submission success")), - When(failed, then=F("upload_submission__fecfile_task_state")), - When(upload_exists, then=Value("Submission pending")), + When(failed, then=F("upload_submission__fecfile_error")), + When( + upload_exists, then=F("upload_submission__fecfile_task_state") + ), # Value("Submission pending")), default=Value("In progress"), output_field=CharField(), ) From 53578c3cec71da0e52ba3c17f2f4a61824e049e0 Mon Sep 17 00:00:00 2001 From: toddlees Date: Tue, 28 Nov 2023 09:23:52 -0500 Subject: [PATCH 030/109] has .fec? --- django-backend/fecfiler/reports/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django-backend/fecfiler/reports/views.py b/django-backend/fecfiler/reports/views.py index 41b260f3b7..4bb31da3a7 100644 --- a/django-backend/fecfiler/reports/views.py +++ b/django-backend/fecfiler/reports/views.py @@ -55,7 +55,7 @@ def get_status_mapping(): When(success, then=Value("Submission success")), When(failed, then=F("upload_submission__fecfile_error")), When( - upload_exists, then=F("upload_submission__fecfile_task_state") + upload_exists, then=F("upload_submission__dot_fec") ), # Value("Submission pending")), default=Value("In progress"), output_field=CharField(), From 7ee530884dccdc3e470d898b17dcf7cd2fe2c3c3 Mon Sep 17 00:00:00 2001 From: toddlees Date: Tue, 28 Nov 2023 11:09:02 -0500 Subject: [PATCH 031/109] does it have a .fec? --- django-backend/fecfiler/reports/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django-backend/fecfiler/reports/views.py b/django-backend/fecfiler/reports/views.py index 4bb31da3a7..fd52a1cea1 100644 --- a/django-backend/fecfiler/reports/views.py +++ b/django-backend/fecfiler/reports/views.py @@ -55,7 +55,7 @@ def get_status_mapping(): When(success, then=Value("Submission success")), When(failed, then=F("upload_submission__fecfile_error")), When( - upload_exists, then=F("upload_submission__dot_fec") + upload_exists, then=F("upload_submission__dot_fec__is_null") ), # Value("Submission pending")), default=Value("In progress"), output_field=CharField(), From 53a3b0f8c92bd162d93567451a41c46c429e3a55 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Tue, 28 Nov 2023 12:34:22 -0500 Subject: [PATCH 032/109] New unit tests added --- .../reports/fixtures/test_f3x_reports.json | 138 ++++++++++++++++++ .../reports/form_3x/tests/test_models.py | 2 +- .../reports/form_3x/tests/test_views.py | 5 + .../fecfiler/web_services/summary/summary.py | 4 +- .../fecfiler/web_services/summary/tasks.py | 7 - .../web_services/summary/test_tasks.py | 25 ++++ 6 files changed, 171 insertions(+), 10 deletions(-) diff --git a/django-backend/fecfiler/reports/fixtures/test_f3x_reports.json b/django-backend/fecfiler/reports/fixtures/test_f3x_reports.json index 879a5b319f..9f26be03eb 100644 --- a/django-backend/fecfiler/reports/fixtures/test_f3x_reports.json +++ b/django-backend/fecfiler/reports/fixtures/test_f3x_reports.json @@ -137,6 +137,144 @@ "updated": "2022-02-09T00:00:00.000Z" } }, + { + "model": "reports.Form3X", + "fields": { + "id": "a6d60d2d-d926-4e89-ad4b-000000000002", + "change_of_address": true, + "street_1": "test_string_value", + "street_2": "test_string_value", + "city": "test_string_value", + "state": "test_string_value", + "zip": "test_string_value", + "election_code": "test_string_value", + "date_of_election": "2005-01-30", + "state_of_election": "test_string_value", + "qualified_committee": false, + "L6b_cash_on_hand_beginning_period": 6, + "L6c_total_receipts_period": 60, + "L6d_subtotal_period": 600, + "L7_total_disbursements_period": 7, + "L8_cash_on_hand_at_close_period": 8, + "L9_debts_to_period": 9, + "L10_debts_by_period": 10, + "L11ai_itemized_period": 11, + "L11aii_unitemized_period": 110, + "L11aiii_total_period": 1100, + "L11b_political_party_committees_period": 11000, + "L11c_other_political_committees_pacs_period": 110000, + "L11d_total_contributions_period": 110000, + "L12_transfers_from_affiliated_other_party_cmtes_period": 12, + "L13_all_loans_received_period": 13, + "L14_loan_repayments_received_period": 14, + "L15_offsets_to_operating_expenditures_refunds_period": 15, + "L16_refunds_of_federal_contributions_period": 16, + "L17_other_federal_receipts_dividends_period": 17, + "L18a_transfers_from_nonfederal_account_h3_period": 18, + "L18b_transfers_from_nonfederal_levin_h5_period": 180, + "L18c_total_nonfederal_transfers_18a_18b_period": 1800, + "L19_total_receipts_period": 19, + "L20_total_federal_receipts_period": 20, + "L21ai_federal_share_period": 21, + "L21aii_nonfederal_share_period": 21, + "L21b_other_federal_operating_expenditures_period": 21, + "L21c_total_operating_expenditures_period": 21, + "L22_transfers_to_affiliated_other_party_cmtes_period": 22, + "L23_contributions_to_federal_candidates_cmtes_period": 23, + "L24_independent_expenditures_period": 24, + "L25_coordinated_expend_made_by_party_cmtes_period": 25, + "L26_loan_repayments_period": 26, + "L27_loans_made_period": 27, + "L28a_individuals_persons_period": 28, + "L28b_political_party_committees_period": 28, + "L28c_other_political_committees_period": 28, + "L28d_total_contributions_refunds_period": 28, + "L29_other_disbursements_period": 29, + "L30ai_shared_federal_activity_h6_fed_share_period": 30, + "L30aii_shared_federal_activity_h6_nonfed_period": 30, + "L30b_nonallocable_fed_election_activity_period": 30, + "L30c_total_federal_election_activity_period": 30, + "L31_total_disbursements_period": 31, + "L32_total_federal_disbursements_period": 32, + "L33_total_contributions_period": 33, + "L34_total_contribution_refunds_period": 34, + "L35_net_contributions_period": 35, + "L36_total_federal_operating_expenditures_period": 36, + "L37_offsets_to_operating_expenditures_period": 37, + "L38_net_operating_expenditures_period": 38, + "L6a_cash_on_hand_jan_1_ytd": 61, + "L6a_year_for_above_ytd": "2005", + "L6c_total_receipts_ytd": 611, + "L6d_subtotal_ytd": 6111, + "L7_total_disbursements_ytd": 71, + "L8_cash_on_hand_close_ytd": 81, + "L11ai_itemized_ytd": 111, + "L11aii_unitemized_ytd": 1111, + "L11aiii_total_ytd": 11111, + "L11b_political_party_committees_ytd": 111111, + "L11c_other_political_committees_pacs_ytd": 1111111, + "L11d_total_contributions_ytd": 11111111, + "L12_transfers_from_affiliated_other_party_cmtes_ytd": 121, + "L13_all_loans_received_ytd": 131, + "L14_loan_repayments_received_ytd": 141, + "L15_offsets_to_operating_expenditures_refunds_ytd": 151, + "L16_refunds_of_federal_contributions_ytd": 161, + "L17_other_federal_receipts_dividends_ytd": 171, + "L18a_transfers_from_nonfederal_account_h3_ytd": 181, + "L18b_transfers_from_nonfederal_levin_h5_ytd": 181, + "L18c_total_nonfederal_transfers_18a_18b_ytd": 181, + "L19_total_receipts_ytd": 191, + "L20_total_federal_receipts_ytd": 201, + "L21ai_federal_share_ytd": 211, + "L21aii_nonfederal_share_ytd": 211, + "L21b_other_federal_operating_expenditures_ytd": 211, + "L21c_total_operating_expenditures_ytd": 211, + "L22_transfers_to_affiliated_other_party_cmtes_ytd": 221, + "L23_contributions_to_federal_candidates_cmtes_ytd": 231, + "L24_independent_expenditures_ytd": 241, + "L25_coordinated_expend_made_by_party_cmtes_ytd": 251, + "L26_loan_repayments_made_ytd": 261, + "L27_loans_made_ytd": 271, + "L28a_individuals_persons_ytd": 281, + "L28b_political_party_committees_ytd": 281, + "L28c_other_political_committees_ytd": 281, + "L28d_total_contributions_refunds_ytd": 281, + "L29_other_disbursements_ytd": 291, + "L30ai_shared_federal_activity_h6_fed_share_ytd": 301, + "L30aii_shared_federal_activity_h6_nonfed_ytd": 301, + "L30b_nonallocable_fed_election_activity_ytd": 301, + "L30c_total_federal_election_activity_ytd": 301, + "L31_total_disbursements_ytd": 311, + "L32_total_federal_disbursements_ytd": 321, + "L33_total_contributions_ytd": 331, + "L34_total_contribution_refunds_ytd": 341, + "L35_net_contributions_ytd": 351, + "L36_total_federal_operating_expenditures_ytd": 361, + "L37_offsets_to_operating_expenditures_ytd": 371, + "L38_net_operating_expenditures_ytd": 381 + } + }, + { + "model": "reports.report", + "fields": { + "id": "b6d60d2d-d926-4e89-ad4b-000000000002", + "form_3x_id": "a6d60d2d-d926-4e89-ad4b-000000000002", + "committee_account_id": "735db943-9446-462a-9be0-c820baadb622", + "form_type": "F3XN", + "report_code": "M4", + "calculation_status": null, + "coverage_from_date": "2005-03-01", + "coverage_through_date": "2005-03-31", + "treasurer_last_name": "Lastname", + "treasurer_first_name": "Firstname", + "treasurer_middle_name": "test_string_value", + "treasurer_prefix": "test_string_value", + "treasurer_suffix": "test_string_value", + "date_signed": "2004-07-29", + "created": "2022-02-09T00:00:00.000Z", + "updated": "2022-02-09T00:00:00.000Z" + } + }, { "model": "reports.Form3X", "fields": { diff --git a/django-backend/fecfiler/reports/form_3x/tests/test_models.py b/django-backend/fecfiler/reports/form_3x/tests/test_models.py index 18de3e677a..5712694910 100644 --- a/django-backend/fecfiler/reports/form_3x/tests/test_models.py +++ b/django-backend/fecfiler/reports/form_3x/tests/test_models.py @@ -21,7 +21,7 @@ def setUp(self): ) def test_get_f3x_summary(self): - f3x_summary = Form3X.objects.get(L6a_year_for_above_ytd="2005") + f3x_summary = Form3X.objects.get(id="a6d60d2d-d926-4e89-ad4b-c47d152a66ae") self.assertEquals(f3x_summary.election_code, "test_string_value") def test_save_and_delete(self): diff --git a/django-backend/fecfiler/reports/form_3x/tests/test_views.py b/django-backend/fecfiler/reports/form_3x/tests/test_views.py index 52f6d0d29d..4413b2eb18 100644 --- a/django-backend/fecfiler/reports/form_3x/tests/test_views.py +++ b/django-backend/fecfiler/reports/form_3x/tests/test_views.py @@ -27,6 +27,11 @@ def test_coverage_dates_happy_path(self): "coverage_through_date": "2005-02-28", "report_code": "Q1", }, + { + "coverage_from_date": "2005-03-01", + "coverage_through_date": "2005-03-31", + "report_code": "M4", + }, { "coverage_from_date": "2006-01-30", "coverage_through_date": "2006-02-28", diff --git a/django-backend/fecfiler/web_services/summary/summary.py b/django-backend/fecfiler/web_services/summary/summary.py index 6507668cce..c975237510 100644 --- a/django-backend/fecfiler/web_services/summary/summary.py +++ b/django-backend/fecfiler/web_services/summary/summary.py @@ -59,7 +59,7 @@ def calculate_summary_column_a(self): summary["line_6b"] = 0 if self.previous_report and self.previous_report.form_3x: - summary["line_6b"] = self.previous_report.form_3x.L8_cash_on_hand_close_period + summary["line_6b"] = self.previous_report.form_3x.L8_cash_on_hand_at_close_period summary["line_9"] = ( summary["temp_sc9"] @@ -209,7 +209,7 @@ def calculate_summary_column_b(self): summary["line_6a"] = 0 if self.previous_report and self.previous_report.form_3x: - summary["line_6a"] = self.previous_report.form_3x.L8_cash_on_hand_at_close_ytd + summary["line_6a"] = self.previous_report.form_3x.L8_cash_on_hand_close_ytd summary["line_11aiii"] = ( summary["line_11ai"] diff --git a/django-backend/fecfiler/web_services/summary/tasks.py b/django-backend/fecfiler/web_services/summary/tasks.py index 97e980457d..c4ed4400fc 100644 --- a/django-backend/fecfiler/web_services/summary/tasks.py +++ b/django-backend/fecfiler/web_services/summary/tasks.py @@ -43,17 +43,12 @@ def calculate_summary(report_id): reports_to_recalculate = get_reports_to_calculate(primary_report) calculation_token = uuid.uuid4() - print("Before update", len(reports_to_recalculate)) reports_to_recalculate.update( calculation_token=calculation_token, calculation_status=CalculationState.CALCULATING ) - print("After update", len(reports_to_recalculate)) - - print("\n\n\n","Recalculating", len(reports_to_recalculate), "Reports") for report in reports_to_recalculate: - print("Recalculating", report.id) summary_service = SummaryService(report) summary = summary_service.calculate_summary() a = summary["a"] @@ -213,8 +208,6 @@ def calculate_summary(report_id): report.form_3x.L38_net_operating_expenditures_period = a.get("line_38", 0) report.form_3x.L38_net_operating_expenditures_ytd = b.get("line_38", 0) - print("\n\n\n",a["line_15"], b["line_15"],"\n\n\n") - report.form_3x.save() # Set the calculation status to SUCCEEDED *only* diff --git a/django-backend/fecfiler/web_services/summary/test_tasks.py b/django-backend/fecfiler/web_services/summary/test_tasks.py index 76419a066c..53780c5b8c 100644 --- a/django-backend/fecfiler/web_services/summary/test_tasks.py +++ b/django-backend/fecfiler/web_services/summary/test_tasks.py @@ -38,4 +38,29 @@ def test_report_with_no_transactions(self): self.assertEqual( report.form_3x.L37_offsets_to_operating_expenditures_period, Decimal("0") ) + self.assertEqual( + report.form_3x.L6b_cash_on_hand_beginning_period, Decimal("0") + ) self.assertEqual(report.calculation_status, CalculationState.SUCCEEDED.value) + + def test_report_b(self): + report = Report.objects.get(id="b6d60d2d-d926-4e89-ad4b-000000000002") + previous_report = Report.objects.get(id="b6d60d2d-d926-4e89-ad4b-c47d152a66ae") + + previous_report.calculation_status = None + previous_report.save() + + report.calculation_status = None + report.save() + + calculate_summary("b6d60d2d-d926-4e89-ad4b-000000000002") + calculated_report = Report.objects.get(id="b6d60d2d-d926-4e89-ad4b-000000000002") + + self.assertEqual( + calculated_report.form_3x.L6b_cash_on_hand_beginning_period, + Decimal("18085.17") + ) + self.assertEqual( + calculated_report.form_3x.L6a_cash_on_hand_jan_1_ytd, + Decimal("18985.17") + ) From 694208e470a292c2eae114d9ba4642fe4f7aa323 Mon Sep 17 00:00:00 2001 From: toddlees Date: Tue, 28 Nov 2023 12:36:29 -0500 Subject: [PATCH 033/109] throw error during write --- django-backend/fecfiler/reports/views.py | 4 +--- .../fecfiler/web_services/web_service_storage.py | 7 ++++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/django-backend/fecfiler/reports/views.py b/django-backend/fecfiler/reports/views.py index fd52a1cea1..eeac365530 100644 --- a/django-backend/fecfiler/reports/views.py +++ b/django-backend/fecfiler/reports/views.py @@ -54,9 +54,7 @@ def get_status_mapping(): return Case( When(success, then=Value("Submission success")), When(failed, then=F("upload_submission__fecfile_error")), - When( - upload_exists, then=F("upload_submission__dot_fec__is_null") - ), # Value("Submission pending")), + When(upload_exists, then=Value("Submission pending")), default=Value("In progress"), output_field=CharField(), ) diff --git a/django-backend/fecfiler/web_services/web_service_storage.py b/django-backend/fecfiler/web_services/web_service_storage.py index bec9bcf116..00234e999a 100644 --- a/django-backend/fecfiler/web_services/web_service_storage.py +++ b/django-backend/fecfiler/web_services/web_service_storage.py @@ -20,7 +20,12 @@ def store_file(file_content, file_name, force_write_to_disk=False): else: logger.info(f"writing file to disk: {file_name}") path = Path(CELERY_LOCAL_STORAGE_DIRECTORY) / file_name - with open(path, "w", encoding="utf-8") as file: + try: + file = open(path, "w", encoding="utf-8") + except Exception as e: + logger.error(f"FAILED file could not be opened to write: {file_name}") + raise e + with file: file.write(file_content) logger.info(f"SUCCESS file was written to disk: {file_name}") From a6d03d3c8324043f9aa8e94c945cc5356d451bc5 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Tue, 28 Nov 2023 12:42:09 -0500 Subject: [PATCH 034/109] Deals with line-too-long warnings --- .../fecfiler/web_services/summary/tasks.py | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/django-backend/fecfiler/web_services/summary/tasks.py b/django-backend/fecfiler/web_services/summary/tasks.py index c4ed4400fc..7d4d561dbd 100644 --- a/django-backend/fecfiler/web_services/summary/tasks.py +++ b/django-backend/fecfiler/web_services/summary/tasks.py @@ -104,7 +104,7 @@ def calculate_summary(report_id): report.form_3x.L14_loan_repayments_received_ytd = b.get("line_14", 0) # line 15 report.form_3x.L15_offsets_to_operating_expenditures_refunds_period = a.get("line_15", 0) # noqa: E501 - report.form_3x.L15_offsets_to_operating_expenditures_refunds_ytd = b.get("line_15", 0) + report.form_3x.L15_offsets_to_operating_expenditures_refunds_ytd = b.get("line_15", 0) # noqa: E501 # line 16 report.form_3x.L16_refunds_of_federal_contributions_period = a.get("line_16", 0) report.form_3x.L16_refunds_of_federal_contributions_ytd = b.get("line_16", 0) @@ -112,13 +112,13 @@ def calculate_summary(report_id): report.form_3x.L17_other_federal_receipts_dividends_period = a.get("line_17", 0) report.form_3x.L17_other_federal_receipts_dividends_ytd = b.get("line_17", 0) # line 18a - report.form_3x.L18a_transfers_from_nonfederal_account_h3_period = a.get("line_18a", 0) - report.form_3x.L18a_transfers_from_nonfederal_account_h3_ytd = b.get("line_18a", 0) + report.form_3x.L18a_transfers_from_nonfederal_account_h3_period = a.get("line_18a", 0) # noqa: E501 + report.form_3x.L18a_transfers_from_nonfederal_account_h3_ytd = b.get("line_18a", 0) # noqa: E501 # line 18b - report.form_3x.L18b_transfers_from_nonfederal_levin_h5_period = a.get("line_18b", 0) + report.form_3x.L18b_transfers_from_nonfederal_levin_h5_period = a.get("line_18b", 0) # noqa: E501 report.form_3x.L18b_transfers_from_nonfederal_levin_h5_ytd = b.get("line_18b", 0) # line 18c - report.form_3x.L18c_total_nonfederal_transfers_18a_18b_period = a.get("line_18c", 0) + report.form_3x.L18c_total_nonfederal_transfers_18a_18b_period = a.get("line_18c", 0) # noqa: E501 report.form_3x.L18c_total_nonfederal_transfers_18a_18b_ytd = b.get("line_18c", 0) # line 19 report.form_3x.L19_total_receipts_period = a.get("line_19", 0) @@ -133,23 +133,23 @@ def calculate_summary(report_id): report.form_3x.L21aii_nonfederal_share_period = a.get("line_21aii", 0) report.form_3x.L21aii_nonfederal_share_ytd = b.get("line_21aii", 0) # line 21b - report.form_3x.L21b_other_federal_operating_expenditures_period = a.get("line_21b", 0) - report.form_3x.L21b_other_federal_operating_expenditures_ytd = b.get("line_21b", 0) + report.form_3x.L21b_other_federal_operating_expenditures_period = a.get("line_21b", 0) # noqa: E501 + report.form_3x.L21b_other_federal_operating_expenditures_ytd = b.get("line_21b", 0) # noqa: E501 # line 21c report.form_3x.L21c_total_operating_expenditures_ytd = b.get("line_21c", 0) report.form_3x.L21c_total_operating_expenditures_period = a.get("line_21c", 0) # line 22 report.form_3x.L22_transfers_to_affiliated_other_party_cmtes_period = a.get("line_22", 0) # noqa: E501 - report.form_3x.L22_transfers_to_affiliated_other_party_cmtes_ytd = b.get("line_22", 0) + report.form_3x.L22_transfers_to_affiliated_other_party_cmtes_ytd = b.get("line_22", 0) # noqa: E501 # line 23 report.form_3x.L23_contributions_to_federal_candidates_cmtes_period = a.get("line_23", 0) # noqa: E501 - report.form_3x.L23_contributions_to_federal_candidates_cmtes_ytd = b.get("line_23", 0) + report.form_3x.L23_contributions_to_federal_candidates_cmtes_ytd = b.get("line_23", 0) # noqa: E501 # line 24 report.form_3x.L24_independent_expenditures_period = a.get("line_24", 0) report.form_3x.L24_independent_expenditures_ytd = b.get("line_24", 0) # line 25 - report.form_3x.L25_coordinated_expend_made_by_party_cmtes_period = a.get("line_25", 0) - report.form_3x.L25_coordinated_expend_made_by_party_cmtes_ytd = b.get("line_25", 0) + report.form_3x.L25_coordinated_expend_made_by_party_cmtes_period = a.get("line_25", 0) # noqa: E501 + report.form_3x.L25_coordinated_expend_made_by_party_cmtes_ytd = b.get("line_25", 0) # noqa: E501 # line 26 report.form_3x.L26_loan_repayments_period = a.get("line_26", 0) report.form_3x.L26_loan_repayments_made_ytd = b.get("line_26", 0) @@ -173,12 +173,12 @@ def calculate_summary(report_id): report.form_3x.L29_other_disbursements_ytd = b.get("line_29", 0) # line 30ai report.form_3x.L30ai_shared_federal_activity_h6_fed_share_period = a.get("line_30ai", 0) # noqa: E501 - report.form_3x.L30ai_shared_federal_activity_h6_fed_share_ytd = b.get("line_30ai", 0) + report.form_3x.L30ai_shared_federal_activity_h6_fed_share_ytd = b.get("line_30ai", 0) # noqa: E501 # line 30aii report.form_3x.L30aii_shared_federal_activity_h6_nonfed_period = a.get("line_30aii", 0) # noqa: E501 - report.form_3x.L30aii_shared_federal_activity_h6_nonfed_ytd = b.get("line_30aii", 0) + report.form_3x.L30aii_shared_federal_activity_h6_nonfed_ytd = b.get("line_30aii", 0) # noqa: E501 # line 30b - report.form_3x.L30b_nonallocable_fed_election_activity_period = a.get("line_30b", 0) + report.form_3x.L30b_nonallocable_fed_election_activity_period = a.get("line_30b", 0) # noqa: E501 report.form_3x.L30b_nonallocable_fed_election_activity_ytd = b.get("line_30b", 0) # line 30c report.form_3x.L30c_total_federal_election_activity_period = a.get("line_30c", 0) @@ -199,7 +199,7 @@ def calculate_summary(report_id): report.form_3x.L35_net_contributions_period = a.get("line_35", 0) report.form_3x.L35_net_contributions_ytd = b.get("line_35", 0) # line 36 - report.form_3x.L36_total_federal_operating_expenditures_period = a.get("line_36", 0) + report.form_3x.L36_total_federal_operating_expenditures_period = a.get("line_36", 0) # noqa: E501 report.form_3x.L36_total_federal_operating_expenditures_ytd = b.get("line_36", 0) # line 37 report.form_3x.L37_offsets_to_operating_expenditures_period = a.get("line_37", 0) From c599b312f728040cd08bf3a609904412562185ba Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Tue, 28 Nov 2023 12:45:50 -0500 Subject: [PATCH 035/109] One more line that was too long --- django-backend/fecfiler/web_services/summary/summary.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django-backend/fecfiler/web_services/summary/summary.py b/django-backend/fecfiler/web_services/summary/summary.py index c975237510..2b6e5a1a12 100644 --- a/django-backend/fecfiler/web_services/summary/summary.py +++ b/django-backend/fecfiler/web_services/summary/summary.py @@ -59,7 +59,7 @@ def calculate_summary_column_a(self): summary["line_6b"] = 0 if self.previous_report and self.previous_report.form_3x: - summary["line_6b"] = self.previous_report.form_3x.L8_cash_on_hand_at_close_period + summary["line_6b"] = self.previous_report.form_3x.L8_cash_on_hand_at_close_period # noqa: E501 summary["line_9"] = ( summary["temp_sc9"] From efba1494d3734371db8581753fedce884b93b970 Mon Sep 17 00:00:00 2001 From: toddlees Date: Tue, 28 Nov 2023 14:14:50 -0500 Subject: [PATCH 036/109] debug --- django-backend/fecfiler/web_services/tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/django-backend/fecfiler/web_services/tasks.py b/django-backend/fecfiler/web_services/tasks.py index 13fdded928..b0865c5e9a 100644 --- a/django-backend/fecfiler/web_services/tasks.py +++ b/django-backend/fecfiler/web_services/tasks.py @@ -40,6 +40,7 @@ def create_dot_fec( if not file_content or not file_name: raise Exception("No file created") store_file(file_content, file_name, force_write_to_disk) + submission.save_error(f"STORED FILE") dot_fec_record = DotFEC(report_id=report_id, file_name=file_name) dot_fec_record.save() From 623d3d430cd490448a46ceaddd8ec12c7ae2fce2 Mon Sep 17 00:00:00 2001 From: toddlees Date: Tue, 28 Nov 2023 14:52:24 -0500 Subject: [PATCH 037/109] try different error handling --- .../fecfiler/web_services/web_service_storage.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/django-backend/fecfiler/web_services/web_service_storage.py b/django-backend/fecfiler/web_services/web_service_storage.py index 00234e999a..e84daa90ac 100644 --- a/django-backend/fecfiler/web_services/web_service_storage.py +++ b/django-backend/fecfiler/web_services/web_service_storage.py @@ -22,12 +22,13 @@ def store_file(file_content, file_name, force_write_to_disk=False): path = Path(CELERY_LOCAL_STORAGE_DIRECTORY) / file_name try: file = open(path, "w", encoding="utf-8") + file.write(file_content) except Exception as e: - logger.error(f"FAILED file could not be opened to write: {file_name}") + logger.error(f"FAILED file could not be written: {file_name}") raise e - with file: - file.write(file_content) - logger.info(f"SUCCESS file was written to disk: {file_name}") + finally: + file.close() + logger.info(f"SUCCESS file was written to disk: {file_name}") def get_file(file_name, force_read_from_disk=False): From f8cf98cf5c3f66ae396050dc204728ce9d24adcb Mon Sep 17 00:00:00 2001 From: toddlees Date: Tue, 28 Nov 2023 15:12:38 -0500 Subject: [PATCH 038/109] i guess the file is saving --- django-backend/fecfiler/web_services/tasks.py | 2 +- .../fecfiler/web_services/web_service_storage.py | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/django-backend/fecfiler/web_services/tasks.py b/django-backend/fecfiler/web_services/tasks.py index b0865c5e9a..c4a853b620 100644 --- a/django-backend/fecfiler/web_services/tasks.py +++ b/django-backend/fecfiler/web_services/tasks.py @@ -40,7 +40,6 @@ def create_dot_fec( if not file_content or not file_name: raise Exception("No file created") store_file(file_content, file_name, force_write_to_disk) - submission.save_error(f"STORED FILE") dot_fec_record = DotFEC(report_id=report_id, file_name=file_name) dot_fec_record.save() @@ -57,6 +56,7 @@ def create_dot_fec( dot_fec=dot_fec_record ) + submission.save_error(f"saved dot fec") return dot_fec_record.id diff --git a/django-backend/fecfiler/web_services/web_service_storage.py b/django-backend/fecfiler/web_services/web_service_storage.py index e84daa90ac..bec9bcf116 100644 --- a/django-backend/fecfiler/web_services/web_service_storage.py +++ b/django-backend/fecfiler/web_services/web_service_storage.py @@ -20,15 +20,9 @@ def store_file(file_content, file_name, force_write_to_disk=False): else: logger.info(f"writing file to disk: {file_name}") path = Path(CELERY_LOCAL_STORAGE_DIRECTORY) / file_name - try: - file = open(path, "w", encoding="utf-8") + with open(path, "w", encoding="utf-8") as file: file.write(file_content) - except Exception as e: - logger.error(f"FAILED file could not be written: {file_name}") - raise e - finally: - file.close() - logger.info(f"SUCCESS file was written to disk: {file_name}") + logger.info(f"SUCCESS file was written to disk: {file_name}") def get_file(file_name, force_read_from_disk=False): From 5e9c1fdf9ea2131fdfff374d1fc99affa8c31cd6 Mon Sep 17 00:00:00 2001 From: toddlees Date: Tue, 28 Nov 2023 15:26:50 -0500 Subject: [PATCH 039/109] broken in submit --- django-backend/fecfiler/reports/views.py | 4 +++- django-backend/fecfiler/web_services/tasks.py | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/django-backend/fecfiler/reports/views.py b/django-backend/fecfiler/reports/views.py index eeac365530..41b260f3b7 100644 --- a/django-backend/fecfiler/reports/views.py +++ b/django-backend/fecfiler/reports/views.py @@ -54,7 +54,9 @@ def get_status_mapping(): return Case( When(success, then=Value("Submission success")), When(failed, then=F("upload_submission__fecfile_error")), - When(upload_exists, then=Value("Submission pending")), + When( + upload_exists, then=F("upload_submission__fecfile_task_state") + ), # Value("Submission pending")), default=Value("In progress"), output_field=CharField(), ) diff --git a/django-backend/fecfiler/web_services/tasks.py b/django-backend/fecfiler/web_services/tasks.py index c4a853b620..13fdded928 100644 --- a/django-backend/fecfiler/web_services/tasks.py +++ b/django-backend/fecfiler/web_services/tasks.py @@ -56,7 +56,6 @@ def create_dot_fec( dot_fec=dot_fec_record ) - submission.save_error(f"saved dot fec") return dot_fec_record.id From 2143b70292a4c8a20681ba780f6090a5565bbdb4 Mon Sep 17 00:00:00 2001 From: toddlees Date: Tue, 28 Nov 2023 16:05:47 -0500 Subject: [PATCH 040/109] try catching --- django-backend/fecfiler/web_services/tasks.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/django-backend/fecfiler/web_services/tasks.py b/django-backend/fecfiler/web_services/tasks.py index 13fdded928..d841133e09 100644 --- a/django-backend/fecfiler/web_services/tasks.py +++ b/django-backend/fecfiler/web_services/tasks.py @@ -74,8 +74,12 @@ def submit_to_fec( if submission.fecfile_task_state == FECSubmissionState.FAILED: return - submission = UploadSubmission.objects.get(id=submission_record_id) - submission.save_state(FECSubmissionState.SUBMITTING) + try: + submission = UploadSubmission.objects.get(id=submission_record_id) + submission.save_state(FECSubmissionState.SUBMITTING) + except Exception as e: + submission = DotFEC.objects.get(id=dot_fec_id).uploadsubmission_set.first() + submission.save_error(f"ERROR GETTING SUBMISSION {submission_record_id}") """Get Password""" if not e_filing_password: From 52aa987ecf6a635697f94cd4b2875143c879ed6c Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Tue, 28 Nov 2023 16:13:41 -0500 Subject: [PATCH 041/109] 1543 change from lexigraphical sort on line number --- .../fecfiler/transactions/managers.py | 42 +++++++++++++++++-- django-backend/fecfiler/transactions/views.py | 1 + 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/django-backend/fecfiler/transactions/managers.py b/django-backend/fecfiler/transactions/managers.py index 03948fa775..fb77e197ea 100644 --- a/django-backend/fecfiler/transactions/managers.py +++ b/django-backend/fecfiler/transactions/managers.py @@ -269,8 +269,10 @@ def get_queryset(self): line_label=Case( # Schedule A When(_form_type="SA11A", then=Value("11(a)")), - When(_form_type="SA11AI", then=Value("11(a)(i)")), - When(_form_type="SA11AII", then=Value("11(a)(i)")), + When(_form_type="SA11AI", itemized=True, then=Value("11(a)(i)")), + When(_form_type="SA11AI", itemized=False, then=Value("11(a)(ii)")), + When(_form_type="SA11AII", itemized=False, then=Value("11(a)(ii)")), + When(_form_type="SA11AII", itemized=True, then=Value("11(a)(i)")), When(_form_type="SA11B", then=Value("11(b)")), When(_form_type="SA11C", then=Value("11(c)")), When(_form_type="SA12", then=Value("12")), @@ -299,8 +301,40 @@ def get_queryset(self): # Schedule E When(_form_type="SE", then=Value("24")), ), - line_label_order_key=Concat( - "line_label", "form_type", output_field=TextField() + line_label_order_key=Case( + # Schedule A + When(_form_type="SA11A", then=Value(3)), + When(_form_type="SA11AI", itemized=True, then=Value(4)), + When(_form_type="SA11AI", itemized=False, then=Value(5)), + When(_form_type="SA11AII", itemized=False, then=Value(5)), + When(_form_type="SA11AII", itemized=True, then=Value(4)), + When(_form_type="SA11B", then=Value(6)), + When(_form_type="SA11C", then=Value(7)), + When(_form_type="SA12", then=Value(8)), + When(_form_type="SA13", then=Value(9)), + When(_form_type="SA14", then=Value(10)), + When(_form_type="SA15", then=Value(11)), + When(_form_type="SA16", then=Value(12)), + When(_form_type="SA17", then=Value(13)), + # Schedule B + When(_form_type="SB21B", then=Value(14)), + When(_form_type="SB22", then=Value(15)), + When(_form_type="SB23", then=Value(16)), + When(_form_type="SB26", then=Value(18)), + When(_form_type="SB27", then=Value(19)), + When(_form_type="SB28A", then=Value(20)), + When(_form_type="SB28B", then=Value(21)), + When(_form_type="SB28C", then=Value(22)), + When(_form_type="SB29", then=Value(23)), + When(_form_type="SB30B", then=Value(24)), + # Schedule C + When(_form_type="SC/10", then=Value(2)), + When(_form_type="SC/9", then=Value(1)), + # Schedule D + When(_form_type="SD9", then=Value(1)), + When(_form_type="SD10", then=Value(2)), + # Schedule E + When(_form_type="SE", then=Value(17)), ), ) .annotate( diff --git a/django-backend/fecfiler/transactions/views.py b/django-backend/fecfiler/transactions/views.py index b6ec48e95a..6cb0f9a435 100644 --- a/django-backend/fecfiler/transactions/views.py +++ b/django-backend/fecfiler/transactions/views.py @@ -64,6 +64,7 @@ class TransactionViewSet(CommitteeOwnedViewSet, ReportViewMixin): filter_backends = [filters.OrderingFilter] ordering_fields = [ "line_label_order_key", + "created", "transaction_type_identifier", "memo_code", "name", From 342ebf5b1a0f1dc44d2d3c6a4ae709acd2d8cebe Mon Sep 17 00:00:00 2001 From: toddlees Date: Tue, 28 Nov 2023 16:25:52 -0500 Subject: [PATCH 042/109] removing stupid bug --- django-backend/fecfiler/web_services/tasks.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/django-backend/fecfiler/web_services/tasks.py b/django-backend/fecfiler/web_services/tasks.py index d841133e09..f9a7cc20f0 100644 --- a/django-backend/fecfiler/web_services/tasks.py +++ b/django-backend/fecfiler/web_services/tasks.py @@ -71,16 +71,12 @@ def submit_to_fec( logger.info(f"FEC API: {FEC_FILING_API}") logger.info(f"api submitter: {api}") + submission = UploadSubmission.objects.get(id=submission_record_id) + submission.save_state(FECSubmissionState.SUBMITTING) + if submission.fecfile_task_state == FECSubmissionState.FAILED: return - try: - submission = UploadSubmission.objects.get(id=submission_record_id) - submission.save_state(FECSubmissionState.SUBMITTING) - except Exception as e: - submission = DotFEC.objects.get(id=dot_fec_id).uploadsubmission_set.first() - submission.save_error(f"ERROR GETTING SUBMISSION {submission_record_id}") - """Get Password""" if not e_filing_password: submission.save_error("No E-Filing Password provided") From 872c007761666e12df0b48fa44cc0ee987075859 Mon Sep 17 00:00:00 2001 From: toddlees Date: Tue, 28 Nov 2023 17:00:58 -0500 Subject: [PATCH 043/109] bring back original messages --- django-backend/fecfiler/reports/views.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/django-backend/fecfiler/reports/views.py b/django-backend/fecfiler/reports/views.py index 41b260f3b7..63cae50cf6 100644 --- a/django-backend/fecfiler/reports/views.py +++ b/django-backend/fecfiler/reports/views.py @@ -53,10 +53,8 @@ def get_status_mapping(): return Case( When(success, then=Value("Submission success")), - When(failed, then=F("upload_submission__fecfile_error")), - When( - upload_exists, then=F("upload_submission__fecfile_task_state") - ), # Value("Submission pending")), + When(failed, then=Value("Submission failure")), + When(upload_exists, then=Value("Submission pending")), default=Value("In progress"), output_field=CharField(), ) From dcddf99414a45f8586b156d9876ad5044892ea46 Mon Sep 17 00:00:00 2001 From: toddlees Date: Tue, 28 Nov 2023 17:24:26 -0500 Subject: [PATCH 044/109] save change --- django-backend/fecfiler/transactions/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/django-backend/fecfiler/transactions/views.py b/django-backend/fecfiler/transactions/views.py index 90908cec6b..8a9169f347 100644 --- a/django-backend/fecfiler/transactions/views.py +++ b/django-backend/fecfiler/transactions/views.py @@ -270,6 +270,7 @@ def save_transaction(self, transaction_data, request): if type(child_transaction_data) is str: child_transaction = self.get_queryset().get(id=child_transaction_data) child_transaction.parent_transaction_id = transaction_instance.id + child_transaction.save() else: child_transaction_data[ "parent_transaction_id" From 5ef15f4b3a76e9646632ff4e9852b561c4b408cb Mon Sep 17 00:00:00 2001 From: toddlees <97105825+toddlees@users.noreply.github.com> Date: Tue, 28 Nov 2023 17:27:06 -0500 Subject: [PATCH 045/109] Update tasks.py --- django-backend/fecfiler/web_services/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django-backend/fecfiler/web_services/tasks.py b/django-backend/fecfiler/web_services/tasks.py index f9a7cc20f0..f87da023d8 100644 --- a/django-backend/fecfiler/web_services/tasks.py +++ b/django-backend/fecfiler/web_services/tasks.py @@ -43,8 +43,8 @@ def create_dot_fec( dot_fec_record = DotFEC(report_id=report_id, file_name=file_name) dot_fec_record.save() - except Exception as e: - submission.save_error(f"Creating .FEC failed {e}") + except Exception: + submission.save_error("Creating .FEC failed") return None if upload_submission_id: From 087374139380189d4a388d9a9284d6da11cbbd57 Mon Sep 17 00:00:00 2001 From: toddlees <97105825+toddlees@users.noreply.github.com> Date: Tue, 28 Nov 2023 17:29:04 -0500 Subject: [PATCH 046/109] Update tasks.py --- django-backend/fecfiler/web_services/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django-backend/fecfiler/web_services/tasks.py b/django-backend/fecfiler/web_services/tasks.py index f87da023d8..03ad7e20f0 100644 --- a/django-backend/fecfiler/web_services/tasks.py +++ b/django-backend/fecfiler/web_services/tasks.py @@ -72,10 +72,10 @@ def submit_to_fec( logger.info(f"api submitter: {api}") submission = UploadSubmission.objects.get(id=submission_record_id) - submission.save_state(FECSubmissionState.SUBMITTING) - if submission.fecfile_task_state == FECSubmissionState.FAILED: return + submission.save_state(FECSubmissionState.SUBMITTING) + """Get Password""" if not e_filing_password: From 17b1709e3a0efcca7a61ef714a250fb48d242d7c Mon Sep 17 00:00:00 2001 From: toddlees Date: Wed, 29 Nov 2023 08:32:54 -0500 Subject: [PATCH 047/109] ordering --- django-backend/fecfiler/transactions/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/django-backend/fecfiler/transactions/views.py b/django-backend/fecfiler/transactions/views.py index 8a9169f347..8920dad982 100644 --- a/django-backend/fecfiler/transactions/views.py +++ b/django-backend/fecfiler/transactions/views.py @@ -64,6 +64,7 @@ class TransactionViewSet(CommitteeOwnedViewSet, ReportViewMixin): filter_backends = [filters.OrderingFilter] ordering_fields = [ "line_label_order_key", + "created", "transaction_type_identifier", "memo_code", "name", From c2318d53213a44e6e90404f3965b87c14abdbae3 Mon Sep 17 00:00:00 2001 From: toddlees Date: Wed, 29 Nov 2023 11:20:33 -0500 Subject: [PATCH 048/109] fix report sorting --- django-backend/fecfiler/f3x_summaries/views.py | 1 - django-backend/fecfiler/reports/form_1m/views.py | 8 -------- django-backend/fecfiler/reports/form_24/views.py | 8 -------- django-backend/fecfiler/reports/form_3x/views.py | 9 --------- django-backend/fecfiler/reports/form_99/views.py | 8 -------- django-backend/fecfiler/reports/views.py | 6 ++++-- 6 files changed, 4 insertions(+), 36 deletions(-) diff --git a/django-backend/fecfiler/f3x_summaries/views.py b/django-backend/fecfiler/f3x_summaries/views.py index ef2ec007a5..80d6835786 100644 --- a/django-backend/fecfiler/f3x_summaries/views.py +++ b/django-backend/fecfiler/f3x_summaries/views.py @@ -63,7 +63,6 @@ class F3XSummaryViewSet(CommitteeOwnedViewSet): "report_code_label", "coverage_through_date", "upload_submission__fec_status", - "submission_status", ] ordering = ["form_type"] diff --git a/django-backend/fecfiler/reports/form_1m/views.py b/django-backend/fecfiler/reports/form_1m/views.py index 2d034c800a..066330c751 100644 --- a/django-backend/fecfiler/reports/form_1m/views.py +++ b/django-backend/fecfiler/reports/form_1m/views.py @@ -20,14 +20,6 @@ class Form1MViewSet(ReportViewSet): ) serializer_class = Form1MSerializer - filter_backends = [filters.OrderingFilter] - ordering_fields = [ - "form_type", - "report_code_label", - "upload_submission__fec_status", - "submission_status", - ] - ordering = ["form_type"] def create(self, request): return super(ModelViewSet, self).create(request) diff --git a/django-backend/fecfiler/reports/form_24/views.py b/django-backend/fecfiler/reports/form_24/views.py index 23b97bc18d..83fcf16583 100644 --- a/django-backend/fecfiler/reports/form_24/views.py +++ b/django-backend/fecfiler/reports/form_24/views.py @@ -20,14 +20,6 @@ class Form24ViewSet(ReportViewSet): ) serializer_class = Form24Serializer - filter_backends = [filters.OrderingFilter] - ordering_fields = [ - "form_type", - "report_code_label", - "upload_submission__fec_status", - "submission_status", - ] - ordering = ["form_type"] def create(self, request): return super(ModelViewSet, self).create(request) diff --git a/django-backend/fecfiler/reports/form_3x/views.py b/django-backend/fecfiler/reports/form_3x/views.py index ecffc541d1..fa6b057ca7 100644 --- a/django-backend/fecfiler/reports/form_3x/views.py +++ b/django-backend/fecfiler/reports/form_3x/views.py @@ -22,15 +22,6 @@ class Form3XViewSet(ReportViewSet): ) serializer_class = Form3XSerializer - filter_backends = [filters.OrderingFilter] - ordering_fields = [ - "form_type", - "report_code_label", - "coverage_through_date", - "upload_submission__fec_status", - "submission_status", - ] - ordering = ["form_type"] @action(detail=False) def coverage_dates(self, request): diff --git a/django-backend/fecfiler/reports/form_99/views.py b/django-backend/fecfiler/reports/form_99/views.py index ec3db06301..3c0b79862d 100644 --- a/django-backend/fecfiler/reports/form_99/views.py +++ b/django-backend/fecfiler/reports/form_99/views.py @@ -20,14 +20,6 @@ class Form99ViewSet(ReportViewSet): ) serializer_class = Form99Serializer - filter_backends = [filters.OrderingFilter] - ordering_fields = [ - "form_type", - "report_code_label", - "upload_submission__fec_status", - "submission_status", - ] - ordering = ["form_type"] def create(self, request): return super(ModelViewSet, self).create(request) diff --git a/django-backend/fecfiler/reports/views.py b/django-backend/fecfiler/reports/views.py index ae5ac7a3af..288bf51b31 100644 --- a/django-backend/fecfiler/reports/views.py +++ b/django-backend/fecfiler/reports/views.py @@ -85,9 +85,11 @@ class ReportViewSet(CommitteeOwnedViewSet): serializer_class = ReportSerializer filter_backends = [filters.OrderingFilter] ordering_fields = [ + "report_code_label", + "coverage_through_date", "form_type", - "upload_submission__fec_status", - "submission_status", + "upload_submission__created", + "report_status", ] ordering = ["form_type"] From 73c2c32a5c1f000d4b1d09df11356e9ba6815223 Mon Sep 17 00:00:00 2001 From: toddlees Date: Wed, 29 Nov 2023 12:35:51 -0500 Subject: [PATCH 049/109] remove unused imports --- django-backend/fecfiler/reports/form_1m/views.py | 1 - django-backend/fecfiler/reports/form_24/views.py | 1 - django-backend/fecfiler/reports/form_3x/views.py | 1 - django-backend/fecfiler/reports/form_99/views.py | 1 - 4 files changed, 4 deletions(-) diff --git a/django-backend/fecfiler/reports/form_1m/views.py b/django-backend/fecfiler/reports/form_1m/views.py index 066330c751..ce572be76d 100644 --- a/django-backend/fecfiler/reports/form_1m/views.py +++ b/django-backend/fecfiler/reports/form_1m/views.py @@ -1,4 +1,3 @@ -from rest_framework import filters from rest_framework.viewsets import ModelViewSet from fecfiler.reports.models import Report from fecfiler.reports.managers import ReportType diff --git a/django-backend/fecfiler/reports/form_24/views.py b/django-backend/fecfiler/reports/form_24/views.py index 83fcf16583..67cce2895e 100644 --- a/django-backend/fecfiler/reports/form_24/views.py +++ b/django-backend/fecfiler/reports/form_24/views.py @@ -1,4 +1,3 @@ -from rest_framework import filters from rest_framework.viewsets import ModelViewSet from fecfiler.reports.models import Report from fecfiler.reports.managers import ReportType diff --git a/django-backend/fecfiler/reports/form_3x/views.py b/django-backend/fecfiler/reports/form_3x/views.py index fa6b057ca7..299d61747d 100644 --- a/django-backend/fecfiler/reports/form_3x/views.py +++ b/django-backend/fecfiler/reports/form_3x/views.py @@ -1,5 +1,4 @@ from django.http import JsonResponse -from rest_framework import filters from rest_framework.decorators import action from rest_framework.viewsets import ModelViewSet from fecfiler.reports.models import Report diff --git a/django-backend/fecfiler/reports/form_99/views.py b/django-backend/fecfiler/reports/form_99/views.py index 3c0b79862d..58f8ea45b5 100644 --- a/django-backend/fecfiler/reports/form_99/views.py +++ b/django-backend/fecfiler/reports/form_99/views.py @@ -1,4 +1,3 @@ -from rest_framework import filters from rest_framework.viewsets import ModelViewSet from fecfiler.reports.models import Report from fecfiler.reports.managers import ReportType From 8ba749e4952e725deb25fafa312f648ee3269328 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Wed, 29 Nov 2023 17:18:08 -0500 Subject: [PATCH 050/109] WIP, actual functionality is turbo-broken??? --- .../fecfiler/web_services/summary/summary.py | 12 +++++++----- .../fecfiler/web_services/summary/tasks.py | 12 ++++++++---- .../fecfiler/web_services/summary/test_tasks.py | 8 +++++--- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/django-backend/fecfiler/web_services/summary/summary.py b/django-backend/fecfiler/web_services/summary/summary.py index 2b6e5a1a12..ca86585a93 100644 --- a/django-backend/fecfiler/web_services/summary/summary.py +++ b/django-backend/fecfiler/web_services/summary/summary.py @@ -27,7 +27,7 @@ def calculate_summary(self): return summary def calculate_summary_column_a(self): - report_transactions = Transaction.objects.filter(report=self.report) + report_transactions = Transaction.objects.filter(report_id=self.report.id) summary = report_transactions.aggregate( line_11ai=self.get_line("SA11AI", itemized=True), line_11aii=self.get_line("SA11AI", itemized=False), @@ -57,7 +57,6 @@ def calculate_summary_column_a(self): temp_sd10=self.get_line("SD10", field="balance_at_close") ) - summary["line_6b"] = 0 if self.previous_report and self.previous_report.form_3x: summary["line_6b"] = self.previous_report.form_3x.L8_cash_on_hand_at_close_period # noqa: E501 @@ -148,7 +147,7 @@ def calculate_summary_column_a(self): + summary["line_18c"] ) summary["line_6d"] = ( - summary["line_6b"] + summary.get("line_6b", 0) + summary["line_6c"] ) summary["line_7"] = ( @@ -182,6 +181,8 @@ def calculate_summary_column_b(self): committee_account=committee, date__year=report_year, date__lte=report_date, ) + logger.info("This many transactions! "+str(len(ytd_transactions))) + # build summary summary = ytd_transactions.aggregate( line_11ai=self.get_line("SA11AI", itemized=True), @@ -207,7 +208,6 @@ def calculate_summary_column_b(self): line_30b=self.get_line("SB30B"), ) - summary["line_6a"] = 0 if self.previous_report and self.previous_report.form_3x: summary["line_6a"] = self.previous_report.form_3x.L8_cash_on_hand_close_ytd @@ -290,7 +290,7 @@ def calculate_summary_column_b(self): + summary["line_18c"] ) summary["line_6d"] = ( - summary["line_6a"] + summary.get("line_6a", 0) + summary["line_6c"] ) summary["line_7"] = ( @@ -308,6 +308,8 @@ def calculate_summary_column_b(self): - summary["line_18c"] ) + logger.info(summary.values()) + return summary def get_line(self, form_type, field="amount", itemized=None): diff --git a/django-backend/fecfiler/web_services/summary/tasks.py b/django-backend/fecfiler/web_services/summary/tasks.py index 7d4d561dbd..eaa979be58 100644 --- a/django-backend/fecfiler/web_services/summary/tasks.py +++ b/django-backend/fecfiler/web_services/summary/tasks.py @@ -47,18 +47,22 @@ def calculate_summary(report_id): calculation_token=calculation_token, calculation_status=CalculationState.CALCULATING ) + claimed_reports = list(reports_to_recalculate) - for report in reports_to_recalculate: + for report in claimed_reports: summary_service = SummaryService(report) summary = summary_service.calculate_summary() a = summary["a"] b = summary["b"] # line 6a - report.form_3x.L6a_cash_on_hand_jan_1_ytd = b.get("line_6a", 0) - report.form_3x.L6a_year_for_above_ytd = b.get("line_6a", 0) + if "line_6a" in b: + report.form_3x.L6a_cash_on_hand_jan_1_ytd = b["line_6a"] # line 6b - report.form_3x.L6b_cash_on_hand_beginning_period = a.get("line_6b", 0) + if "line_6b" in a: + report.form_3x.L6b_cash_on_hand_beginning_period = a["line_6b"] + else: + report.form_3x.L6b_cash_on_hand_beginning_period = report.form_3x.L6a_cash_on_hand_jan_1_ytd # noqa: E501 # line 6c report.form_3x.L6c_total_receipts_period = a.get("line_6c", 0) report.form_3x.L6c_total_receipts_ytd = b.get("line_6c", 0) diff --git a/django-backend/fecfiler/web_services/summary/test_tasks.py b/django-backend/fecfiler/web_services/summary/test_tasks.py index 53780c5b8c..6b1419b11a 100644 --- a/django-backend/fecfiler/web_services/summary/test_tasks.py +++ b/django-backend/fecfiler/web_services/summary/test_tasks.py @@ -38,9 +38,6 @@ def test_report_with_no_transactions(self): self.assertEqual( report.form_3x.L37_offsets_to_operating_expenditures_period, Decimal("0") ) - self.assertEqual( - report.form_3x.L6b_cash_on_hand_beginning_period, Decimal("0") - ) self.assertEqual(report.calculation_status, CalculationState.SUCCEEDED.value) def test_report_b(self): @@ -55,6 +52,7 @@ def test_report_b(self): calculate_summary("b6d60d2d-d926-4e89-ad4b-000000000002") calculated_report = Report.objects.get(id="b6d60d2d-d926-4e89-ad4b-000000000002") + calculated_prev_report = Report.objects.get(id="b6d60d2d-d926-4e89-ad4b-c47d152a66ae") # noqa: E501 self.assertEqual( calculated_report.form_3x.L6b_cash_on_hand_beginning_period, @@ -64,3 +62,7 @@ def test_report_b(self): calculated_report.form_3x.L6a_cash_on_hand_jan_1_ytd, Decimal("18985.17") ) + self.assertEqual( + calculated_prev_report.form_3x.L15_offsets_to_operating_expenditures_refunds_period, # noqa: E501 + Decimal("2125.79"), + ) From 5a81b1690cf1dc62fb53db7f02bb4dbf38cef580 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Wed, 29 Nov 2023 17:28:00 -0500 Subject: [PATCH 051/109] Add reatt_redes_total --- .../fecfiler/transactions/managers.py | 2 ++ .../fecfiler/transactions/serializers.py | 16 ++++++++++ .../fecfiler/transactions/tests/test_views.py | 29 ++++++++++++------- django-backend/fecfiler/transactions/views.py | 15 ---------- 4 files changed, 36 insertions(+), 26 deletions(-) diff --git a/django-backend/fecfiler/transactions/managers.py b/django-backend/fecfiler/transactions/managers.py index 03948fa775..9b2176815d 100644 --- a/django-backend/fecfiler/transactions/managers.py +++ b/django-backend/fecfiler/transactions/managers.py @@ -255,12 +255,14 @@ def get_queryset(self): output_field=TextField(), ), back_reference_tran_id_number=Coalesce( + F("reatt_redes__transaction_id"), F("parent_transaction__transaction_id"), F("debt__transaction_id"), F("loan__transaction_id"), Value(None), ), back_reference_sched_name=Coalesce( + F("reatt_redes___form_type"), F("parent_transaction___form_type"), F("debt___form_type"), F("loan___form_type"), diff --git a/django-backend/fecfiler/transactions/serializers.py b/django-backend/fecfiler/transactions/serializers.py index 17d4a7cd81..aa9f1e345b 100644 --- a/django-backend/fecfiler/transactions/serializers.py +++ b/django-backend/fecfiler/transactions/serializers.py @@ -1,5 +1,7 @@ import logging +from django.db.models import Sum +from decimal import Decimal from fecfiler.committee_accounts.serializers import CommitteeOwnedSerializer from fecfiler.contacts.serializers import LinkedContactSerializerMixin from fecfiler.memo_text.serializers import LinkedMemoTextSerializerMixin @@ -220,11 +222,25 @@ def to_representation(self, instance): if schedule_a: representation["contribution_aggregate"] = representation.get("aggregate") + + # For REATTRIBUTED transactions, calculate the amount that has been reattributed for the transaction + total = instance.reatt_redes_associations.filter( + schedule_a__reattribution_redesignation_tag="REATTRIBUTION_TO" + ).aggregate(Sum("amount"))['amount__sum'] or 0.0 + representation["reatt_redes_total"] = str(total) + for property in schedule_a: if not representation.get(property): representation[property] = schedule_a[property] if schedule_b: representation["aggregate_amount"] = representation.get("aggregate") + + # For REDESIGNATED transactions, calculate the amount that has been redesignated for the transaction + total = instance.reatt_redes_associations.filter( + schedule_b__reattribution_redesignation_tag="REDESIGNATION_TO" + ).aggregate(Sum("amount"))['amount__sum'] or 0.0 + representation["reatt_redes_total"] = str(total) + for property in schedule_b: if not representation.get(property): representation[property] = schedule_b[property] diff --git a/django-backend/fecfiler/transactions/tests/test_views.py b/django-backend/fecfiler/transactions/tests/test_views.py index f22b991602..087d022611 100644 --- a/django-backend/fecfiler/transactions/tests/test_views.py +++ b/django-backend/fecfiler/transactions/tests/test_views.py @@ -159,21 +159,28 @@ def test_inherited_election_aggregate(self): self.assertEqual(transaction.get("calendar_ytd_per_election_office"), 58.00) def test_multisave_transactions(self): - - request = self.request([self.payloads["IN_KIND"] for _ in range(3)]) - response = TransactionViewSet().save_transactions(request) - txn1 = deepcopy(self.payloads["IN_KIND"]) - txn1["id"] = str(response.data[0]["id"]) txn1["contributor_last_name"] = "one" txn2 = deepcopy(self.payloads["IN_KIND"]) - txn2["id"] = str(response.data[1]["id"]) txn2["contributor_last_name"] = "two" txn3 = deepcopy(self.payloads["IN_KIND"]) - txn3["id"] = str(response.data[2]["id"]) txn3["contributor_last_name"] = "three" - request = self.request([txn1, txn2, txn3]) - response = TransactionViewSet().save_transactions(request) - self.assertEqual(len(response.data), 3) - # self.assertEqual("one", updated_transaction.schedule_a.contributor_last_name) + payload = [txn1, txn2, txn3] + + view_set = TransactionViewSet() + view_set.format_kwarg = {} + request = self.factory.put( + "/api/v1/transactions/multisave/", + json.dumps(payload), + content_type="application/json", + ) + request.user = self.user + request.data = deepcopy(payload) + view_set.request = request + + view_set = TransactionViewSet() + response = view_set.save_transactions(self.request(payload)) + transactions = response.data + self.assertEqual(len(transactions), 3) + # self.assertEqual("one", transactions[0]["contributor_last_name"]) diff --git a/django-backend/fecfiler/transactions/views.py b/django-backend/fecfiler/transactions/views.py index 88e5f34605..e3f5d1365a 100644 --- a/django-backend/fecfiler/transactions/views.py +++ b/django-backend/fecfiler/transactions/views.py @@ -288,21 +288,6 @@ def save_transactions(self, request): [TransactionSerializer().to_representation(data) for data in saved_data] ) - @action(detail=False, methods=["get"], url_path=r"reattributions-redesignations") - def transaction_reattributions_redesignations(self, request): - transaction_id = request.query_params.get("transaction_id", None) - query = self.get_queryset().filter( - Q(reatt_redes_id=transaction_id), - ) - transactions = query.all() - - if len(transactions) > 0: - return Response(data=[self.get_serializer(t).data for t in transactions]) - - response = {"message": "Did not find reattributions or redesignations."} - return Response(response, status=status.HTTP_404_NOT_FOUND) - - def noop(transaction, is_existing): pass From 6b0c5216859464e0854442cf6326b4e4183e1ac8 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Wed, 29 Nov 2023 17:53:07 -0500 Subject: [PATCH 052/109] Fix linting issues reported by SonarCloud --- django-backend/fecfiler/transactions/serializers.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/django-backend/fecfiler/transactions/serializers.py b/django-backend/fecfiler/transactions/serializers.py index aa9f1e345b..55d2942564 100644 --- a/django-backend/fecfiler/transactions/serializers.py +++ b/django-backend/fecfiler/transactions/serializers.py @@ -1,7 +1,6 @@ import logging from django.db.models import Sum -from decimal import Decimal from fecfiler.committee_accounts.serializers import CommitteeOwnedSerializer from fecfiler.contacts.serializers import LinkedContactSerializerMixin from fecfiler.memo_text.serializers import LinkedMemoTextSerializerMixin @@ -226,7 +225,7 @@ def to_representation(self, instance): # For REATTRIBUTED transactions, calculate the amount that has been reattributed for the transaction total = instance.reatt_redes_associations.filter( schedule_a__reattribution_redesignation_tag="REATTRIBUTION_TO" - ).aggregate(Sum("amount"))['amount__sum'] or 0.0 + ).aggregate(Sum("amount"))['amount__sum'] or 0.0 representation["reatt_redes_total"] = str(total) for property in schedule_a: @@ -238,7 +237,7 @@ def to_representation(self, instance): # For REDESIGNATED transactions, calculate the amount that has been redesignated for the transaction total = instance.reatt_redes_associations.filter( schedule_b__reattribution_redesignation_tag="REDESIGNATION_TO" - ).aggregate(Sum("amount"))['amount__sum'] or 0.0 + ).aggregate(Sum("amount"))['amount__sum'] or 0.0 representation["reatt_redes_total"] = str(total) for property in schedule_b: From 604028cf080b5c4b8ca091740d7cb6f1f3a60fac Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Wed, 29 Nov 2023 17:58:55 -0500 Subject: [PATCH 053/109] Fix linting issues reported by SonarCloud --- django-backend/fecfiler/transactions/serializers.py | 6 ++++-- django-backend/fecfiler/transactions/views.py | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/django-backend/fecfiler/transactions/serializers.py b/django-backend/fecfiler/transactions/serializers.py index 55d2942564..a3366c87c0 100644 --- a/django-backend/fecfiler/transactions/serializers.py +++ b/django-backend/fecfiler/transactions/serializers.py @@ -222,7 +222,8 @@ def to_representation(self, instance): if schedule_a: representation["contribution_aggregate"] = representation.get("aggregate") - # For REATTRIBUTED transactions, calculate the amount that has been reattributed for the transaction + # For REATTRIBUTED transactions, calculate the amount that has + # been reattributed for the transaction total = instance.reatt_redes_associations.filter( schedule_a__reattribution_redesignation_tag="REATTRIBUTION_TO" ).aggregate(Sum("amount"))['amount__sum'] or 0.0 @@ -234,7 +235,8 @@ def to_representation(self, instance): if schedule_b: representation["aggregate_amount"] = representation.get("aggregate") - # For REDESIGNATED transactions, calculate the amount that has been redesignated for the transaction + # For REDESIGNATED transactions, calculate the amount that has + # been redesignated for the transaction total = instance.reatt_redes_associations.filter( schedule_b__reattribution_redesignation_tag="REDESIGNATION_TO" ).aggregate(Sum("amount"))['amount__sum'] or 0.0 diff --git a/django-backend/fecfiler/transactions/views.py b/django-backend/fecfiler/transactions/views.py index e3f5d1365a..0ad101a0c1 100644 --- a/django-backend/fecfiler/transactions/views.py +++ b/django-backend/fecfiler/transactions/views.py @@ -288,6 +288,7 @@ def save_transactions(self, request): [TransactionSerializer().to_representation(data) for data in saved_data] ) + def noop(transaction, is_existing): pass From 0954a8b883943465f0615565b2874ec7278806fb Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Thu, 30 Nov 2023 12:54:04 -0500 Subject: [PATCH 054/109] Fix linting errors reported by SonarCloud --- django-backend/fecfiler/reports/views.py | 2 +- django-backend/fecfiler/web_services/tasks.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/django-backend/fecfiler/reports/views.py b/django-backend/fecfiler/reports/views.py index 63cae50cf6..4c248bca7b 100644 --- a/django-backend/fecfiler/reports/views.py +++ b/django-backend/fecfiler/reports/views.py @@ -9,7 +9,7 @@ from fecfiler.memo_text.models import MemoText from fecfiler.web_services.models import DotFEC, UploadSubmission, WebPrintSubmission from .serializers import ReportSerializer -from django.db.models import Case, Value, When, Q, F, CharField +from django.db.models import Case, Value, When, Q, CharField import logging logger = logging.getLogger(__name__) diff --git a/django-backend/fecfiler/web_services/tasks.py b/django-backend/fecfiler/web_services/tasks.py index 03ad7e20f0..2243bb480f 100644 --- a/django-backend/fecfiler/web_services/tasks.py +++ b/django-backend/fecfiler/web_services/tasks.py @@ -76,7 +76,6 @@ def submit_to_fec( return submission.save_state(FECSubmissionState.SUBMITTING) - """Get Password""" if not e_filing_password: submission.save_error("No E-Filing Password provided") From 0563c58fa1df364a25404d13da96ab4d555b8547 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Fri, 1 Dec 2023 11:04:05 -0500 Subject: [PATCH 055/109] Removes debug print calls --- django-backend/fecfiler/web_services/summary/summary.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/django-backend/fecfiler/web_services/summary/summary.py b/django-backend/fecfiler/web_services/summary/summary.py index ca86585a93..ec55aa791f 100644 --- a/django-backend/fecfiler/web_services/summary/summary.py +++ b/django-backend/fecfiler/web_services/summary/summary.py @@ -181,8 +181,6 @@ def calculate_summary_column_b(self): committee_account=committee, date__year=report_year, date__lte=report_date, ) - logger.info("This many transactions! "+str(len(ytd_transactions))) - # build summary summary = ytd_transactions.aggregate( line_11ai=self.get_line("SA11AI", itemized=True), @@ -308,8 +306,6 @@ def calculate_summary_column_b(self): - summary["line_18c"] ) - logger.info(summary.values()) - return summary def get_line(self, form_type, field="amount", itemized=None): From 81ac0a3842b92f9684e435cc31aeaef3ca4a905c Mon Sep 17 00:00:00 2001 From: toddlees Date: Fri, 1 Dec 2023 11:06:36 -0500 Subject: [PATCH 056/109] loan_agreement_id --- django-backend/fecfiler/transactions/serializers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/django-backend/fecfiler/transactions/serializers.py b/django-backend/fecfiler/transactions/serializers.py index fc4c3c982a..73a6bc3ff4 100644 --- a/django-backend/fecfiler/transactions/serializers.py +++ b/django-backend/fecfiler/transactions/serializers.py @@ -226,11 +226,11 @@ def to_representation(self, instance): if not representation.get(property): representation[property] = schedule_b[property] if schedule_c: - loan_agreement = instance.children.filter( + loan_agreement_id = instance.children.filter( transaction_type_identifier="C1_LOAN_AGREEMENT" ).first() - representation["loan_agreement"] = ( - loan_agreement.id if loan_agreement else None + representation["loan_agreement_id"] = ( + loan_agreement_id.id if loan_agreement_id else None ) for property in schedule_c: if not representation.get(property): From 890aba14c521788364fa9df97f469e1e7e29b61d Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Fri, 1 Dec 2023 11:47:24 -0500 Subject: [PATCH 057/109] Merge conflicts in migrations and a math fix --- ...culation_token.py => 0006_report_calculation_token.py} | 2 +- django-backend/fecfiler/web_services/summary/summary.py | 4 ++++ django-backend/fecfiler/web_services/summary/tasks.py | 8 ++------ 3 files changed, 7 insertions(+), 7 deletions(-) rename django-backend/fecfiler/reports/migrations/{0005_report_calculation_token.py => 0006_report_calculation_token.py} (87%) diff --git a/django-backend/fecfiler/reports/migrations/0005_report_calculation_token.py b/django-backend/fecfiler/reports/migrations/0006_report_calculation_token.py similarity index 87% rename from django-backend/fecfiler/reports/migrations/0005_report_calculation_token.py rename to django-backend/fecfiler/reports/migrations/0006_report_calculation_token.py index 3a72007b5f..408fb3d739 100644 --- a/django-backend/fecfiler/reports/migrations/0005_report_calculation_token.py +++ b/django-backend/fecfiler/reports/migrations/0006_report_calculation_token.py @@ -6,7 +6,7 @@ class Migration(migrations.Migration): dependencies = [ - ('reports', '0004_form1m_report_form_1m'), + ('reports', '0005_form99_text'), ] operations = [ diff --git a/django-backend/fecfiler/web_services/summary/summary.py b/django-backend/fecfiler/web_services/summary/summary.py index 47cb2555e3..a2a5ae1531 100644 --- a/django-backend/fecfiler/web_services/summary/summary.py +++ b/django-backend/fecfiler/web_services/summary/summary.py @@ -59,6 +59,8 @@ def calculate_summary_column_a(self): if self.previous_report and self.previous_report.form_3x: summary["line_6b"] = self.previous_report.form_3x.L8_cash_on_hand_at_close_period # noqa: E501 + else: + summary["line_6b"] = self.report.form_3x.L6a_cash_on_hand_jan_1_ytd # noqa: E501 summary["line_9"] = ( summary["temp_sc9"] @@ -208,6 +210,8 @@ def calculate_summary_column_b(self): if self.previous_report and self.previous_report.form_3x: summary["line_6a"] = self.previous_report.form_3x.L8_cash_on_hand_close_ytd + else: + summary["line_6a"] = self.report.form_3x.L6a_cash_on_hand_jan_1_ytd summary["line_11aiii"] = ( summary["line_11ai"] diff --git a/django-backend/fecfiler/web_services/summary/tasks.py b/django-backend/fecfiler/web_services/summary/tasks.py index eaa979be58..9491b4021d 100644 --- a/django-backend/fecfiler/web_services/summary/tasks.py +++ b/django-backend/fecfiler/web_services/summary/tasks.py @@ -56,13 +56,9 @@ def calculate_summary(report_id): b = summary["b"] # line 6a - if "line_6a" in b: - report.form_3x.L6a_cash_on_hand_jan_1_ytd = b["line_6a"] + report.form_3x.L6a_cash_on_hand_jan_1_ytd = b["line_6a"] # line 6b - if "line_6b" in a: - report.form_3x.L6b_cash_on_hand_beginning_period = a["line_6b"] - else: - report.form_3x.L6b_cash_on_hand_beginning_period = report.form_3x.L6a_cash_on_hand_jan_1_ytd # noqa: E501 + report.form_3x.L6b_cash_on_hand_beginning_period = a["line_6b"] # line 6c report.form_3x.L6c_total_receipts_period = a.get("line_6c", 0) report.form_3x.L6c_total_receipts_ytd = b.get("line_6c", 0) From f35084bb6fba523a9a2b31911091a95c0f9a8791 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Fri, 1 Dec 2023 11:57:40 -0500 Subject: [PATCH 058/109] Fixes unit tests --- django-backend/fecfiler/web_services/summary/test_summary.py | 4 ++-- django-backend/fecfiler/web_services/summary/test_tasks.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/django-backend/fecfiler/web_services/summary/test_summary.py b/django-backend/fecfiler/web_services/summary/test_summary.py index fe45553502..2fbac7cf78 100644 --- a/django-backend/fecfiler/web_services/summary/test_summary.py +++ b/django-backend/fecfiler/web_services/summary/test_summary.py @@ -28,7 +28,7 @@ def test_calculate_summary_column_a(self): self.assertEqual( summary_a["line_6d"], Decimal("0") + # line_6b - + Decimal("18085.17") # line_6c + + Decimal("18146.17") # line_6c ) self.assertEqual( summary_a["line_7"], @@ -129,7 +129,7 @@ def test_calculate_summary_column_b(self): self.assertEqual( summary_b["line_6d"], Decimal("0") # line_6a - + Decimal("18985.17") # line_6c + + Decimal("19046.17") # line_6c ) self.assertEqual( summary_b["line_7"], diff --git a/django-backend/fecfiler/web_services/summary/test_tasks.py b/django-backend/fecfiler/web_services/summary/test_tasks.py index 6b1419b11a..fb238a8c60 100644 --- a/django-backend/fecfiler/web_services/summary/test_tasks.py +++ b/django-backend/fecfiler/web_services/summary/test_tasks.py @@ -56,11 +56,11 @@ def test_report_b(self): self.assertEqual( calculated_report.form_3x.L6b_cash_on_hand_beginning_period, - Decimal("18085.17") + Decimal("18146.17") ) self.assertEqual( calculated_report.form_3x.L6a_cash_on_hand_jan_1_ytd, - Decimal("18985.17") + Decimal("19046.17") ) self.assertEqual( calculated_prev_report.form_3x.L15_offsets_to_operating_expenditures_refunds_period, # noqa: E501 From f593daa4fc87f7aa2ca958dd005225f41f077461 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Fri, 1 Dec 2023 12:13:09 -0500 Subject: [PATCH 059/109] Small tweak to loan_agreement_id --- django-backend/fecfiler/transactions/serializers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django-backend/fecfiler/transactions/serializers.py b/django-backend/fecfiler/transactions/serializers.py index 73a6bc3ff4..68ae92f4fe 100644 --- a/django-backend/fecfiler/transactions/serializers.py +++ b/django-backend/fecfiler/transactions/serializers.py @@ -226,11 +226,11 @@ def to_representation(self, instance): if not representation.get(property): representation[property] = schedule_b[property] if schedule_c: - loan_agreement_id = instance.children.filter( + loan_agreement = instance.children.filter( transaction_type_identifier="C1_LOAN_AGREEMENT" ).first() representation["loan_agreement_id"] = ( - loan_agreement_id.id if loan_agreement_id else None + loan_agreement.id if loan_agreement else None ) for property in schedule_c: if not representation.get(property): From 4e1a5c9d11178d2f8b78933522aa416250775467 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Fri, 1 Dec 2023 12:16:21 -0500 Subject: [PATCH 060/109] Makes a tweak for consistency --- django-backend/fecfiler/web_services/summary/summary.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django-backend/fecfiler/web_services/summary/summary.py b/django-backend/fecfiler/web_services/summary/summary.py index a2a5ae1531..44cfa827b9 100644 --- a/django-backend/fecfiler/web_services/summary/summary.py +++ b/django-backend/fecfiler/web_services/summary/summary.py @@ -149,7 +149,7 @@ def calculate_summary_column_a(self): + summary["line_18c"] ) summary["line_6d"] = ( - summary.get("line_6b", 0) + summary["line_6b"] + summary["line_6c"] ) summary["line_7"] = ( @@ -292,7 +292,7 @@ def calculate_summary_column_b(self): + summary["line_18c"] ) summary["line_6d"] = ( - summary.get("line_6a", 0) + summary["line_6a"] + summary["line_6c"] ) summary["line_7"] = ( From 2176d6e9b98f40ac2fb83be74b5709e3fae6c320 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Fri, 1 Dec 2023 12:17:51 -0500 Subject: [PATCH 061/109] Renames a test for clarity --- django-backend/fecfiler/web_services/summary/test_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django-backend/fecfiler/web_services/summary/test_tasks.py b/django-backend/fecfiler/web_services/summary/test_tasks.py index fb238a8c60..58158ebb0c 100644 --- a/django-backend/fecfiler/web_services/summary/test_tasks.py +++ b/django-backend/fecfiler/web_services/summary/test_tasks.py @@ -40,7 +40,7 @@ def test_report_with_no_transactions(self): ) self.assertEqual(report.calculation_status, CalculationState.SUCCEEDED.value) - def test_report_b(self): + def test_report_group_recalculation(self): report = Report.objects.get(id="b6d60d2d-d926-4e89-ad4b-000000000002") previous_report = Report.objects.get(id="b6d60d2d-d926-4e89-ad4b-c47d152a66ae") From db1317ca6052748bea97f44b9d1e8663af76d1f8 Mon Sep 17 00:00:00 2001 From: toddlees Date: Fri, 1 Dec 2023 12:26:32 -0500 Subject: [PATCH 062/109] guarantor and loan agreement common name not set up --- django-backend/fecfiler/transactions/views.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/django-backend/fecfiler/transactions/views.py b/django-backend/fecfiler/transactions/views.py index 8920dad982..a2f1923e9b 100644 --- a/django-backend/fecfiler/transactions/views.py +++ b/django-backend/fecfiler/transactions/views.py @@ -34,6 +34,7 @@ class TransactionListPagination(pagination.PageNumberPagination): "schedule_a__contributor_organization_name", "schedule_b__payee_organization_name", "schedule_c__lender_organization_name", + "schedule_c1__lender_organization_name", "schedule_d__creditor_organization_name", "schedule_e__payee_organization_name", ), @@ -42,6 +43,7 @@ class TransactionListPagination(pagination.PageNumberPagination): "schedule_a__contributor_last_name", "schedule_b__payee_last_name", "schedule_c__lender_last_name", + "schedule_c2__guarantor_last_name", "schedule_d__creditor_last_name", "schedule_e__payee_last_name", ), @@ -50,6 +52,7 @@ class TransactionListPagination(pagination.PageNumberPagination): "schedule_a__contributor_first_name", "schedule_b__payee_first_name", "schedule_c__lender_first_name", + "schedule_c2__guarantor_first_name", "schedule_d__creditor_first_name", "schedule_e__payee_first_name", ), From cf9229c326848f35ff4bb70f71c3499040a0a4fb Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Mon, 4 Dec 2023 13:25:41 -0500 Subject: [PATCH 063/109] Adds a comment for calculation_token --- django-backend/fecfiler/reports/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django-backend/fecfiler/reports/models.py b/django-backend/fecfiler/reports/models.py index 83a2e5fcdf..190e01e8fb 100644 --- a/django-backend/fecfiler/reports/models.py +++ b/django-backend/fecfiler/reports/models.py @@ -42,7 +42,7 @@ class Report(SoftDeleteModel, CommitteeOwnedModel): treasurer_suffix = models.TextField(null=True, blank=True) date_signed = models.DateField(null=True, blank=True) calculation_status = models.CharField(max_length=255, null=True, blank=True) - calculation_token = models.UUIDField( + calculation_token = models.UUIDField( # Prevents race conditions in summary calc. null=True, blank=True, default=None From 8d0a46f294a083bbeae5ae53c506e000ff56d7d7 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Mon, 4 Dec 2023 16:38:16 -0500 Subject: [PATCH 064/109] Fixes compounding of transaction amounts --- .../fecfiler/web_services/summary/summary.py | 13 ++++++++----- .../fecfiler/web_services/summary/test_summary.py | 5 +++-- .../fecfiler/web_services/summary/test_tasks.py | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/django-backend/fecfiler/web_services/summary/summary.py b/django-backend/fecfiler/web_services/summary/summary.py index 44cfa827b9..8abc631b42 100644 --- a/django-backend/fecfiler/web_services/summary/summary.py +++ b/django-backend/fecfiler/web_services/summary/summary.py @@ -19,9 +19,12 @@ def __init__(self, report) -> None: ).order_by("-coverage_through_date").first() def calculate_summary(self): + summary_a = self.calculate_summary_column_a() + summary_b = self.calculate_summary_column_b(summary_a) + summary = { - "a": self.calculate_summary_column_a(), - "b": self.calculate_summary_column_b(), + "a": summary_a, + "b": summary_b, } return summary @@ -174,7 +177,7 @@ def calculate_summary_column_a(self): return summary - def calculate_summary_column_b(self): + def calculate_summary_column_b(self, summary_a): committee = self.report.committee_account report_date = self.report.coverage_through_date report_year = report_date.year @@ -293,14 +296,14 @@ def calculate_summary_column_b(self): ) summary["line_6d"] = ( summary["line_6a"] - + summary["line_6c"] + + summary_a["line_6c"] ) summary["line_7"] = ( summary["line_31"] ) summary["line_8"] = ( summary["line_6d"] - - summary["line_7"] + - summary_a["line_7"] ) summary["line_19"] = ( summary["line_6c"] diff --git a/django-backend/fecfiler/web_services/summary/test_summary.py b/django-backend/fecfiler/web_services/summary/test_summary.py index 2fbac7cf78..f43a034c38 100644 --- a/django-backend/fecfiler/web_services/summary/test_summary.py +++ b/django-backend/fecfiler/web_services/summary/test_summary.py @@ -120,6 +120,7 @@ def test_calculate_summary_column_b(self): summary_service = SummaryService(f3x) summary = summary_service.calculate_summary() + summary_a = summary["a"] summary_b = summary["b"] t = Transaction.objects.get(id="aaaaaaaa-4d75-46f0-bce2-111000000001") @@ -129,7 +130,7 @@ def test_calculate_summary_column_b(self): self.assertEqual( summary_b["line_6d"], Decimal("0") # line_6a - + Decimal("19046.17") # line_6c + + Decimal("18146.17") # line_6c ) self.assertEqual( summary_b["line_7"], @@ -147,7 +148,7 @@ def test_calculate_summary_column_b(self): self.assertEqual( summary_b["line_8"], summary_b["line_6d"] - - summary_b["line_7"] + - summary_a["line_7"] ) self.assertEqual(summary_b["line_11ai"], Decimal("10000.23")) self.assertEqual(summary_b["line_11aii"], Decimal("103.77")) diff --git a/django-backend/fecfiler/web_services/summary/test_tasks.py b/django-backend/fecfiler/web_services/summary/test_tasks.py index 58158ebb0c..8449185b1b 100644 --- a/django-backend/fecfiler/web_services/summary/test_tasks.py +++ b/django-backend/fecfiler/web_services/summary/test_tasks.py @@ -60,7 +60,7 @@ def test_report_group_recalculation(self): ) self.assertEqual( calculated_report.form_3x.L6a_cash_on_hand_jan_1_ytd, - Decimal("19046.17") + Decimal("18146.17") ) self.assertEqual( calculated_prev_report.form_3x.L15_offsets_to_operating_expenditures_refunds_period, # noqa: E501 From 82ad133a1fe15faedde7afe9b99b647e9a7b83ac Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Tue, 5 Dec 2023 15:29:44 -0500 Subject: [PATCH 065/109] Changes L6a calculation to be based on the previous year's Cash on Hand --- .../fecfiler/web_services/summary/summary.py | 39 +++++++++++++------ .../web_services/summary/test_summary.py | 6 +-- .../web_services/summary/test_tasks.py | 2 +- 3 files changed, 32 insertions(+), 15 deletions(-) diff --git a/django-backend/fecfiler/web_services/summary/summary.py b/django-backend/fecfiler/web_services/summary/summary.py index 8abc631b42..b10fe76bcd 100644 --- a/django-backend/fecfiler/web_services/summary/summary.py +++ b/django-backend/fecfiler/web_services/summary/summary.py @@ -1,7 +1,7 @@ from decimal import Decimal from fecfiler.transactions.models import Transaction from fecfiler.reports.models import Report -from django.db.models import Q, Sum +from django.db.models import Q, Sum, Case, When from django.db.models.functions import Coalesce import logging @@ -19,8 +19,8 @@ def __init__(self, report) -> None: ).order_by("-coverage_through_date").first() def calculate_summary(self): - summary_a = self.calculate_summary_column_a() - summary_b = self.calculate_summary_column_b(summary_a) + summary_b = self.calculate_summary_column_b() + summary_a = self.calculate_summary_column_a(summary_b) summary = { "a": summary_a, @@ -29,7 +29,7 @@ def calculate_summary(self): return summary - def calculate_summary_column_a(self): + def calculate_summary_column_a(self, summary_b): report_transactions = Transaction.objects.filter(report_id=self.report.id) summary = report_transactions.aggregate( line_11ai=self.get_line("SA11AI"), @@ -63,7 +63,10 @@ def calculate_summary_column_a(self): if self.previous_report and self.previous_report.form_3x: summary["line_6b"] = self.previous_report.form_3x.L8_cash_on_hand_at_close_period # noqa: E501 else: - summary["line_6b"] = self.report.form_3x.L6a_cash_on_hand_jan_1_ytd # noqa: E501 + summary["line_6b"] = summary_b["line_6a"] + + if summary["line_6b"] == None: + summary["line_6b"] = Decimal("0.00") summary["line_9"] = ( summary["temp_sc9"] @@ -177,7 +180,7 @@ def calculate_summary_column_a(self): return summary - def calculate_summary_column_b(self, summary_a): + def calculate_summary_column_b(self): committee = self.report.committee_account report_date = self.report.coverage_through_date report_year = report_date.year @@ -211,10 +214,24 @@ def calculate_summary_column_b(self, summary_a): line_30b=self.get_line("SB30B"), ) - if self.previous_report and self.previous_report.form_3x: - summary["line_6a"] = self.previous_report.form_3x.L8_cash_on_hand_close_ytd + reports_from_prior_years = Report.objects.filter( + committee_account=self.report.committee_account, + coverage_through_date__year__lt=self.report.coverage_from_date.year, + form_3x__isnull=False + ).order_by("coverage_from_date") + + if len(reports_from_prior_years): + summary["line_6a"] = reports_from_prior_years.last().form_3x.L8_cash_on_hand_close_ytd # noqa: E501 else: - summary["line_6a"] = self.report.form_3x.L6a_cash_on_hand_jan_1_ytd + earliest_l6a_source_in_year = Report.objects.filter( + coverage_through_date__year=self.report.coverage_from_date.year, + form_3x__L6a_cash_on_hand_jan_1_ytd__gt=0 + ).order_by("coverage_from_date").first() + + if earliest_l6a_source_in_year: + summary["line_6a"] = earliest_l6a_source_in_year.form_3x.L6a_cash_on_hand_jan_1_ytd # noqa: E501 + else: + summary["line_6a"] = 0 summary["line_11aiii"] = ( summary["line_11ai"] @@ -296,14 +313,14 @@ def calculate_summary_column_b(self, summary_a): ) summary["line_6d"] = ( summary["line_6a"] - + summary_a["line_6c"] + + summary["line_6c"] ) summary["line_7"] = ( summary["line_31"] ) summary["line_8"] = ( summary["line_6d"] - - summary_a["line_7"] + - summary["line_7"] ) summary["line_19"] = ( summary["line_6c"] diff --git a/django-backend/fecfiler/web_services/summary/test_summary.py b/django-backend/fecfiler/web_services/summary/test_summary.py index f43a034c38..946bf7bf1c 100644 --- a/django-backend/fecfiler/web_services/summary/test_summary.py +++ b/django-backend/fecfiler/web_services/summary/test_summary.py @@ -129,8 +129,8 @@ def test_calculate_summary_column_b(self): self.assertEqual(summary_b["line_6c"], Decimal("18985.17")) self.assertEqual( summary_b["line_6d"], - Decimal("0") # line_6a - + Decimal("18146.17") # line_6c + Decimal("61") # line_6a + + Decimal("18985.17") # line_6c ) self.assertEqual( summary_b["line_7"], @@ -148,7 +148,7 @@ def test_calculate_summary_column_b(self): self.assertEqual( summary_b["line_8"], summary_b["line_6d"] - - summary_a["line_7"] + - summary_b["line_7"] ) self.assertEqual(summary_b["line_11ai"], Decimal("10000.23")) self.assertEqual(summary_b["line_11aii"], Decimal("103.77")) diff --git a/django-backend/fecfiler/web_services/summary/test_tasks.py b/django-backend/fecfiler/web_services/summary/test_tasks.py index 8449185b1b..335b389c11 100644 --- a/django-backend/fecfiler/web_services/summary/test_tasks.py +++ b/django-backend/fecfiler/web_services/summary/test_tasks.py @@ -60,7 +60,7 @@ def test_report_group_recalculation(self): ) self.assertEqual( calculated_report.form_3x.L6a_cash_on_hand_jan_1_ytd, - Decimal("18146.17") + Decimal("61.00") ) self.assertEqual( calculated_prev_report.form_3x.L15_offsets_to_operating_expenditures_refunds_period, # noqa: E501 From f969def206faa858e5da327c0510c631d75074b0 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Tue, 5 Dec 2023 15:33:12 -0500 Subject: [PATCH 066/109] Small cleanup for consistency --- .../fecfiler/web_services/summary/summary.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/django-backend/fecfiler/web_services/summary/summary.py b/django-backend/fecfiler/web_services/summary/summary.py index b10fe76bcd..6be72b9ec6 100644 --- a/django-backend/fecfiler/web_services/summary/summary.py +++ b/django-backend/fecfiler/web_services/summary/summary.py @@ -223,15 +223,16 @@ def calculate_summary_column_b(self): if len(reports_from_prior_years): summary["line_6a"] = reports_from_prior_years.last().form_3x.L8_cash_on_hand_close_ytd # noqa: E501 else: - earliest_l6a_source_in_year = Report.objects.filter( + l6a_sources_in_year = Report.objects.filter( + committee_account=self.report.committee_account, coverage_through_date__year=self.report.coverage_from_date.year, form_3x__L6a_cash_on_hand_jan_1_ytd__gt=0 - ).order_by("coverage_from_date").first() + ).order_by("coverage_from_date") - if earliest_l6a_source_in_year: - summary["line_6a"] = earliest_l6a_source_in_year.form_3x.L6a_cash_on_hand_jan_1_ytd # noqa: E501 + if len(l6a_sources_in_year) > 0: + summary["line_6a"] = l6a_sources_in_year.first().form_3x.L6a_cash_on_hand_jan_1_ytd # noqa: E501 else: - summary["line_6a"] = 0 + summary["line_6a"] = Decimal("0.00") summary["line_11aiii"] = ( summary["line_11ai"] From 801376a3eaa1c79a04a001313a87224cb854ec50 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Tue, 5 Dec 2023 15:33:59 -0500 Subject: [PATCH 067/109] Fixes small typo --- django-backend/fecfiler/web_services/summary/summary.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django-backend/fecfiler/web_services/summary/summary.py b/django-backend/fecfiler/web_services/summary/summary.py index 6be72b9ec6..cb1af5c6d6 100644 --- a/django-backend/fecfiler/web_services/summary/summary.py +++ b/django-backend/fecfiler/web_services/summary/summary.py @@ -220,7 +220,7 @@ def calculate_summary_column_b(self): form_3x__isnull=False ).order_by("coverage_from_date") - if len(reports_from_prior_years): + if len(reports_from_prior_years) > 0: summary["line_6a"] = reports_from_prior_years.last().form_3x.L8_cash_on_hand_close_ytd # noqa: E501 else: l6a_sources_in_year = Report.objects.filter( From 8c2475d8d9485691518c75858780bac292a972de Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Tue, 5 Dec 2023 15:46:06 -0500 Subject: [PATCH 068/109] Fixes linting errors --- django-backend/fecfiler/web_services/summary/summary.py | 2 +- django-backend/fecfiler/web_services/summary/test_summary.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/django-backend/fecfiler/web_services/summary/summary.py b/django-backend/fecfiler/web_services/summary/summary.py index cb1af5c6d6..b1ec659341 100644 --- a/django-backend/fecfiler/web_services/summary/summary.py +++ b/django-backend/fecfiler/web_services/summary/summary.py @@ -65,7 +65,7 @@ def calculate_summary_column_a(self, summary_b): else: summary["line_6b"] = summary_b["line_6a"] - if summary["line_6b"] == None: + if summary["line_6b"] is None: summary["line_6b"] = Decimal("0.00") summary["line_9"] = ( diff --git a/django-backend/fecfiler/web_services/summary/test_summary.py b/django-backend/fecfiler/web_services/summary/test_summary.py index 946bf7bf1c..d89383cd0b 100644 --- a/django-backend/fecfiler/web_services/summary/test_summary.py +++ b/django-backend/fecfiler/web_services/summary/test_summary.py @@ -120,7 +120,6 @@ def test_calculate_summary_column_b(self): summary_service = SummaryService(f3x) summary = summary_service.calculate_summary() - summary_a = summary["a"] summary_b = summary["b"] t = Transaction.objects.get(id="aaaaaaaa-4d75-46f0-bce2-111000000001") From 7ac8b6c1b6eb6e75e7f965df94b17872fd39eb63 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Tue, 5 Dec 2023 16:08:45 -0500 Subject: [PATCH 069/109] More linting fixes --- django-backend/fecfiler/web_services/summary/summary.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/django-backend/fecfiler/web_services/summary/summary.py b/django-backend/fecfiler/web_services/summary/summary.py index b1ec659341..781cba5f02 100644 --- a/django-backend/fecfiler/web_services/summary/summary.py +++ b/django-backend/fecfiler/web_services/summary/summary.py @@ -1,7 +1,7 @@ from decimal import Decimal from fecfiler.transactions.models import Transaction from fecfiler.reports.models import Report -from django.db.models import Q, Sum, Case, When +from django.db.models import Q, Sum from django.db.models.functions import Coalesce import logging @@ -220,7 +220,7 @@ def calculate_summary_column_b(self): form_3x__isnull=False ).order_by("coverage_from_date") - if len(reports_from_prior_years) > 0: + if reports_from_prior_years.count() > 0: summary["line_6a"] = reports_from_prior_years.last().form_3x.L8_cash_on_hand_close_ytd # noqa: E501 else: l6a_sources_in_year = Report.objects.filter( @@ -229,7 +229,7 @@ def calculate_summary_column_b(self): form_3x__L6a_cash_on_hand_jan_1_ytd__gt=0 ).order_by("coverage_from_date") - if len(l6a_sources_in_year) > 0: + if l6a_sources_in_year.count() > 0: summary["line_6a"] = l6a_sources_in_year.first().form_3x.L6a_cash_on_hand_jan_1_ytd # noqa: E501 else: summary["line_6a"] = Decimal("0.00") From 0ade81d453828f69de857d42e6d0952a107a22bd Mon Sep 17 00:00:00 2001 From: toddlees Date: Thu, 7 Dec 2023 09:29:25 -0500 Subject: [PATCH 070/109] patch --- .../fecfiler/web_services/summary/summary.py | 72 ++++++++----------- .../fecfiler/web_services/summary/tasks.py | 4 +- 2 files changed, 32 insertions(+), 44 deletions(-) diff --git a/django-backend/fecfiler/web_services/summary/summary.py b/django-backend/fecfiler/web_services/summary/summary.py index 781cba5f02..16ea01469d 100644 --- a/django-backend/fecfiler/web_services/summary/summary.py +++ b/django-backend/fecfiler/web_services/summary/summary.py @@ -11,25 +11,43 @@ class SummaryService: def __init__(self, report) -> None: self.report = report - self.previous_report = Report.objects.filter( - ~Q(id=report.id), - form_3x__isnull=False, - coverage_from_date__year=report.coverage_from_date.year, - coverage_through_date__lt=report.coverage_from_date - ).order_by("-coverage_through_date").first() + self.previous_report = ( + Report.objects.filter( + ~Q(id=report.id), + committee_account=report.committee_account, + form_3x__isnull=False, + coverage_through_date__lt=report.coverage_from_date, + ) + .order_by("-coverage_through_date") + .first() + ) def calculate_summary(self): + summary_a = self.calculate_summary_column_a() summary_b = self.calculate_summary_column_b() - summary_a = self.calculate_summary_column_a(summary_b) - summary = { - "a": summary_a, - "b": summary_b, - } - return summary - def calculate_summary_column_a(self, summary_b): + reports_from_prior_years = Report.objects.filter( + committee_account=self.report.committee_account, + coverage_through_date__year__lt=self.report.coverage_from_date.year, + form_3x__isnull=False + ).order_by("coverage_from_date") + + if reports_from_prior_years.count() > 0: + summary_b["line_6a"] = reports_from_prior_years.last().form_3x.L8_cash_on_hand_close_ytd # noqa: E501 + elif self.previous_report: + summary_b["line_6a"] = self.previous_report.form_3x.L6a_cash_on_hand_jan_1_ytd # noqa: E501 + + if self.previous_report: + summary_a["line_6b"] = self.previous_report.form_3x.L8_cash_on_hand_at_close_period # noqa: E501 + else: + summary_a["line_6b"] = summary_b["line_6a"] + + + return summary_a, summary_b + + def calculate_summary_column_a(self): report_transactions = Transaction.objects.filter(report_id=self.report.id) summary = report_transactions.aggregate( line_11ai=self.get_line("SA11AI"), @@ -60,14 +78,6 @@ def calculate_summary_column_a(self, summary_b): temp_sd10=self.get_line("SD10", field="balance_at_close") ) - if self.previous_report and self.previous_report.form_3x: - summary["line_6b"] = self.previous_report.form_3x.L8_cash_on_hand_at_close_period # noqa: E501 - else: - summary["line_6b"] = summary_b["line_6a"] - - if summary["line_6b"] is None: - summary["line_6b"] = Decimal("0.00") - summary["line_9"] = ( summary["temp_sc9"] + summary["temp_sd9"] @@ -214,26 +224,6 @@ def calculate_summary_column_b(self): line_30b=self.get_line("SB30B"), ) - reports_from_prior_years = Report.objects.filter( - committee_account=self.report.committee_account, - coverage_through_date__year__lt=self.report.coverage_from_date.year, - form_3x__isnull=False - ).order_by("coverage_from_date") - - if reports_from_prior_years.count() > 0: - summary["line_6a"] = reports_from_prior_years.last().form_3x.L8_cash_on_hand_close_ytd # noqa: E501 - else: - l6a_sources_in_year = Report.objects.filter( - committee_account=self.report.committee_account, - coverage_through_date__year=self.report.coverage_from_date.year, - form_3x__L6a_cash_on_hand_jan_1_ytd__gt=0 - ).order_by("coverage_from_date") - - if l6a_sources_in_year.count() > 0: - summary["line_6a"] = l6a_sources_in_year.first().form_3x.L6a_cash_on_hand_jan_1_ytd # noqa: E501 - else: - summary["line_6a"] = Decimal("0.00") - summary["line_11aiii"] = ( summary["line_11ai"] + summary["line_11aii"] diff --git a/django-backend/fecfiler/web_services/summary/tasks.py b/django-backend/fecfiler/web_services/summary/tasks.py index 9491b4021d..11347fa2d1 100644 --- a/django-backend/fecfiler/web_services/summary/tasks.py +++ b/django-backend/fecfiler/web_services/summary/tasks.py @@ -51,9 +51,7 @@ def calculate_summary(report_id): for report in claimed_reports: summary_service = SummaryService(report) - summary = summary_service.calculate_summary() - a = summary["a"] - b = summary["b"] + a, b = summary_service.calculate_summary() # line 6a report.form_3x.L6a_cash_on_hand_jan_1_ytd = b["line_6a"] From 99f4cc35eca0fc2fa27f786281ac4128cb220c37 Mon Sep 17 00:00:00 2001 From: toddlees Date: Fri, 8 Dec 2023 08:22:26 -0500 Subject: [PATCH 071/109] handle first report --- .../fecfiler/web_services/summary/summary.py | 428 +++++++++--------- 1 file changed, 217 insertions(+), 211 deletions(-) diff --git a/django-backend/fecfiler/web_services/summary/summary.py b/django-backend/fecfiler/web_services/summary/summary.py index 16ea01469d..6efee96b47 100644 --- a/django-backend/fecfiler/web_services/summary/summary.py +++ b/django-backend/fecfiler/web_services/summary/summary.py @@ -23,8 +23,8 @@ def __init__(self, report) -> None: ) def calculate_summary(self): - summary_a = self.calculate_summary_column_a() - summary_b = self.calculate_summary_column_b() + column_a = self.calculate_summary_column_a() + column_b = self.calculate_summary_column_b() @@ -35,21 +35,43 @@ def calculate_summary(self): ).order_by("coverage_from_date") if reports_from_prior_years.count() > 0: - summary_b["line_6a"] = reports_from_prior_years.last().form_3x.L8_cash_on_hand_close_ytd # noqa: E501 + column_b["line_6a"] = reports_from_prior_years.last().form_3x.L8_cash_on_hand_close_ytd # noqa: E501 elif self.previous_report: - summary_b["line_6a"] = self.previous_report.form_3x.L6a_cash_on_hand_jan_1_ytd # noqa: E501 + column_b["line_6a"] = self.previous_report.form_3x.L6a_cash_on_hand_jan_1_ytd # noqa: E501 + else: + #first report + column_b["line_6a"] = self.report.form_3x.L6a_cash_on_hand_jan_1_ytd if self.previous_report: - summary_a["line_6b"] = self.previous_report.form_3x.L8_cash_on_hand_at_close_period # noqa: E501 + column_a["line_6b"] = self.previous_report.form_3x.L8_cash_on_hand_at_close_period # noqa: E501 else: - summary_a["line_6b"] = summary_b["line_6a"] + # first report + column_a["line_6b"] = self.report.form_3x.L6a_cash_on_hand_jan_1_ytd + + column_a["line_6d"] = ( + column_a["line_6b"] + + column_a["line_6c"] + ) + column_a["line_8"] = ( + column_a["line_6d"] + - column_a["line_7"] + ) - return summary_a, summary_b + column_b["line_6d"] = ( + column_b["line_6a"] + + column_b["line_6c"] + ) + column_b["line_8"] = ( + column_b["line_6d"] + - column_b["line_7"] + ) + return column_a, column_b + def calculate_summary_column_a(self): report_transactions = Transaction.objects.filter(report_id=self.report.id) - summary = report_transactions.aggregate( + column_a = report_transactions.aggregate( line_11ai=self.get_line("SA11AI"), line_11aii=self.get_line("SA11AII"), line_11b=self.get_line("SA11B"), @@ -78,117 +100,109 @@ def calculate_summary_column_a(self): temp_sd10=self.get_line("SD10", field="balance_at_close") ) - summary["line_9"] = ( - summary["temp_sc9"] - + summary["temp_sd9"] - ) - summary["line_10"] = ( - summary["temp_sc10"] - + summary["temp_sd10"] - ) - summary["line_11aiii"] = ( - summary["line_11ai"] - + summary["line_11aii"] - ) - summary["line_11d"] = ( - summary["line_11aiii"] - + summary["line_11b"] - + summary["line_11c"] - ) - summary["line_18c"] = Decimal(0) # Stubbed out until a future ticket - summary["line_21ai"] = Decimal(0) # Stubbed out until a future ticket - summary["line_21aii"] = Decimal(0) # Stubbed out until a future ticket - summary["line_21c"] = ( - summary["line_21ai"] - + summary["line_21aii"] - + summary["line_21b"] - ) - summary["line_25"] = Decimal(0) # Stubbed out until a future ticket - summary["line_28d"] = ( - summary["line_28a"] - + summary["line_28b"] - + summary["line_28c"] - ) - summary["line_30ai"] = Decimal(0) # Stubbed out until a future ticket - summary["line_30aii"] = Decimal(0) # Stubbed out until a future ticket - summary["line_30c"] = ( - summary["line_30ai"] - + summary["line_30aii"] - + summary["line_30b"] - ) - summary["line_31"] = ( - summary["line_21c"] - + summary["line_22"] - + summary["line_23"] - + summary["line_24"] - + summary["line_25"] - + summary["line_26"] - + summary["line_27"] - + summary["line_28d"] - + summary["line_29"] - + summary["line_30c"] - ) - summary["line_32"] = ( - summary["line_31"] - - summary["line_21aii"] - - summary["line_30aii"] - ) - summary["line_33"] = ( - summary["line_11d"] - ) - summary["line_34"] = ( - summary["line_28d"] - ) - summary["line_35"] = ( - summary["line_33"] - - summary["line_34"] - ) - summary["line_36"] = ( - summary["line_21ai"] - + summary["line_21b"] - ) - summary["line_37"] = ( - summary["line_15"] - ) - summary["line_38"] = ( - summary["line_36"] - - summary["line_37"] - ) - summary["line_6c"] = ( - summary["line_11d"] - + summary["line_12"] - + summary["line_13"] - + summary["line_14"] - + summary["line_15"] - + summary["line_16"] - + summary["line_17"] - + summary["line_18c"] - ) - summary["line_6d"] = ( - summary["line_6b"] - + summary["line_6c"] - ) - summary["line_7"] = ( - summary["line_31"] - ) - summary["line_8"] = ( - summary["line_6d"] - - summary["line_7"] - ) - summary["line_19"] = ( - summary["line_6c"] - ) - summary["line_20"] = ( - summary["line_19"] - - summary["line_18c"] + column_a["line_9"] = ( + column_a["temp_sc9"] + + column_a["temp_sd9"] + ) + column_a["line_10"] = ( + column_a["temp_sc10"] + + column_a["temp_sd10"] + ) + column_a["line_11aiii"] = ( + column_a["line_11ai"] + + column_a["line_11aii"] + ) + column_a["line_11d"] = ( + column_a["line_11aiii"] + + column_a["line_11b"] + + column_a["line_11c"] + ) + column_a["line_18c"] = Decimal(0) # Stubbed out until a future ticket + column_a["line_21ai"] = Decimal(0) # Stubbed out until a future ticket + column_a["line_21aii"] = Decimal(0) # Stubbed out until a future ticket + column_a["line_21c"] = ( + column_a["line_21ai"] + + column_a["line_21aii"] + + column_a["line_21b"] + ) + column_a["line_25"] = Decimal(0) # Stubbed out until a future ticket + column_a["line_28d"] = ( + column_a["line_28a"] + + column_a["line_28b"] + + column_a["line_28c"] + ) + column_a["line_30ai"] = Decimal(0) # Stubbed out until a future ticket + column_a["line_30aii"] = Decimal(0) # Stubbed out until a future ticket + column_a["line_30c"] = ( + column_a["line_30ai"] + + column_a["line_30aii"] + + column_a["line_30b"] + ) + column_a["line_31"] = ( + column_a["line_21c"] + + column_a["line_22"] + + column_a["line_23"] + + column_a["line_24"] + + column_a["line_25"] + + column_a["line_26"] + + column_a["line_27"] + + column_a["line_28d"] + + column_a["line_29"] + + column_a["line_30c"] + ) + column_a["line_32"] = ( + column_a["line_31"] + - column_a["line_21aii"] + - column_a["line_30aii"] + ) + column_a["line_33"] = ( + column_a["line_11d"] + ) + column_a["line_34"] = ( + column_a["line_28d"] + ) + column_a["line_35"] = ( + column_a["line_33"] + - column_a["line_34"] + ) + column_a["line_36"] = ( + column_a["line_21ai"] + + column_a["line_21b"] + ) + column_a["line_37"] = ( + column_a["line_15"] + ) + column_a["line_38"] = ( + column_a["line_36"] + - column_a["line_37"] + ) + column_a["line_6c"] = ( + column_a["line_11d"] + + column_a["line_12"] + + column_a["line_13"] + + column_a["line_14"] + + column_a["line_15"] + + column_a["line_16"] + + column_a["line_17"] + + column_a["line_18c"] + ) + column_a["line_7"] = ( + column_a["line_31"] + ) + column_a["line_19"] = ( + column_a["line_6c"] + ) + column_a["line_20"] = ( + column_a["line_19"] + - column_a["line_18c"] ) # Remove temporary aggregations to clean up the summary - for key in list(summary.keys()): + for key in list(column_a.keys()): if key.startswith("temp_"): - summary.pop(key) + column_a.pop(key) - return summary + return column_a def calculate_summary_column_b(self): committee = self.report.committee_account @@ -200,7 +214,7 @@ def calculate_summary_column_b(self): ) # build summary - summary = ytd_transactions.aggregate( + column_b = ytd_transactions.aggregate( line_11ai=self.get_line("SA11AI"), line_11aii=self.get_line("SA11AII"), line_11b=self.get_line("SA11B"), @@ -224,104 +238,96 @@ def calculate_summary_column_b(self): line_30b=self.get_line("SB30B"), ) - summary["line_11aiii"] = ( - summary["line_11ai"] - + summary["line_11aii"] - ) - summary["line_11d"] = ( - summary["line_11aiii"] - + summary["line_11b"] - + summary["line_11c"] - ) - summary["line_18c"] = Decimal(0) # Stubbed out until a future ticket - summary["line_21ai"] = Decimal(0) # Stubbed out until a future ticket - summary["line_21aii"] = Decimal(0) # Stubbed out until a future ticket - summary["line_21c"] = ( - summary["line_21ai"] - + summary["line_21aii"] - + summary["line_21b"] - ) - summary["line_25"] = Decimal(0) # Stubbed out until a future ticket - summary["line_28d"] = ( - summary["line_28a"] - + summary["line_28b"] - + summary["line_28c"] - ) - summary["line_30ai"] = Decimal(0) # Stubbed out until a future ticket - summary["line_30aii"] = Decimal(0) # Stubbed out until a future ticket - summary["line_30c"] = ( - summary["line_30ai"] - + summary["line_30aii"] - + summary["line_30b"] - ) - summary["line_31"] = ( - summary["line_21c"] - + summary["line_22"] - + summary["line_23"] - + summary["line_24"] - + summary["line_25"] - + summary["line_26"] - + summary["line_27"] - + summary["line_28d"] - + summary["line_29"] - + summary["line_30c"] - ) - summary["line_32"] = ( - summary["line_31"] - - summary["line_21aii"] - - summary["line_30aii"] - ) - summary["line_33"] = ( - summary["line_11d"] - ) - summary["line_34"] = ( - summary["line_28d"] - ) - summary["line_35"] = ( - summary["line_33"] - - summary["line_34"] - ) - summary["line_36"] = ( - summary["line_21ai"] - + summary["line_21b"] - ) - summary["line_37"] = ( - summary["line_15"] - ) - summary["line_38"] = ( - summary["line_36"] - - summary["line_37"] - ) - summary["line_6c"] = ( - summary["line_11d"] - + summary["line_12"] - + summary["line_13"] - + summary["line_14"] - + summary["line_15"] - + summary["line_16"] - + summary["line_17"] - + summary["line_18c"] - ) - summary["line_6d"] = ( - summary["line_6a"] - + summary["line_6c"] - ) - summary["line_7"] = ( - summary["line_31"] - ) - summary["line_8"] = ( - summary["line_6d"] - - summary["line_7"] - ) - summary["line_19"] = ( - summary["line_6c"] - ) - summary["line_20"] = ( - summary["line_19"] - - summary["line_18c"] + column_b["line_11aiii"] = ( + column_b["line_11ai"] + + column_b["line_11aii"] + ) + column_b["line_11d"] = ( + column_b["line_11aiii"] + + column_b["line_11b"] + + column_b["line_11c"] + ) + column_b["line_18c"] = Decimal(0) # Stubbed out until a future ticket + column_b["line_21ai"] = Decimal(0) # Stubbed out until a future ticket + column_b["line_21aii"] = Decimal(0) # Stubbed out until a future ticket + column_b["line_21c"] = ( + column_b["line_21ai"] + + column_b["line_21aii"] + + column_b["line_21b"] + ) + column_b["line_25"] = Decimal(0) # Stubbed out until a future ticket + column_b["line_28d"] = ( + column_b["line_28a"] + + column_b["line_28b"] + + column_b["line_28c"] + ) + column_b["line_30ai"] = Decimal(0) # Stubbed out until a future ticket + column_b["line_30aii"] = Decimal(0) # Stubbed out until a future ticket + column_b["line_30c"] = ( + column_b["line_30ai"] + + column_b["line_30aii"] + + column_b["line_30b"] + ) + column_b["line_31"] = ( + column_b["line_21c"] + + column_b["line_22"] + + column_b["line_23"] + + column_b["line_24"] + + column_b["line_25"] + + column_b["line_26"] + + column_b["line_27"] + + column_b["line_28d"] + + column_b["line_29"] + + column_b["line_30c"] + ) + column_b["line_32"] = ( + column_b["line_31"] + - column_b["line_21aii"] + - column_b["line_30aii"] + ) + column_b["line_33"] = ( + column_b["line_11d"] + ) + column_b["line_34"] = ( + column_b["line_28d"] + ) + column_b["line_35"] = ( + column_b["line_33"] + - column_b["line_34"] + ) + column_b["line_36"] = ( + column_b["line_21ai"] + + column_b["line_21b"] + ) + column_b["line_37"] = ( + column_b["line_15"] + ) + column_b["line_38"] = ( + column_b["line_36"] + - column_b["line_37"] + ) + column_b["line_6c"] = ( + column_b["line_11d"] + + column_b["line_12"] + + column_b["line_13"] + + column_b["line_14"] + + column_b["line_15"] + + column_b["line_16"] + + column_b["line_17"] + + column_b["line_18c"] + ) + column_b["line_7"] = ( + column_b["line_31"] + ) + column_b["line_19"] = ( + column_b["line_6c"] + ) + column_b["line_20"] = ( + column_b["line_19"] + - column_b["line_18c"] ) - return summary + return column_b def get_line(self, form_type, field="amount"): query = Q(~Q(memo_code=True), form_type=form_type) From d2cfc0b39570678243a1f2d7d2f8c18ac548edf6 Mon Sep 17 00:00:00 2001 From: toddlees Date: Fri, 8 Dec 2023 08:48:26 -0500 Subject: [PATCH 072/109] handle unentered cash on hand in previous report --- .../fecfiler/web_services/summary/summary.py | 219 ++++++------------ .../fecfiler/web_services/summary/tasks.py | 138 +++++++---- .../web_services/summary/test_summary.py | 14 +- 3 files changed, 169 insertions(+), 202 deletions(-) diff --git a/django-backend/fecfiler/web_services/summary/summary.py b/django-backend/fecfiler/web_services/summary/summary.py index 6efee96b47..11169af472 100644 --- a/django-backend/fecfiler/web_services/summary/summary.py +++ b/django-backend/fecfiler/web_services/summary/summary.py @@ -26,48 +26,9 @@ def calculate_summary(self): column_a = self.calculate_summary_column_a() column_b = self.calculate_summary_column_b() + column_a, column_b = self.calculate_cash_on_hand_fields(column_a, column_b) - - reports_from_prior_years = Report.objects.filter( - committee_account=self.report.committee_account, - coverage_through_date__year__lt=self.report.coverage_from_date.year, - form_3x__isnull=False - ).order_by("coverage_from_date") - - if reports_from_prior_years.count() > 0: - column_b["line_6a"] = reports_from_prior_years.last().form_3x.L8_cash_on_hand_close_ytd # noqa: E501 - elif self.previous_report: - column_b["line_6a"] = self.previous_report.form_3x.L6a_cash_on_hand_jan_1_ytd # noqa: E501 - else: - #first report - column_b["line_6a"] = self.report.form_3x.L6a_cash_on_hand_jan_1_ytd - - if self.previous_report: - column_a["line_6b"] = self.previous_report.form_3x.L8_cash_on_hand_at_close_period # noqa: E501 - else: - # first report - column_a["line_6b"] = self.report.form_3x.L6a_cash_on_hand_jan_1_ytd - - column_a["line_6d"] = ( - column_a["line_6b"] - + column_a["line_6c"] - ) - - column_a["line_8"] = ( - column_a["line_6d"] - - column_a["line_7"] - ) - - column_b["line_6d"] = ( - column_b["line_6a"] - + column_b["line_6c"] - ) - column_b["line_8"] = ( - column_b["line_6d"] - - column_b["line_7"] - ) return column_a, column_b - def calculate_summary_column_a(self): report_transactions = Transaction.objects.filter(report_id=self.report.id) @@ -97,46 +58,29 @@ def calculate_summary_column_a(self): temp_sc9=self.get_line("SC/9", field="loan_balance"), temp_sd9=self.get_line("SD9", field="balance_at_close"), temp_sc10=self.get_line("SC/10", field="loan_balance"), - temp_sd10=self.get_line("SD10", field="balance_at_close") + temp_sd10=self.get_line("SD10", field="balance_at_close"), ) - column_a["line_9"] = ( - column_a["temp_sc9"] - + column_a["temp_sd9"] - ) - column_a["line_10"] = ( - column_a["temp_sc10"] - + column_a["temp_sd10"] - ) - column_a["line_11aiii"] = ( - column_a["line_11ai"] - + column_a["line_11aii"] - ) + column_a["line_9"] = column_a["temp_sc9"] + column_a["temp_sd9"] + column_a["line_10"] = column_a["temp_sc10"] + column_a["temp_sd10"] + column_a["line_11aiii"] = column_a["line_11ai"] + column_a["line_11aii"] column_a["line_11d"] = ( - column_a["line_11aiii"] - + column_a["line_11b"] - + column_a["line_11c"] + column_a["line_11aiii"] + column_a["line_11b"] + column_a["line_11c"] ) column_a["line_18c"] = Decimal(0) # Stubbed out until a future ticket column_a["line_21ai"] = Decimal(0) # Stubbed out until a future ticket column_a["line_21aii"] = Decimal(0) # Stubbed out until a future ticket column_a["line_21c"] = ( - column_a["line_21ai"] - + column_a["line_21aii"] - + column_a["line_21b"] + column_a["line_21ai"] + column_a["line_21aii"] + column_a["line_21b"] ) column_a["line_25"] = Decimal(0) # Stubbed out until a future ticket column_a["line_28d"] = ( - column_a["line_28a"] - + column_a["line_28b"] - + column_a["line_28c"] + column_a["line_28a"] + column_a["line_28b"] + column_a["line_28c"] ) column_a["line_30ai"] = Decimal(0) # Stubbed out until a future ticket column_a["line_30aii"] = Decimal(0) # Stubbed out until a future ticket column_a["line_30c"] = ( - column_a["line_30ai"] - + column_a["line_30aii"] - + column_a["line_30b"] + column_a["line_30ai"] + column_a["line_30aii"] + column_a["line_30b"] ) column_a["line_31"] = ( column_a["line_21c"] @@ -151,31 +95,14 @@ def calculate_summary_column_a(self): + column_a["line_30c"] ) column_a["line_32"] = ( - column_a["line_31"] - - column_a["line_21aii"] - - column_a["line_30aii"] - ) - column_a["line_33"] = ( - column_a["line_11d"] - ) - column_a["line_34"] = ( - column_a["line_28d"] - ) - column_a["line_35"] = ( - column_a["line_33"] - - column_a["line_34"] - ) - column_a["line_36"] = ( - column_a["line_21ai"] - + column_a["line_21b"] - ) - column_a["line_37"] = ( - column_a["line_15"] - ) - column_a["line_38"] = ( - column_a["line_36"] - - column_a["line_37"] - ) + column_a["line_31"] - column_a["line_21aii"] - column_a["line_30aii"] + ) + column_a["line_33"] = column_a["line_11d"] + column_a["line_34"] = column_a["line_28d"] + column_a["line_35"] = column_a["line_33"] - column_a["line_34"] + column_a["line_36"] = column_a["line_21ai"] + column_a["line_21b"] + column_a["line_37"] = column_a["line_15"] + column_a["line_38"] = column_a["line_36"] - column_a["line_37"] column_a["line_6c"] = ( column_a["line_11d"] + column_a["line_12"] @@ -186,16 +113,9 @@ def calculate_summary_column_a(self): + column_a["line_17"] + column_a["line_18c"] ) - column_a["line_7"] = ( - column_a["line_31"] - ) - column_a["line_19"] = ( - column_a["line_6c"] - ) - column_a["line_20"] = ( - column_a["line_19"] - - column_a["line_18c"] - ) + column_a["line_7"] = column_a["line_31"] + column_a["line_19"] = column_a["line_6c"] + column_a["line_20"] = column_a["line_19"] - column_a["line_18c"] # Remove temporary aggregations to clean up the summary for key in list(column_a.keys()): @@ -210,7 +130,9 @@ def calculate_summary_column_b(self): report_year = report_date.year ytd_transactions = Transaction.objects.filter( - committee_account=committee, date__year=report_year, date__lte=report_date, + committee_account=committee, + date__year=report_year, + date__lte=report_date, ) # build summary @@ -238,35 +160,24 @@ def calculate_summary_column_b(self): line_30b=self.get_line("SB30B"), ) - column_b["line_11aiii"] = ( - column_b["line_11ai"] - + column_b["line_11aii"] - ) + column_b["line_11aiii"] = column_b["line_11ai"] + column_b["line_11aii"] column_b["line_11d"] = ( - column_b["line_11aiii"] - + column_b["line_11b"] - + column_b["line_11c"] + column_b["line_11aiii"] + column_b["line_11b"] + column_b["line_11c"] ) column_b["line_18c"] = Decimal(0) # Stubbed out until a future ticket column_b["line_21ai"] = Decimal(0) # Stubbed out until a future ticket column_b["line_21aii"] = Decimal(0) # Stubbed out until a future ticket column_b["line_21c"] = ( - column_b["line_21ai"] - + column_b["line_21aii"] - + column_b["line_21b"] + column_b["line_21ai"] + column_b["line_21aii"] + column_b["line_21b"] ) column_b["line_25"] = Decimal(0) # Stubbed out until a future ticket column_b["line_28d"] = ( - column_b["line_28a"] - + column_b["line_28b"] - + column_b["line_28c"] + column_b["line_28a"] + column_b["line_28b"] + column_b["line_28c"] ) column_b["line_30ai"] = Decimal(0) # Stubbed out until a future ticket column_b["line_30aii"] = Decimal(0) # Stubbed out until a future ticket column_b["line_30c"] = ( - column_b["line_30ai"] - + column_b["line_30aii"] - + column_b["line_30b"] + column_b["line_30ai"] + column_b["line_30aii"] + column_b["line_30b"] ) column_b["line_31"] = ( column_b["line_21c"] @@ -281,31 +192,14 @@ def calculate_summary_column_b(self): + column_b["line_30c"] ) column_b["line_32"] = ( - column_b["line_31"] - - column_b["line_21aii"] - - column_b["line_30aii"] - ) - column_b["line_33"] = ( - column_b["line_11d"] - ) - column_b["line_34"] = ( - column_b["line_28d"] - ) - column_b["line_35"] = ( - column_b["line_33"] - - column_b["line_34"] - ) - column_b["line_36"] = ( - column_b["line_21ai"] - + column_b["line_21b"] - ) - column_b["line_37"] = ( - column_b["line_15"] - ) - column_b["line_38"] = ( - column_b["line_36"] - - column_b["line_37"] - ) + column_b["line_31"] - column_b["line_21aii"] - column_b["line_30aii"] + ) + column_b["line_33"] = column_b["line_11d"] + column_b["line_34"] = column_b["line_28d"] + column_b["line_35"] = column_b["line_33"] - column_b["line_34"] + column_b["line_36"] = column_b["line_21ai"] + column_b["line_21b"] + column_b["line_37"] = column_b["line_15"] + column_b["line_38"] = column_b["line_36"] - column_b["line_37"] column_b["line_6c"] = ( column_b["line_11d"] + column_b["line_12"] @@ -316,19 +210,40 @@ def calculate_summary_column_b(self): + column_b["line_17"] + column_b["line_18c"] ) - column_b["line_7"] = ( - column_b["line_31"] - ) - column_b["line_19"] = ( - column_b["line_6c"] - ) - column_b["line_20"] = ( - column_b["line_19"] - - column_b["line_18c"] - ) + column_b["line_7"] = column_b["line_31"] + column_b["line_19"] = column_b["line_6c"] + column_b["line_20"] = column_b["line_19"] - column_b["line_18c"] return column_b + def calculate_cash_on_hand_fields(self, column_a, column_b): + reports_from_prior_years = Report.objects.filter( + committee_account=self.report.committee_account, + coverage_through_date__year__lt=self.report.coverage_from_date.year, + form_3x__isnull=False, + ).order_by("coverage_from_date") + + if reports_from_prior_years.count() > 0: + column_b["line_6a"] = reports_from_prior_years.last().form_3x.L8_cash_on_hand_close_ytd # noqa: E501 + elif self.previous_report: + column_b["line_6a"] = self.previous_report.form_3x.L6a_cash_on_hand_jan_1_ytd # noqa: E501 + else: + # first report + column_b["line_6a"] = self.report.form_3x.L6a_cash_on_hand_jan_1_ytd + + if self.previous_report: + column_a["line_6b"] = self.previous_report.form_3x.L8_cash_on_hand_at_close_period # noqa: E501 + else: + # first report + column_a["line_6b"] = self.report.form_3x.L6a_cash_on_hand_jan_1_ytd + + if column_a["line_6b"] and column_b["line_6a"]: + column_a["line_6d"] = column_a["line_6b"] + column_a["line_6c"] + column_a["line_8"] = column_a["line_6d"] - column_a["line_7"] + column_b["line_6d"] = column_b["line_6a"] + column_b["line_6c"] + column_b["line_8"] = column_b["line_6d"] - column_b["line_7"] + return column_a, column_b + def get_line(self, form_type, field="amount"): query = Q(~Q(memo_code=True), form_type=form_type) return Coalesce(Sum(field, filter=query), Decimal(0.0)) diff --git a/django-backend/fecfiler/web_services/summary/tasks.py b/django-backend/fecfiler/web_services/summary/tasks.py index 11347fa2d1..d4f894a389 100644 --- a/django-backend/fecfiler/web_services/summary/tasks.py +++ b/django-backend/fecfiler/web_services/summary/tasks.py @@ -26,7 +26,7 @@ def get_reports_to_calculate(primary_report): reports_to_recalculate = Report.objects.filter( ~Q(calculation_status=CalculationState.SUCCEEDED), coverage_from_date__year=report_year, - coverage_through_date__lte=primary_report.coverage_through_date + coverage_through_date__lte=primary_report.coverage_through_date, ).order_by("coverage_through_date") if len(reports_to_recalculate) > 0: @@ -45,7 +45,7 @@ def calculate_summary(report_id): calculation_token = uuid.uuid4() reports_to_recalculate.update( calculation_token=calculation_token, - calculation_status=CalculationState.CALCULATING + calculation_status=CalculationState.CALCULATING, ) claimed_reports = list(reports_to_recalculate) @@ -54,21 +54,21 @@ def calculate_summary(report_id): a, b = summary_service.calculate_summary() # line 6a - report.form_3x.L6a_cash_on_hand_jan_1_ytd = b["line_6a"] + report.form_3x.L6a_cash_on_hand_jan_1_ytd = b.get("line_6a", None) # line 6b - report.form_3x.L6b_cash_on_hand_beginning_period = a["line_6b"] + report.form_3x.L6b_cash_on_hand_beginning_period = a.get("line_6b", None) # line 6c report.form_3x.L6c_total_receipts_period = a.get("line_6c", 0) report.form_3x.L6c_total_receipts_ytd = b.get("line_6c", 0) # line 6d - report.form_3x.L6d_subtotal_period = a.get("line_6d", 0) - report.form_3x.L6d_subtotal_ytd = b.get("line_6d", 0) + report.form_3x.L6d_subtotal_period = a.get("line_6d", None) + report.form_3x.L6d_subtotal_ytd = b.get("line_6d", None) # line 7 report.form_3x.L7_total_disbursements_period = a.get("line_7", 0) report.form_3x.L7_total_disbursements_ytd = b.get("line_7", 0) # line 8 - report.form_3x.L8_cash_on_hand_at_close_period = a.get("line_8", 0) - report.form_3x.L8_cash_on_hand_close_ytd = b.get("line_8", 0) + report.form_3x.L8_cash_on_hand_at_close_period = a.get("line_8", None) + report.form_3x.L8_cash_on_hand_close_ytd = b.get("line_8", None) # line 9 report.form_3x.L9_debts_to_period = a.get("line_9", 0) # line 10 @@ -86,14 +86,20 @@ def calculate_summary(report_id): report.form_3x.L11b_political_party_committees_period = a.get("line_11b", 0) report.form_3x.L11b_political_party_committees_ytd = b.get("line_11b", 0) # line 11c - report.form_3x.L11c_other_political_committees_pacs_period = a.get("line_11c", 0) + report.form_3x.L11c_other_political_committees_pacs_period = a.get( + "line_11c", 0 + ) report.form_3x.L11c_other_political_committees_pacs_ytd = b.get("line_11c", 0) # line 11d report.form_3x.L11d_total_contributions_period = a.get("line_11d", 0) report.form_3x.L11d_total_contributions_ytd = b.get("line_11d", 0) # line 12 - report.form_3x.L12_transfers_from_affiliated_other_party_cmtes_period = a.get("line_12", 0) # noqa: E501 - report.form_3x.L12_transfers_from_affiliated_other_party_cmtes_ytd = b.get("line_12", 0) # noqa: E501 + report.form_3x.L12_transfers_from_affiliated_other_party_cmtes_period = a.get( + "line_12", 0 + ) # noqa: E501 + report.form_3x.L12_transfers_from_affiliated_other_party_cmtes_ytd = b.get( + "line_12", 0 + ) # noqa: E501 # line 13 report.form_3x.L13_all_loans_received_period = a.get("line_13", 0) report.form_3x.L13_all_loans_received_ytd = b.get("line_13", 0) @@ -101,8 +107,12 @@ def calculate_summary(report_id): report.form_3x.L14_loan_repayments_received_period = a.get("line_14", 0) report.form_3x.L14_loan_repayments_received_ytd = b.get("line_14", 0) # line 15 - report.form_3x.L15_offsets_to_operating_expenditures_refunds_period = a.get("line_15", 0) # noqa: E501 - report.form_3x.L15_offsets_to_operating_expenditures_refunds_ytd = b.get("line_15", 0) # noqa: E501 + report.form_3x.L15_offsets_to_operating_expenditures_refunds_period = a.get( + "line_15", 0 + ) # noqa: E501 + report.form_3x.L15_offsets_to_operating_expenditures_refunds_ytd = b.get( + "line_15", 0 + ) # noqa: E501 # line 16 report.form_3x.L16_refunds_of_federal_contributions_period = a.get("line_16", 0) report.form_3x.L16_refunds_of_federal_contributions_ytd = b.get("line_16", 0) @@ -110,14 +120,26 @@ def calculate_summary(report_id): report.form_3x.L17_other_federal_receipts_dividends_period = a.get("line_17", 0) report.form_3x.L17_other_federal_receipts_dividends_ytd = b.get("line_17", 0) # line 18a - report.form_3x.L18a_transfers_from_nonfederal_account_h3_period = a.get("line_18a", 0) # noqa: E501 - report.form_3x.L18a_transfers_from_nonfederal_account_h3_ytd = b.get("line_18a", 0) # noqa: E501 + report.form_3x.L18a_transfers_from_nonfederal_account_h3_period = a.get( + "line_18a", 0 + ) # noqa: E501 + report.form_3x.L18a_transfers_from_nonfederal_account_h3_ytd = b.get( + "line_18a", 0 + ) # noqa: E501 # line 18b - report.form_3x.L18b_transfers_from_nonfederal_levin_h5_period = a.get("line_18b", 0) # noqa: E501 - report.form_3x.L18b_transfers_from_nonfederal_levin_h5_ytd = b.get("line_18b", 0) + report.form_3x.L18b_transfers_from_nonfederal_levin_h5_period = a.get( + "line_18b", 0 + ) # noqa: E501 + report.form_3x.L18b_transfers_from_nonfederal_levin_h5_ytd = b.get( + "line_18b", 0 + ) # line 18c - report.form_3x.L18c_total_nonfederal_transfers_18a_18b_period = a.get("line_18c", 0) # noqa: E501 - report.form_3x.L18c_total_nonfederal_transfers_18a_18b_ytd = b.get("line_18c", 0) + report.form_3x.L18c_total_nonfederal_transfers_18a_18b_period = a.get( + "line_18c", 0 + ) # noqa: E501 + report.form_3x.L18c_total_nonfederal_transfers_18a_18b_ytd = b.get( + "line_18c", 0 + ) # line 19 report.form_3x.L19_total_receipts_period = a.get("line_19", 0) report.form_3x.L19_total_receipts_ytd = b.get("line_19", 0) @@ -131,23 +153,39 @@ def calculate_summary(report_id): report.form_3x.L21aii_nonfederal_share_period = a.get("line_21aii", 0) report.form_3x.L21aii_nonfederal_share_ytd = b.get("line_21aii", 0) # line 21b - report.form_3x.L21b_other_federal_operating_expenditures_period = a.get("line_21b", 0) # noqa: E501 - report.form_3x.L21b_other_federal_operating_expenditures_ytd = b.get("line_21b", 0) # noqa: E501 + report.form_3x.L21b_other_federal_operating_expenditures_period = a.get( + "line_21b", 0 + ) # noqa: E501 + report.form_3x.L21b_other_federal_operating_expenditures_ytd = b.get( + "line_21b", 0 + ) # noqa: E501 # line 21c report.form_3x.L21c_total_operating_expenditures_ytd = b.get("line_21c", 0) report.form_3x.L21c_total_operating_expenditures_period = a.get("line_21c", 0) # line 22 - report.form_3x.L22_transfers_to_affiliated_other_party_cmtes_period = a.get("line_22", 0) # noqa: E501 - report.form_3x.L22_transfers_to_affiliated_other_party_cmtes_ytd = b.get("line_22", 0) # noqa: E501 + report.form_3x.L22_transfers_to_affiliated_other_party_cmtes_period = a.get( + "line_22", 0 + ) # noqa: E501 + report.form_3x.L22_transfers_to_affiliated_other_party_cmtes_ytd = b.get( + "line_22", 0 + ) # noqa: E501 # line 23 - report.form_3x.L23_contributions_to_federal_candidates_cmtes_period = a.get("line_23", 0) # noqa: E501 - report.form_3x.L23_contributions_to_federal_candidates_cmtes_ytd = b.get("line_23", 0) # noqa: E501 + report.form_3x.L23_contributions_to_federal_candidates_cmtes_period = a.get( + "line_23", 0 + ) # noqa: E501 + report.form_3x.L23_contributions_to_federal_candidates_cmtes_ytd = b.get( + "line_23", 0 + ) # noqa: E501 # line 24 report.form_3x.L24_independent_expenditures_period = a.get("line_24", 0) report.form_3x.L24_independent_expenditures_ytd = b.get("line_24", 0) # line 25 - report.form_3x.L25_coordinated_expend_made_by_party_cmtes_period = a.get("line_25", 0) # noqa: E501 - report.form_3x.L25_coordinated_expend_made_by_party_cmtes_ytd = b.get("line_25", 0) # noqa: E501 + report.form_3x.L25_coordinated_expend_made_by_party_cmtes_period = a.get( + "line_25", 0 + ) # noqa: E501 + report.form_3x.L25_coordinated_expend_made_by_party_cmtes_ytd = b.get( + "line_25", 0 + ) # noqa: E501 # line 26 report.form_3x.L26_loan_repayments_period = a.get("line_26", 0) report.form_3x.L26_loan_repayments_made_ytd = b.get("line_26", 0) @@ -170,16 +208,30 @@ def calculate_summary(report_id): report.form_3x.L29_other_disbursements_period = a.get("line_29", 0) report.form_3x.L29_other_disbursements_ytd = b.get("line_29", 0) # line 30ai - report.form_3x.L30ai_shared_federal_activity_h6_fed_share_period = a.get("line_30ai", 0) # noqa: E501 - report.form_3x.L30ai_shared_federal_activity_h6_fed_share_ytd = b.get("line_30ai", 0) # noqa: E501 + report.form_3x.L30ai_shared_federal_activity_h6_fed_share_period = a.get( + "line_30ai", 0 + ) # noqa: E501 + report.form_3x.L30ai_shared_federal_activity_h6_fed_share_ytd = b.get( + "line_30ai", 0 + ) # noqa: E501 # line 30aii - report.form_3x.L30aii_shared_federal_activity_h6_nonfed_period = a.get("line_30aii", 0) # noqa: E501 - report.form_3x.L30aii_shared_federal_activity_h6_nonfed_ytd = b.get("line_30aii", 0) # noqa: E501 + report.form_3x.L30aii_shared_federal_activity_h6_nonfed_period = a.get( + "line_30aii", 0 + ) # noqa: E501 + report.form_3x.L30aii_shared_federal_activity_h6_nonfed_ytd = b.get( + "line_30aii", 0 + ) # noqa: E501 # line 30b - report.form_3x.L30b_nonallocable_fed_election_activity_period = a.get("line_30b", 0) # noqa: E501 - report.form_3x.L30b_nonallocable_fed_election_activity_ytd = b.get("line_30b", 0) + report.form_3x.L30b_nonallocable_fed_election_activity_period = a.get( + "line_30b", 0 + ) # noqa: E501 + report.form_3x.L30b_nonallocable_fed_election_activity_ytd = b.get( + "line_30b", 0 + ) # line 30c - report.form_3x.L30c_total_federal_election_activity_period = a.get("line_30c", 0) + report.form_3x.L30c_total_federal_election_activity_period = a.get( + "line_30c", 0 + ) report.form_3x.L30c_total_federal_election_activity_ytd = b.get("line_30c", 0) # line 31 report.form_3x.L31_total_disbursements_period = a.get("line_31", 0) @@ -197,10 +249,16 @@ def calculate_summary(report_id): report.form_3x.L35_net_contributions_period = a.get("line_35", 0) report.form_3x.L35_net_contributions_ytd = b.get("line_35", 0) # line 36 - report.form_3x.L36_total_federal_operating_expenditures_period = a.get("line_36", 0) # noqa: E501 - report.form_3x.L36_total_federal_operating_expenditures_ytd = b.get("line_36", 0) + report.form_3x.L36_total_federal_operating_expenditures_period = a.get( + "line_36", 0 + ) # noqa: E501 + report.form_3x.L36_total_federal_operating_expenditures_ytd = b.get( + "line_36", 0 + ) # line 37 - report.form_3x.L37_offsets_to_operating_expenditures_period = a.get("line_37", 0) + report.form_3x.L37_offsets_to_operating_expenditures_period = a.get( + "line_37", 0 + ) report.form_3x.L37_offsets_to_operating_expenditures_ytd = b.get("line_37", 0) # line 38 report.form_3x.L38_net_operating_expenditures_period = a.get("line_38", 0) @@ -216,11 +274,9 @@ def calculate_summary(report_id): # reports that have been invalidated updated = bool( Report.objects.filter( - id=report.id, - calculation_token=calculation_token + id=report.id, calculation_token=calculation_token ).update( - calculation_status=CalculationState.SUCCEEDED, - calculation_token=None + calculation_status=CalculationState.SUCCEEDED, calculation_token=None ) ) diff --git a/django-backend/fecfiler/web_services/summary/test_summary.py b/django-backend/fecfiler/web_services/summary/test_summary.py index d89383cd0b..7a715dc705 100644 --- a/django-backend/fecfiler/web_services/summary/test_summary.py +++ b/django-backend/fecfiler/web_services/summary/test_summary.py @@ -20,9 +20,7 @@ class F3XReportTestCase(TestCase): def test_calculate_summary_column_a(self): f3x = Report.objects.get(id="b6d60d2d-d926-4e89-ad4b-c47d152a66ae") summary_service = SummaryService(f3x) - summary = summary_service.calculate_summary() - - summary_a = summary["a"] + summary_a, _ = summary_service.calculate_summary() self.assertEqual(summary_a["line_6c"], Decimal("18085.17")) self.assertEqual( @@ -118,9 +116,7 @@ def test_calculate_summary_column_a(self): def test_calculate_summary_column_b(self): f3x = Report.objects.get(id="b6d60d2d-d926-4e89-ad4b-c47d152a66ae") summary_service = SummaryService(f3x) - summary = summary_service.calculate_summary() - - summary_b = summary["b"] + _, summary_b = summary_service.calculate_summary() t = Transaction.objects.get(id="aaaaaaaa-4d75-46f0-bce2-111000000001") self.assertEqual(t.itemized, False) @@ -213,6 +209,6 @@ def test_calculate_summary_column_b(self): def test_report_with_no_transactions(self): f3x = Report.objects.get(id="a07c8c65-1b2d-4e6e-bcaa-fa8d39e50965") summary_service = SummaryService(f3x) - summary = summary_service.calculate_summary() - self.assertEqual(summary["a"]["line_15"], Decimal("0")) - self.assertEqual(summary["a"]["line_17"], Decimal("0")) + summary_a, _ = summary_service.calculate_summary() + self.assertEqual(summary_a["line_15"], Decimal("0")) + self.assertEqual(summary_a["line_17"], Decimal("0")) From f2d5b4575a034bc791f1ac365fee92d74695e21b Mon Sep 17 00:00:00 2001 From: toddlees Date: Fri, 8 Dec 2023 09:02:12 -0500 Subject: [PATCH 073/109] lint --- django-backend/fecfiler/web_services/summary/summary.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django-backend/fecfiler/web_services/summary/summary.py b/django-backend/fecfiler/web_services/summary/summary.py index 11169af472..8be73e9a74 100644 --- a/django-backend/fecfiler/web_services/summary/summary.py +++ b/django-backend/fecfiler/web_services/summary/summary.py @@ -228,13 +228,13 @@ def calculate_cash_on_hand_fields(self, column_a, column_b): elif self.previous_report: column_b["line_6a"] = self.previous_report.form_3x.L6a_cash_on_hand_jan_1_ytd # noqa: E501 else: - # first report + # first report column_b["line_6a"] = self.report.form_3x.L6a_cash_on_hand_jan_1_ytd if self.previous_report: column_a["line_6b"] = self.previous_report.form_3x.L8_cash_on_hand_at_close_period # noqa: E501 else: - # first report + # first report column_a["line_6b"] = self.report.form_3x.L6a_cash_on_hand_jan_1_ytd if column_a["line_6b"] and column_b["line_6a"]: From 33ba3885673748b223b76c4fca0c528c6bca4e34 Mon Sep 17 00:00:00 2001 From: toddlees Date: Fri, 8 Dec 2023 09:06:51 -0500 Subject: [PATCH 074/109] update comment --- django-backend/fecfiler/web_services/summary/summary.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django-backend/fecfiler/web_services/summary/summary.py b/django-backend/fecfiler/web_services/summary/summary.py index 8be73e9a74..02385ccf36 100644 --- a/django-backend/fecfiler/web_services/summary/summary.py +++ b/django-backend/fecfiler/web_services/summary/summary.py @@ -228,13 +228,13 @@ def calculate_cash_on_hand_fields(self, column_a, column_b): elif self.previous_report: column_b["line_6a"] = self.previous_report.form_3x.L6a_cash_on_hand_jan_1_ytd # noqa: E501 else: - # first report + # user defined cash on hand column_b["line_6a"] = self.report.form_3x.L6a_cash_on_hand_jan_1_ytd if self.previous_report: column_a["line_6b"] = self.previous_report.form_3x.L8_cash_on_hand_at_close_period # noqa: E501 else: - # first report + # user defined cash on hand column_a["line_6b"] = self.report.form_3x.L6a_cash_on_hand_jan_1_ytd if column_a["line_6b"] and column_b["line_6a"]: From 58574106c75e6715a14e45b671f3d893ecf1e95a Mon Sep 17 00:00:00 2001 From: toddlees Date: Fri, 8 Dec 2023 09:08:21 -0500 Subject: [PATCH 075/109] lint --- django-backend/fecfiler/web_services/summary/summary.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/django-backend/fecfiler/web_services/summary/summary.py b/django-backend/fecfiler/web_services/summary/summary.py index 02385ccf36..974f9eb143 100644 --- a/django-backend/fecfiler/web_services/summary/summary.py +++ b/django-backend/fecfiler/web_services/summary/summary.py @@ -224,17 +224,17 @@ def calculate_cash_on_hand_fields(self, column_a, column_b): ).order_by("coverage_from_date") if reports_from_prior_years.count() > 0: - column_b["line_6a"] = reports_from_prior_years.last().form_3x.L8_cash_on_hand_close_ytd # noqa: E501 + column_b["line_6a"] = reports_from_prior_years.last().form_3x.L8_cash_on_hand_close_ytd # noqa: E501 elif self.previous_report: column_b["line_6a"] = self.previous_report.form_3x.L6a_cash_on_hand_jan_1_ytd # noqa: E501 else: - # user defined cash on hand + # user defined cash on hand column_b["line_6a"] = self.report.form_3x.L6a_cash_on_hand_jan_1_ytd if self.previous_report: - column_a["line_6b"] = self.previous_report.form_3x.L8_cash_on_hand_at_close_period # noqa: E501 + column_a["line_6b"] = self.previous_report.form_3x.L8_cash_on_hand_at_close_period # noqa: E501 else: - # user defined cash on hand + # user defined cash on hand column_a["line_6b"] = self.report.form_3x.L6a_cash_on_hand_jan_1_ytd if column_a["line_6b"] and column_b["line_6a"]: From 55083beac7bf9329f52ba2e5588e86e357affe5f Mon Sep 17 00:00:00 2001 From: toddlees Date: Tue, 12 Dec 2023 09:50:19 -0500 Subject: [PATCH 076/109] add is_first --- django-backend/fecfiler/reports/managers.py | 14 +++++++++----- django-backend/fecfiler/reports/serializers.py | 6 ++++-- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/django-backend/fecfiler/reports/managers.py b/django-backend/fecfiler/reports/managers.py index 0f369af1d6..d535ef22c6 100644 --- a/django-backend/fecfiler/reports/managers.py +++ b/django-backend/fecfiler/reports/managers.py @@ -1,9 +1,5 @@ from fecfiler.soft_delete.managers import SoftDeleteManager -from django.db.models import ( - Case, - When, - Value, -) +from django.db.models import Case, When, Value, OuterRef, Exists, Q from enum import Enum """Manager to deterimine fields that are used the same way across reports, @@ -12,6 +8,13 @@ class ReportManager(SoftDeleteManager): def get_queryset(self): + foo = super().get_queryset().filter(form_3x__isnull=False) + print(f"AHOY{foo}") + older_f3x = ( + super() + .get_queryset() + .filter(form_3x__isnull=False, created__lt=OuterRef("created")) + ) queryset = ( super() .get_queryset() @@ -21,6 +24,7 @@ def get_queryset(self): When(form_24__isnull=False, then=ReportType.F24.value), When(form_99__isnull=False, then=ReportType.F99.value), ), + is_first=~Exists(older_f3x), ) ) return queryset diff --git a/django-backend/fecfiler/reports/serializers.py b/django-backend/fecfiler/reports/serializers.py index 9b9e438570..d1ed5ee1a2 100644 --- a/django-backend/fecfiler/reports/serializers.py +++ b/django-backend/fecfiler/reports/serializers.py @@ -3,6 +3,7 @@ ModelSerializer, CharField, UUIDField, + BooleanField, ) from fecfiler.committee_accounts.serializers import CommitteeOwnedSerializer from fecfiler.web_services.serializers import ( @@ -75,6 +76,7 @@ class ReportSerializer(CommitteeOwnedSerializer, FecSchemaValidatorSerializerMix report_code_label = CharField( read_only=True, ) + is_first = BooleanField(read_only=True) form_3x = Form3XSerializer(required=False) form_24 = Form24Serializer(required=False) @@ -135,7 +137,7 @@ def get_fields(): "dotfec", "report", ] - ] + ["report_status", "fields_to_validate", "report_code_label"] + ] + ["report_status", "fields_to_validate", "report_code_label", "is_first"] fields = get_fields() - read_only_fields = ["id", "deleted", "created", "updated"] + read_only_fields = ["id", "deleted", "created", "updated", "is_first"] From 160578078a2f752e1a465d6a9b437708c0510066 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Tue, 12 Dec 2023 13:53:53 -0500 Subject: [PATCH 077/109] Updates validator commit hash --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b80a2ee6d1..9546019607 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ django-cors-headers==3.13.0 django-storages==1.13.1 djangorestframework==3.14.0 drf-spectacular==0.24.2 -git+https://github.com/fecgov/fecfile-validate@90d5633b29316bb2830c44d4ce878fab19798233#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@ee9f62ad1a450426f81722c6ee92e2ae76fc73ee#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.35 gunicorn==20.1.0 Jinja2==3.1.2 From 373fea2563e65046996905ec8e2a7202bec0f434 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Tue, 12 Dec 2023 16:33:25 -0500 Subject: [PATCH 078/109] Fixes form-99 router --- django-backend/fecfiler/reports/urls.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django-backend/fecfiler/reports/urls.py b/django-backend/fecfiler/reports/urls.py index d1e9b58d38..70379e63d7 100644 --- a/django-backend/fecfiler/reports/urls.py +++ b/django-backend/fecfiler/reports/urls.py @@ -1,4 +1,5 @@ from django.urls import path, include +from fecfiler.reports.form_99.views import Form99ViewSet from rest_framework.routers import DefaultRouter from .form_3x.views import Form3XViewSet from .form_24.views import Form24ViewSet @@ -8,7 +9,7 @@ router = DefaultRouter() router.register(r"form-3x", Form3XViewSet, basename="form-3x") router.register(r"form-24", Form24ViewSet, basename="form-24") -router.register(r"form-99", Form24ViewSet, basename="form-99") +router.register(r"form-99", Form99ViewSet, basename="form-99") router.register(r"", ReportViewSet, basename="reports") # The API URLs are now determined automatically by the router. From b687b7a72db9a71d8e037b2695508b2e67bc852a Mon Sep 17 00:00:00 2001 From: Sasha Dresden Date: Tue, 12 Dec 2023 16:11:18 -0500 Subject: [PATCH 079/109] Add endpoint to fetch candidate details so the front end doesn't need to know the fec api key --- .../fecfiler/contacts/test_views.py | 38 ++++++++++++------- django-backend/fecfiler/contacts/views.py | 20 ++++++++++ django-backend/fecfiler/settings/base.py | 1 + 3 files changed, 46 insertions(+), 13 deletions(-) diff --git a/django-backend/fecfiler/contacts/test_views.py b/django-backend/fecfiler/contacts/test_views.py index 491dbf5e40..90d937b015 100644 --- a/django-backend/fecfiler/contacts/test_views.py +++ b/django-backend/fecfiler/contacts/test_views.py @@ -8,6 +8,17 @@ from .models import Contact from .views import ContactViewSet, DeletedContactsViewSet +mock_results = { + "results": [ + {"name": "BIDEN, JOE R", "id": "P60012143", "office_sought": "P"}, + { + "name": "BIDEN, JR., JOSEPH R.", + "id": "P60012465", + "office_sought": "P", + }, + ] +} + def mocked_requests_get_candidates(*args, **kwargs): class MockResponse: @@ -18,19 +29,7 @@ def __init__(self, json_data, status_code): def json(self): return self.json_data - return MockResponse( - { - "results": [ - {"name": "BIDEN, JOE R", "id": "P60012143", "office_sought": "P"}, - { - "name": "BIDEN, JR., JOSEPH R.", - "id": "P60012465", - "office_sought": "P", - }, - ] - }, - 200, - ) + return MockResponse(mock_results, 200) def mocked_requests_get_committees(*args, **kwargs): @@ -60,6 +59,19 @@ def setUp(self): self.user = Account.objects.get(cmtee_id="C12345678") self.factory = RequestFactory() + @mock.patch("requests.get", side_effect=mocked_requests_get_candidates) + def test_candidate(self, mock_get): + request = self.factory.get( + "/api/v1/contacts/candidate?" + "candidate_id=P60012143" + ) + request.user = self.user + response = ContactViewSet.as_view({"get": "candidate"})(request) + + expected_json = mock_results + self.assertEqual(response.status_code, 200) + self.assertJSONEqual(str(response.content, encoding="utf8"), expected_json) + def test_candidate_lookup_no_auth(self): request = self.factory.get("/api/v1/contacts/candidate_lookup") diff --git a/django-backend/fecfiler/contacts/views.py b/django-backend/fecfiler/contacts/views.py index 2c8aae0f01..11dd3b72fa 100644 --- a/django-backend/fecfiler/contacts/views.py +++ b/django-backend/fecfiler/contacts/views.py @@ -11,6 +11,7 @@ from fecfiler.settings import ( FEC_API_CANDIDATE_LOOKUP_ENDPOINT, FEC_API_COMMITTEE_LOOKUP_ENDPOINT, + FEC_API_CANDIDATE_ENDPOINT, FEC_API_KEY, ) from rest_framework.decorators import action @@ -32,6 +33,14 @@ ) +def validate_candidate(candidate_id): + return len(candidate_id) == 9 + + +def sanitize(candidate_id): + return candidate_id + + class ContactViewSet(CommitteeOwnedViewSet): """ This viewset automatically provides `list`, `create`, `retrieve`, @@ -72,6 +81,17 @@ class ContactViewSet(CommitteeOwnedViewSet): ] ordering = ["-created"] + @action(detail=False) + def candidate(self, request): + candidate_id = request.query_params.get("candidate_id") + if candidate_id is None: + return HttpResponseBadRequest() + if validate_candidate(candidate_id) is False: + return HttpResponseBadRequest() + url = FEC_API_CANDIDATE_ENDPOINT + sanitize(candidate_id) + "/" + return JsonResponse(requests.get(url, params=urlencode( + {"api_key": FEC_API_KEY})).json()) + @action(detail=False) def candidate_lookup(self, request): q = request.GET.get("q") diff --git a/django-backend/fecfiler/settings/base.py b/django-backend/fecfiler/settings/base.py index 95f997d850..3aa67b4862 100644 --- a/django-backend/fecfiler/settings/base.py +++ b/django-backend/fecfiler/settings/base.py @@ -261,3 +261,4 @@ "FEC_API_COMMITTEE_LOOKUP_IDS_OVERRIDE" ) FEC_API_CANDIDATE_LOOKUP_ENDPOINT = str(FEC_API) + "candidates/" +FEC_API_CANDIDATE_ENDPOINT = str(FEC_API) + "candidate/" From 1be5c3cc968a32b5e32a126181bc67e7feb22cf2 Mon Sep 17 00:00:00 2001 From: toddlees Date: Wed, 13 Dec 2023 10:13:57 -0500 Subject: [PATCH 080/109] limit subquery to same committee account --- django-backend/fecfiler/reports/managers.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/django-backend/fecfiler/reports/managers.py b/django-backend/fecfiler/reports/managers.py index d535ef22c6..04303cf691 100644 --- a/django-backend/fecfiler/reports/managers.py +++ b/django-backend/fecfiler/reports/managers.py @@ -8,12 +8,14 @@ class ReportManager(SoftDeleteManager): def get_queryset(self): - foo = super().get_queryset().filter(form_3x__isnull=False) - print(f"AHOY{foo}") older_f3x = ( super() .get_queryset() - .filter(form_3x__isnull=False, created__lt=OuterRef("created")) + .filter( + form_3x__isnull=False, + committee_account_id=OuterRef("committee_account_id"), + created__lt=OuterRef("created"), + ) ) queryset = ( super() From b012aa041b23a331427df1b289a08dd670d4afc4 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Wed, 13 Dec 2023 13:32:57 -0500 Subject: [PATCH 081/109] F3X Reports have their L6a cash on hand year field set on creation --- django-backend/fecfiler/reports/form_3x/serializers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django-backend/fecfiler/reports/form_3x/serializers.py b/django-backend/fecfiler/reports/form_3x/serializers.py index fb8ffcab3b..44f87a3b88 100644 --- a/django-backend/fecfiler/reports/form_3x/serializers.py +++ b/django-backend/fecfiler/reports/form_3x/serializers.py @@ -345,8 +345,9 @@ def to_internal_value(self, data): def create(self, validated_data: dict): with transaction.atomic(): form_3x_data = get_model_data(validated_data, Form3X) - form_3x = Form3X.objects.create(**form_3x_data) report_data = get_model_data(validated_data, Report) + form_3x_data["L6a_year_for_above_ytd"] = report_data["coverage_from_date"].year # noqa: E501 + form_3x = Form3X.objects.create(**form_3x_data) report_data["form_3x_id"] = form_3x.id report = super().create(report_data) return report From 111ac072b94833c9472944c9d80e802195f426bf Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Wed, 13 Dec 2023 13:33:22 -0500 Subject: [PATCH 082/109] Adds migration to fix pre-existing reports --- .../migrations/0007_update_l6a_coh_year.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 django-backend/fecfiler/reports/migrations/0007_update_l6a_coh_year.py diff --git a/django-backend/fecfiler/reports/migrations/0007_update_l6a_coh_year.py b/django-backend/fecfiler/reports/migrations/0007_update_l6a_coh_year.py new file mode 100644 index 0000000000..d2c2875794 --- /dev/null +++ b/django-backend/fecfiler/reports/migrations/0007_update_l6a_coh_year.py @@ -0,0 +1,24 @@ +from django.db import migrations + + +def update_coh_year(apps, schema_editor): + Report = apps.get_model("reports", "Report") # noqa + reports_to_update = Report.objects.filter( + form_3x__isnull=False + ) + for report in reports_to_update: + year = report.coverage_from_date.year + report.form_3x.L6a_year_for_above_ytd = year + report.form_3x.save() + + +class Migration(migrations.Migration): + initial = True + + dependencies = [ + ("reports", "0006_report_calculation_token"), + ] + + operations = [ + migrations.RunPython(update_coh_year, migrations.RunPython.noop), + ] From 9959bacf97332cb33aa8cedfdbe385e1eb5b8ddc Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Thu, 14 Dec 2023 11:13:11 -0500 Subject: [PATCH 083/109] Adds a filter to the new migration to skip over reports that have no coverage from date --- .../fecfiler/reports/migrations/0007_update_l6a_coh_year.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django-backend/fecfiler/reports/migrations/0007_update_l6a_coh_year.py b/django-backend/fecfiler/reports/migrations/0007_update_l6a_coh_year.py index d2c2875794..79f4402247 100644 --- a/django-backend/fecfiler/reports/migrations/0007_update_l6a_coh_year.py +++ b/django-backend/fecfiler/reports/migrations/0007_update_l6a_coh_year.py @@ -4,7 +4,8 @@ def update_coh_year(apps, schema_editor): Report = apps.get_model("reports", "Report") # noqa reports_to_update = Report.objects.filter( - form_3x__isnull=False + form_3x__isnull=False, + coverage_from_date__isnull=False ) for report in reports_to_update: year = report.coverage_from_date.year From 004e121f4f00e22616b6aa18f017395b296819b3 Mon Sep 17 00:00:00 2001 From: toddlees Date: Fri, 15 Dec 2023 10:48:49 -0500 Subject: [PATCH 084/109] lint --- django-backend/fecfiler/reports/managers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django-backend/fecfiler/reports/managers.py b/django-backend/fecfiler/reports/managers.py index 04303cf691..dd70d367ae 100644 --- a/django-backend/fecfiler/reports/managers.py +++ b/django-backend/fecfiler/reports/managers.py @@ -1,5 +1,5 @@ from fecfiler.soft_delete.managers import SoftDeleteManager -from django.db.models import Case, When, Value, OuterRef, Exists, Q +from django.db.models import Case, When, Value, OuterRef, Exists from enum import Enum """Manager to deterimine fields that are used the same way across reports, From c6d6360985a72b1787066c8294d3e7f522c7948d Mon Sep 17 00:00:00 2001 From: toddlees Date: Fri, 15 Dec 2023 13:43:58 -0500 Subject: [PATCH 085/109] handle create --- django-backend/fecfiler/reports/form_3x/serializers.py | 5 +++-- django-backend/fecfiler/reports/serializers.py | 5 +++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/django-backend/fecfiler/reports/form_3x/serializers.py b/django-backend/fecfiler/reports/form_3x/serializers.py index fb8ffcab3b..dbd8d89585 100644 --- a/django-backend/fecfiler/reports/form_3x/serializers.py +++ b/django-backend/fecfiler/reports/form_3x/serializers.py @@ -23,6 +23,7 @@ class Form3XSerializer(ReportSerializer): schema_name = "F3X" + is_first = BooleanField(read_only=True, allow_null=True) change_of_address = BooleanField(required=False, allow_null=True) street_1 = CharField(required=False, allow_null=True) street_2 = CharField(required=False, allow_null=True) @@ -392,7 +393,7 @@ class Meta(ReportSerializer.Meta): for f in Form3X._meta.get_fields() if f.name not in ["committee_name", "report"] ] - + ["fields_to_validate"] + + ["fields_to_validate", "is_first"] ) - read_only_fields = ["id"] + read_only_fields = ["id", "deleted", "created", "updated", "is_first"] diff --git a/django-backend/fecfiler/reports/serializers.py b/django-backend/fecfiler/reports/serializers.py index d1ed5ee1a2..a5690df4ab 100644 --- a/django-backend/fecfiler/reports/serializers.py +++ b/django-backend/fecfiler/reports/serializers.py @@ -110,6 +110,11 @@ def to_representation(self, instance, depth=0): for property in form_1m: if not representation.get(property): representation[property] = form_1m[property] + + if not representation["is_first"]: + this_report = Report.objects.get(id=representation["id"]) + representation["is_first"] = this_report.is_first if this_report else True + return representation def validate(self, data): From db1b571197cfa36d709fa57331432e67e15004da Mon Sep 17 00:00:00 2001 From: Sasha Dresden Date: Fri, 15 Dec 2023 13:32:48 -0500 Subject: [PATCH 086/109] Update validation to only check if no id provided, otherwise, let remote call handle the validation to reduce duplication --- django-backend/fecfiler/contacts/views.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/django-backend/fecfiler/contacts/views.py b/django-backend/fecfiler/contacts/views.py index 11dd3b72fa..7f57edc434 100644 --- a/django-backend/fecfiler/contacts/views.py +++ b/django-backend/fecfiler/contacts/views.py @@ -33,11 +33,9 @@ ) -def validate_candidate(candidate_id): - return len(candidate_id) == 9 - - -def sanitize(candidate_id): +def validate_and_sanitize_candidate(candidate_id): + if candidate_id is None: + raise AssertionError("No Candidate ID provided") return candidate_id @@ -84,13 +82,13 @@ class ContactViewSet(CommitteeOwnedViewSet): @action(detail=False) def candidate(self, request): candidate_id = request.query_params.get("candidate_id") - if candidate_id is None: - return HttpResponseBadRequest() - if validate_candidate(candidate_id) is False: + try: + url = (FEC_API_CANDIDATE_ENDPOINT + + validate_and_sanitize_candidate(candidate_id) + "/") + return JsonResponse(requests.get(url, params=urlencode( + {"api_key": FEC_API_KEY})).json()) + except AssertionError: return HttpResponseBadRequest() - url = FEC_API_CANDIDATE_ENDPOINT + sanitize(candidate_id) + "/" - return JsonResponse(requests.get(url, params=urlencode( - {"api_key": FEC_API_KEY})).json()) @action(detail=False) def candidate_lookup(self, request): @@ -246,7 +244,7 @@ def update(self, request, *args, **kwargs): return super().update(request, *args, **kwargs) def get_int_param_value( - self, request, param_name: str, default_param_value: int, max_param_value: int + self, request, param_name: str, default_param_value: int, max_param_value: int ): if request: param_val = request.GET.get(param_name, "") From 14b50528723a7fbe13e12874dee4010959c2a70f Mon Sep 17 00:00:00 2001 From: toddlees Date: Fri, 15 Dec 2023 13:54:11 -0500 Subject: [PATCH 087/109] safe get --- django-backend/fecfiler/reports/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django-backend/fecfiler/reports/serializers.py b/django-backend/fecfiler/reports/serializers.py index a5690df4ab..daea126112 100644 --- a/django-backend/fecfiler/reports/serializers.py +++ b/django-backend/fecfiler/reports/serializers.py @@ -111,7 +111,7 @@ def to_representation(self, instance, depth=0): if not representation.get(property): representation[property] = form_1m[property] - if not representation["is_first"]: + if not representation.get("is_first"): this_report = Report.objects.get(id=representation["id"]) representation["is_first"] = this_report.is_first if this_report else True From 4ccd514d87b90b8e4a7262b9a0d3189833a1ab62 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Fri, 15 Dec 2023 15:39:25 -0500 Subject: [PATCH 088/109] Updates form_99 to change field name --- .../fecfiler/reports/form_99/models.py | 3 +-- .../fecfiler/reports/form_99/serializers.py | 2 +- .../0007_rename_text_form99_text_message.py | 18 ++++++++++++++++++ 3 files changed, 20 insertions(+), 3 deletions(-) create mode 100644 django-backend/fecfiler/reports/migrations/0007_rename_text_form99_text_message.py diff --git a/django-backend/fecfiler/reports/form_99/models.py b/django-backend/fecfiler/reports/form_99/models.py index 497f459ac8..fc21ea341b 100644 --- a/django-backend/fecfiler/reports/form_99/models.py +++ b/django-backend/fecfiler/reports/form_99/models.py @@ -23,8 +23,7 @@ class Form99(models.Model): state = models.TextField(null=True, blank=True) zip = models.TextField(null=True, blank=True) text_code = models.TextField(null=True, blank=True) - - text = models.TextField(null=True, blank=True) + text_message = models.TextField(null=True, blank=True) class Meta: app_label = "reports" diff --git a/django-backend/fecfiler/reports/form_99/serializers.py b/django-backend/fecfiler/reports/form_99/serializers.py index aca345e50d..48cd93a587 100644 --- a/django-backend/fecfiler/reports/form_99/serializers.py +++ b/django-backend/fecfiler/reports/form_99/serializers.py @@ -19,7 +19,7 @@ class Form99Serializer(ReportSerializer): state = CharField(required=False, allow_null=True) zip = CharField(required=False, allow_null=True) text_code = CharField(required=False, allow_null=True) - text = CharField(required=False, allow_null=True) + text_message = CharField(required=False, allow_null=True) def to_internal_value(self, data): internal = super().to_internal_value(data) diff --git a/django-backend/fecfiler/reports/migrations/0007_rename_text_form99_text_message.py b/django-backend/fecfiler/reports/migrations/0007_rename_text_form99_text_message.py new file mode 100644 index 0000000000..8252a05154 --- /dev/null +++ b/django-backend/fecfiler/reports/migrations/0007_rename_text_form99_text_message.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.7 on 2023-12-15 20:28 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('reports', '0006_report_calculation_token'), + ] + + operations = [ + migrations.RenameField( + model_name='form99', + old_name='text', + new_name='text_message', + ), + ] From 0db666e6f162fc1b3359c4c9d63af5ee5f286566 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Fri, 15 Dec 2023 17:51:21 -0500 Subject: [PATCH 089/109] Use fictitious names in test data --- django-backend/fecfiler/contacts/test_views.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/django-backend/fecfiler/contacts/test_views.py b/django-backend/fecfiler/contacts/test_views.py index 90d937b015..6dc3709a21 100644 --- a/django-backend/fecfiler/contacts/test_views.py +++ b/django-backend/fecfiler/contacts/test_views.py @@ -10,9 +10,9 @@ mock_results = { "results": [ - {"name": "BIDEN, JOE R", "id": "P60012143", "office_sought": "P"}, + {"name": "LNAME, FNAME I", "id": "P60012143", "office_sought": "P"}, { - "name": "BIDEN, JR., JOSEPH R.", + "name": "LNAME, FNAME", "id": "P60012465", "office_sought": "P", }, @@ -100,9 +100,9 @@ def test_candidate_lookup_happy_path(self, mock_get): expected_json = { "fec_api_candidates": [ - {"name": "BIDEN, JOE R", "id": "P60012143", "office_sought": "P"}, + {"name": "LNAME, FNAME I", "id": "P60012143", "office_sought": "P"}, { - "name": "BIDEN, JR., JOSEPH R.", + "name": "LNAME, FNAME", "id": "P60012465", "office_sought": "P", }, From f367a1fedf6a02e4362d389a61620c65b7bf6901 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Mon, 18 Dec 2023 12:56:05 -0500 Subject: [PATCH 090/109] Fixes merge conflict in migrations --- ..._text_message.py => 0008_rename_text_form99_text_message.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename django-backend/fecfiler/reports/migrations/{0007_rename_text_form99_text_message.py => 0008_rename_text_form99_text_message.py} (85%) diff --git a/django-backend/fecfiler/reports/migrations/0007_rename_text_form99_text_message.py b/django-backend/fecfiler/reports/migrations/0008_rename_text_form99_text_message.py similarity index 85% rename from django-backend/fecfiler/reports/migrations/0007_rename_text_form99_text_message.py rename to django-backend/fecfiler/reports/migrations/0008_rename_text_form99_text_message.py index 8252a05154..ead37c769d 100644 --- a/django-backend/fecfiler/reports/migrations/0007_rename_text_form99_text_message.py +++ b/django-backend/fecfiler/reports/migrations/0008_rename_text_form99_text_message.py @@ -6,7 +6,7 @@ class Migration(migrations.Migration): dependencies = [ - ('reports', '0006_report_calculation_token'), + ('reports', '0007_update_l6a_coh_year'), ] operations = [ From 8b57f38c193d584e48f8b4aada76a6987eb9d701 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Mon, 18 Dec 2023 16:23:23 -0500 Subject: [PATCH 091/109] Updates validator commit hash --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9546019607..67dc494420 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ django-cors-headers==3.13.0 django-storages==1.13.1 djangorestframework==3.14.0 drf-spectacular==0.24.2 -git+https://github.com/fecgov/fecfile-validate@ee9f62ad1a450426f81722c6ee92e2ae76fc73ee#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@0d7ea38492c5e5407b3d4a31d494527bf32b7133#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.35 gunicorn==20.1.0 Jinja2==3.1.2 From 26e4b27e27d6df2f3af510c567c4e8304feb1e4d Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Tue, 19 Dec 2023 07:18:45 -0500 Subject: [PATCH 092/109] Renamed text_message to message_text to match spec --- django-backend/fecfiler/reports/form_99/models.py | 2 +- django-backend/fecfiler/reports/form_99/serializers.py | 2 +- .../reports/migrations/0008_rename_text_form99_text_message.py | 2 +- requirements.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/django-backend/fecfiler/reports/form_99/models.py b/django-backend/fecfiler/reports/form_99/models.py index fc21ea341b..d64d4daa27 100644 --- a/django-backend/fecfiler/reports/form_99/models.py +++ b/django-backend/fecfiler/reports/form_99/models.py @@ -23,7 +23,7 @@ class Form99(models.Model): state = models.TextField(null=True, blank=True) zip = models.TextField(null=True, blank=True) text_code = models.TextField(null=True, blank=True) - text_message = models.TextField(null=True, blank=True) + message_text = models.TextField(null=True, blank=True) class Meta: app_label = "reports" diff --git a/django-backend/fecfiler/reports/form_99/serializers.py b/django-backend/fecfiler/reports/form_99/serializers.py index 48cd93a587..2a591b93a4 100644 --- a/django-backend/fecfiler/reports/form_99/serializers.py +++ b/django-backend/fecfiler/reports/form_99/serializers.py @@ -19,7 +19,7 @@ class Form99Serializer(ReportSerializer): state = CharField(required=False, allow_null=True) zip = CharField(required=False, allow_null=True) text_code = CharField(required=False, allow_null=True) - text_message = CharField(required=False, allow_null=True) + message_text = CharField(required=False, allow_null=True) def to_internal_value(self, data): internal = super().to_internal_value(data) diff --git a/django-backend/fecfiler/reports/migrations/0008_rename_text_form99_text_message.py b/django-backend/fecfiler/reports/migrations/0008_rename_text_form99_text_message.py index ead37c769d..96c6c64862 100644 --- a/django-backend/fecfiler/reports/migrations/0008_rename_text_form99_text_message.py +++ b/django-backend/fecfiler/reports/migrations/0008_rename_text_form99_text_message.py @@ -13,6 +13,6 @@ class Migration(migrations.Migration): migrations.RenameField( model_name='form99', old_name='text', - new_name='text_message', + new_name='message_text', ), ] diff --git a/requirements.txt b/requirements.txt index 67dc494420..5a9a0b0e7c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ django-cors-headers==3.13.0 django-storages==1.13.1 djangorestframework==3.14.0 drf-spectacular==0.24.2 -git+https://github.com/fecgov/fecfile-validate@0d7ea38492c5e5407b3d4a31d494527bf32b7133#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@076d0859d1aaef8d9188fdb0df2069be3eb9f7558#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.35 gunicorn==20.1.0 Jinja2==3.1.2 From 409f8823972b8541f8d1cb731017fdeccb3341a4 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Tue, 19 Dec 2023 07:21:39 -0500 Subject: [PATCH 093/109] Update validate commit hash --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5a9a0b0e7c..8a4a2a6f2a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ django-cors-headers==3.13.0 django-storages==1.13.1 djangorestframework==3.14.0 drf-spectacular==0.24.2 -git+https://github.com/fecgov/fecfile-validate@076d0859d1aaef8d9188fdb0df2069be3eb9f7558#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@76d0859d1aaef8d9188fdb0df2069be3eb9f7558#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.35 gunicorn==20.1.0 Jinja2==3.1.2 From 912b60775d1ce874369a396a08b9d4b259eebc35 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Tue, 19 Dec 2023 07:44:07 -0500 Subject: [PATCH 094/109] Fix F99 unit test --- .../fecfiler/web_services/dot_fec/dot_fec_composer.py | 2 +- django-backend/fecfiler/web_services/fixtures/test_f99.json | 5 ++++- requirements.txt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/django-backend/fecfiler/web_services/dot_fec/dot_fec_composer.py b/django-backend/fecfiler/web_services/dot_fec/dot_fec_composer.py index 10e86bac2e..e280ab0559 100644 --- a/django-backend/fecfiler/web_services/dot_fec/dot_fec_composer.py +++ b/django-backend/fecfiler/web_services/dot_fec/dot_fec_composer.py @@ -211,7 +211,7 @@ def compose_dot_fec(report_id, upload_submission_record_id): """Free text""" if report.get_form_name() == "F99": - file_content = add_free_text(file_content, report.form_99.text) + file_content = add_free_text(file_content, report.form_99.message_text) return file_content except Exception as error: diff --git a/django-backend/fecfiler/web_services/fixtures/test_f99.json b/django-backend/fecfiler/web_services/fixtures/test_f99.json index aa77ab5ff4..2e8131b9e7 100644 --- a/django-backend/fecfiler/web_services/fixtures/test_f99.json +++ b/django-backend/fecfiler/web_services/fixtures/test_f99.json @@ -8,8 +8,11 @@ "city": "test_string_value", "state": "test_string_value", "zip": "test_string_value", + "treasurer_first_name": "test_string_value", + "treasurer_last_name": "test_string_value", + "date_signed": "2005-01-30", "text_code": "ABC", - "text": "\nBEHOLD! A large text string\nwith new lines" + "message_text": "\nBEHOLD! A large text string\nwith new lines" } }, { diff --git a/requirements.txt b/requirements.txt index 8a4a2a6f2a..0e08f99a97 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ django-cors-headers==3.13.0 django-storages==1.13.1 djangorestframework==3.14.0 drf-spectacular==0.24.2 -git+https://github.com/fecgov/fecfile-validate@76d0859d1aaef8d9188fdb0df2069be3eb9f7558#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@1fe969faf4549a10b6a2576636ce20b128e3826f#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.35 gunicorn==20.1.0 Jinja2==3.1.2 From a2474ca44de2bc2741b06dbf5f0da55111ac87de Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Tue, 19 Dec 2023 08:27:40 -0500 Subject: [PATCH 095/109] Fix F99 unit test --- .../fecfiler/web_services/dot_fec/dot_fec_serializer.py | 2 +- .../fecfiler/web_services/dot_fec/test_dot_fec_composer.py | 2 +- django-backend/fecfiler/web_services/fixtures/test_f99.json | 3 --- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/django-backend/fecfiler/web_services/dot_fec/dot_fec_serializer.py b/django-backend/fecfiler/web_services/dot_fec/dot_fec_serializer.py index 085d5ecc4c..0cf23a4c84 100644 --- a/django-backend/fecfiler/web_services/dot_fec/dot_fec_serializer.py +++ b/django-backend/fecfiler/web_services/dot_fec/dot_fec_serializer.py @@ -121,7 +121,7 @@ def extract_row_config(schema_name): schema = validate.get_schema(schema_name) schema_properties = schema.get("properties", {}).items() column_sequences = { - v.get("fec_spec", {}).get("COL_SEQ", None): k for k, v in schema_properties + v.get("fec_spec", {}).get("COL_SEQ", None): k for k, v in schema_properties if v.get("fec_spec", {}).get("COL_SEQ", None) } row_length = max(column_sequences.keys()) return column_sequences, row_length diff --git a/django-backend/fecfiler/web_services/dot_fec/test_dot_fec_composer.py b/django-backend/fecfiler/web_services/dot_fec/test_dot_fec_composer.py index b3266d6841..88507983b1 100644 --- a/django-backend/fecfiler/web_services/dot_fec/test_dot_fec_composer.py +++ b/django-backend/fecfiler/web_services/dot_fec/test_dot_fec_composer.py @@ -52,7 +52,7 @@ def test_f99(self): content = compose_dot_fec("11111111-1111-1111-1111-111111111111", None) split_content = content.split("\n") split_report_row = split_content[1].split(FS_STR) - self.assertEqual(split_report_row[15], "ABC\r") + self.assertEqual(split_report_row[14], "ABC\r") free_text = content[content.find("[BEGINTEXT]"):] self.assertEqual( free_text, diff --git a/django-backend/fecfiler/web_services/fixtures/test_f99.json b/django-backend/fecfiler/web_services/fixtures/test_f99.json index 2e8131b9e7..f569bdf09c 100644 --- a/django-backend/fecfiler/web_services/fixtures/test_f99.json +++ b/django-backend/fecfiler/web_services/fixtures/test_f99.json @@ -8,9 +8,6 @@ "city": "test_string_value", "state": "test_string_value", "zip": "test_string_value", - "treasurer_first_name": "test_string_value", - "treasurer_last_name": "test_string_value", - "date_signed": "2005-01-30", "text_code": "ABC", "message_text": "\nBEHOLD! A large text string\nwith new lines" } From d85c5508ca237f6b98550d86f1cd96450dd320b6 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Tue, 19 Dec 2023 08:37:37 -0500 Subject: [PATCH 096/109] Fix unit test for F99 serializer --- .../fecfiler/reports/form_99/tests/test_serializers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/django-backend/fecfiler/reports/form_99/tests/test_serializers.py b/django-backend/fecfiler/reports/form_99/tests/test_serializers.py index a0e5570a79..8376c72ae6 100644 --- a/django-backend/fecfiler/reports/form_99/tests/test_serializers.py +++ b/django-backend/fecfiler/reports/form_99/tests/test_serializers.py @@ -23,6 +23,7 @@ def setUp(self): "state": "AL", "zip": "12345", "text_code": "MSM", + "message_text": "A message to FEC", "fields_to_validate": [f.name for f in Form99._meta.get_fields()], } From b0fbaa220a518120e0a6aeebd04216b64ceecf7a Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Tue, 19 Dec 2023 08:44:05 -0500 Subject: [PATCH 097/109] Fix linting error --- .../fecfiler/web_services/dot_fec/dot_fec_serializer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django-backend/fecfiler/web_services/dot_fec/dot_fec_serializer.py b/django-backend/fecfiler/web_services/dot_fec/dot_fec_serializer.py index 0cf23a4c84..83ddc86eac 100644 --- a/django-backend/fecfiler/web_services/dot_fec/dot_fec_serializer.py +++ b/django-backend/fecfiler/web_services/dot_fec/dot_fec_serializer.py @@ -121,7 +121,8 @@ def extract_row_config(schema_name): schema = validate.get_schema(schema_name) schema_properties = schema.get("properties", {}).items() column_sequences = { - v.get("fec_spec", {}).get("COL_SEQ", None): k for k, v in schema_properties if v.get("fec_spec", {}).get("COL_SEQ", None) + v.get("fec_spec", {}).get("COL_SEQ", None): k for k, v in \ + schema_properties if v.get("fec_spec", {}).get("COL_SEQ", None) } row_length = max(column_sequences.keys()) return column_sequences, row_length From 8123083171aa93b55a1b5f0faeeb16bd6e03ab0f Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Tue, 19 Dec 2023 08:50:24 -0500 Subject: [PATCH 098/109] Fix linting error reported by sonarcloud --- .../fecfiler/web_services/dot_fec/dot_fec_serializer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django-backend/fecfiler/web_services/dot_fec/dot_fec_serializer.py b/django-backend/fecfiler/web_services/dot_fec/dot_fec_serializer.py index 83ddc86eac..fbb68c9c38 100644 --- a/django-backend/fecfiler/web_services/dot_fec/dot_fec_serializer.py +++ b/django-backend/fecfiler/web_services/dot_fec/dot_fec_serializer.py @@ -121,7 +121,7 @@ def extract_row_config(schema_name): schema = validate.get_schema(schema_name) schema_properties = schema.get("properties", {}).items() column_sequences = { - v.get("fec_spec", {}).get("COL_SEQ", None): k for k, v in \ + v.get("fec_spec", {}).get("COL_SEQ", None): k for k, v in schema_properties if v.get("fec_spec", {}).get("COL_SEQ", None) } row_length = max(column_sequences.keys()) From 83af253f52cd9bfb721547430cbbc078d7e6e3a8 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Tue, 19 Dec 2023 10:38:06 -0500 Subject: [PATCH 099/109] Move filter for fec serializer --- .../fecfiler/web_services/dot_fec/dot_fec_serializer.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/django-backend/fecfiler/web_services/dot_fec/dot_fec_serializer.py b/django-backend/fecfiler/web_services/dot_fec/dot_fec_serializer.py index fbb68c9c38..3aebb7f60f 100644 --- a/django-backend/fecfiler/web_services/dot_fec/dot_fec_serializer.py +++ b/django-backend/fecfiler/web_services/dot_fec/dot_fec_serializer.py @@ -121,8 +121,7 @@ def extract_row_config(schema_name): schema = validate.get_schema(schema_name) schema_properties = schema.get("properties", {}).items() column_sequences = { - v.get("fec_spec", {}).get("COL_SEQ", None): k for k, v in - schema_properties if v.get("fec_spec", {}).get("COL_SEQ", None) + v.get("fec_spec", {}).get("COL_SEQ", None): k for k, v in schema_properties } - row_length = max(column_sequences.keys()) + row_length = max(filter(lambda k: k is not None, column_sequences.keys())) return column_sequences, row_length From fbbd8599e07e9c6120b48400b8e33f41da35e5ca Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Tue, 19 Dec 2023 10:58:09 -0500 Subject: [PATCH 100/109] Add committee_name field to F99 --- .../fecfiler/reports/form_99/models.py | 1 + .../migrations/0009_form99_committee_name.py | 18 ++++++++++++++++++ requirements.txt | 2 +- 3 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 django-backend/fecfiler/reports/migrations/0009_form99_committee_name.py diff --git a/django-backend/fecfiler/reports/form_99/models.py b/django-backend/fecfiler/reports/form_99/models.py index d64d4daa27..d35d102c2c 100644 --- a/django-backend/fecfiler/reports/form_99/models.py +++ b/django-backend/fecfiler/reports/form_99/models.py @@ -17,6 +17,7 @@ class Form99(models.Model): unique=True, ) + committee_name = models.TextField(null=True, blank=True) street_1 = models.TextField(null=True, blank=True) street_2 = models.TextField(null=True, blank=True) city = models.TextField(null=True, blank=True) diff --git a/django-backend/fecfiler/reports/migrations/0009_form99_committee_name.py b/django-backend/fecfiler/reports/migrations/0009_form99_committee_name.py new file mode 100644 index 0000000000..fd7e3bdff5 --- /dev/null +++ b/django-backend/fecfiler/reports/migrations/0009_form99_committee_name.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.7 on 2023-12-19 15:50 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('reports', '0008_rename_text_form99_text_message'), + ] + + operations = [ + migrations.AddField( + model_name='form99', + name='committee_name', + field=models.TextField(blank=True, null=True), + ), + ] diff --git a/requirements.txt b/requirements.txt index 0e08f99a97..3df4dbc7e1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ django-cors-headers==3.13.0 django-storages==1.13.1 djangorestframework==3.14.0 drf-spectacular==0.24.2 -git+https://github.com/fecgov/fecfile-validate@1fe969faf4549a10b6a2576636ce20b128e3826f#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@653cb0202dae5f828f8ae3b83391c0d978c566eb#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.35 gunicorn==20.1.0 Jinja2==3.1.2 From 31792994b1df7ad85bbd27e86aabcb583a35deb4 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Tue, 19 Dec 2023 11:21:58 -0500 Subject: [PATCH 101/109] Revert "Add committee_name field to F99" This reverts commit fbbd8599e07e9c6120b48400b8e33f41da35e5ca. --- .../fecfiler/reports/form_99/models.py | 1 - .../migrations/0009_form99_committee_name.py | 18 ------------------ requirements.txt | 2 +- 3 files changed, 1 insertion(+), 20 deletions(-) delete mode 100644 django-backend/fecfiler/reports/migrations/0009_form99_committee_name.py diff --git a/django-backend/fecfiler/reports/form_99/models.py b/django-backend/fecfiler/reports/form_99/models.py index d35d102c2c..d64d4daa27 100644 --- a/django-backend/fecfiler/reports/form_99/models.py +++ b/django-backend/fecfiler/reports/form_99/models.py @@ -17,7 +17,6 @@ class Form99(models.Model): unique=True, ) - committee_name = models.TextField(null=True, blank=True) street_1 = models.TextField(null=True, blank=True) street_2 = models.TextField(null=True, blank=True) city = models.TextField(null=True, blank=True) diff --git a/django-backend/fecfiler/reports/migrations/0009_form99_committee_name.py b/django-backend/fecfiler/reports/migrations/0009_form99_committee_name.py deleted file mode 100644 index fd7e3bdff5..0000000000 --- a/django-backend/fecfiler/reports/migrations/0009_form99_committee_name.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 4.2.7 on 2023-12-19 15:50 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('reports', '0008_rename_text_form99_text_message'), - ] - - operations = [ - migrations.AddField( - model_name='form99', - name='committee_name', - field=models.TextField(blank=True, null=True), - ), - ] diff --git a/requirements.txt b/requirements.txt index 3df4dbc7e1..0e08f99a97 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ django-cors-headers==3.13.0 django-storages==1.13.1 djangorestframework==3.14.0 drf-spectacular==0.24.2 -git+https://github.com/fecgov/fecfile-validate@653cb0202dae5f828f8ae3b83391c0d978c566eb#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@1fe969faf4549a10b6a2576636ce20b128e3826f#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.35 gunicorn==20.1.0 Jinja2==3.1.2 From ad2e8ba8ff525891bf50e4d37b99d5802974369d Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Tue, 19 Dec 2023 12:03:43 -0500 Subject: [PATCH 102/109] update validate commit hash --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0e08f99a97..3df4dbc7e1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ django-cors-headers==3.13.0 django-storages==1.13.1 djangorestframework==3.14.0 drf-spectacular==0.24.2 -git+https://github.com/fecgov/fecfile-validate@1fe969faf4549a10b6a2576636ce20b128e3826f#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@653cb0202dae5f828f8ae3b83391c0d978c566eb#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.35 gunicorn==20.1.0 Jinja2==3.1.2 From b3cfaf89a79c0c005e34384bde61810f9073b906 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Tue, 19 Dec 2023 14:24:39 -0500 Subject: [PATCH 103/109] Revert "Revert "Add committee_name field to F99"" This reverts commit 31792994b1df7ad85bbd27e86aabcb583a35deb4. --- .../fecfiler/reports/form_99/models.py | 1 + .../migrations/0009_form99_committee_name.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 django-backend/fecfiler/reports/migrations/0009_form99_committee_name.py diff --git a/django-backend/fecfiler/reports/form_99/models.py b/django-backend/fecfiler/reports/form_99/models.py index d64d4daa27..d35d102c2c 100644 --- a/django-backend/fecfiler/reports/form_99/models.py +++ b/django-backend/fecfiler/reports/form_99/models.py @@ -17,6 +17,7 @@ class Form99(models.Model): unique=True, ) + committee_name = models.TextField(null=True, blank=True) street_1 = models.TextField(null=True, blank=True) street_2 = models.TextField(null=True, blank=True) city = models.TextField(null=True, blank=True) diff --git a/django-backend/fecfiler/reports/migrations/0009_form99_committee_name.py b/django-backend/fecfiler/reports/migrations/0009_form99_committee_name.py new file mode 100644 index 0000000000..fd7e3bdff5 --- /dev/null +++ b/django-backend/fecfiler/reports/migrations/0009_form99_committee_name.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.7 on 2023-12-19 15:50 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('reports', '0008_rename_text_form99_text_message'), + ] + + operations = [ + migrations.AddField( + model_name='form99', + name='committee_name', + field=models.TextField(blank=True, null=True), + ), + ] From 0e79dc90b24cb9988578a2f88a307aaa08175c8f Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Tue, 19 Dec 2023 14:38:48 -0500 Subject: [PATCH 104/109] Fix unit tests to handle f99 committee_name --- django-backend/fecfiler/reports/form_99/serializers.py | 3 ++- .../fecfiler/reports/form_99/tests/test_serializers.py | 1 + .../fecfiler/web_services/dot_fec/schema_fields/F99.json | 4 +++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/django-backend/fecfiler/reports/form_99/serializers.py b/django-backend/fecfiler/reports/form_99/serializers.py index 2a591b93a4..4486267f64 100644 --- a/django-backend/fecfiler/reports/form_99/serializers.py +++ b/django-backend/fecfiler/reports/form_99/serializers.py @@ -13,6 +13,7 @@ class Form99Serializer(ReportSerializer): schema_name = "F99" + committee_name = CharField(required=False, allow_null=True) street_1 = CharField(required=False, allow_null=True) street_2 = CharField(required=False, allow_null=True) city = CharField(required=False, allow_null=True) @@ -58,7 +59,7 @@ class Meta(ReportSerializer.Meta): + [ f.name for f in Form99._meta.get_fields() - if f.name not in ["committee_name", "report"] + if f.name not in ["report"] ] + ["fields_to_validate"] ) diff --git a/django-backend/fecfiler/reports/form_99/tests/test_serializers.py b/django-backend/fecfiler/reports/form_99/tests/test_serializers.py index 8376c72ae6..e8154c9c32 100644 --- a/django-backend/fecfiler/reports/form_99/tests/test_serializers.py +++ b/django-backend/fecfiler/reports/form_99/tests/test_serializers.py @@ -14,6 +14,7 @@ class F99SerializerTestCase(TestCase): def setUp(self): self.valid_f99_report = { "form_type": "F99N", + "committee_name": "my committee", "treasurer_last_name": "Testerson", "treasurer_first_name": "George", "date_signed": "2023-11-1", diff --git a/django-backend/fecfiler/web_services/dot_fec/schema_fields/F99.json b/django-backend/fecfiler/web_services/dot_fec/schema_fields/F99.json index 3575b0ce1e..d6eb63bb0c 100644 --- a/django-backend/fecfiler/web_services/dot_fec/schema_fields/F99.json +++ b/django-backend/fecfiler/web_services/dot_fec/schema_fields/F99.json @@ -1,7 +1,9 @@ { "form_type": {}, "filer_committee_id_number": {}, - "committee_name": {}, + "committee_name": { + "path": "form_99.committee_name" + }, "street_1": { "path": "form_99.street_1" }, From b4ea231c9a0742e334ddd186e63761a1be0b1eb0 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Tue, 19 Dec 2023 17:00:24 -0500 Subject: [PATCH 105/109] Add fecfiler_committee_id_number to report payload --- django-backend/fecfiler/reports/serializers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/django-backend/fecfiler/reports/serializers.py b/django-backend/fecfiler/reports/serializers.py index daea126112..41c82ebdf7 100644 --- a/django-backend/fecfiler/reports/serializers.py +++ b/django-backend/fecfiler/reports/serializers.py @@ -115,6 +115,9 @@ def to_representation(self, instance, depth=0): this_report = Report.objects.get(id=representation["id"]) representation["is_first"] = this_report.is_first if this_report else True + request = self.context["request"] + representation["filer_committee_id_number"] = request.user.cmtee_id + return representation def validate(self, data): From af2a20f6e74e21aa7ce4ee49024c539837761668 Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Wed, 20 Dec 2023 08:03:39 -0500 Subject: [PATCH 106/109] Add conditional to putting fecfiler_committee_id_number into report payload --- django-backend/fecfiler/reports/serializers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/django-backend/fecfiler/reports/serializers.py b/django-backend/fecfiler/reports/serializers.py index 41c82ebdf7..14c4b3347c 100644 --- a/django-backend/fecfiler/reports/serializers.py +++ b/django-backend/fecfiler/reports/serializers.py @@ -115,8 +115,9 @@ def to_representation(self, instance, depth=0): this_report = Report.objects.get(id=representation["id"]) representation["is_first"] = this_report.is_first if this_report else True - request = self.context["request"] - representation["filer_committee_id_number"] = request.user.cmtee_id + if self.context: + request = self.context["request"] + representation["filer_committee_id_number"] = request.user.cmtee_id return representation From 84fa67827ffd9b1206e49a1e183efa9aa5aced47 Mon Sep 17 00:00:00 2001 From: toddlees Date: Wed, 20 Dec 2023 13:42:23 -0500 Subject: [PATCH 107/109] protect against undefined text --- .../fecfiler/web_services/dot_fec/dot_fec_composer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django-backend/fecfiler/web_services/dot_fec/dot_fec_composer.py b/django-backend/fecfiler/web_services/dot_fec/dot_fec_composer.py index e280ab0559..0060d3b901 100644 --- a/django-backend/fecfiler/web_services/dot_fec/dot_fec_composer.py +++ b/django-backend/fecfiler/web_services/dot_fec/dot_fec_composer.py @@ -135,7 +135,7 @@ def add_free_text(content, text): (content or "") + "[BEGINTEXT]" + CRLF_STR - + text + + (text or "") + CRLF_STR + "[ENDTEXT]" + CRLF_STR From 1362144df677e90de054b935f5802c41d2e97b43 Mon Sep 17 00:00:00 2001 From: toddlees Date: Fri, 22 Dec 2023 15:07:21 -0500 Subject: [PATCH 108/109] flag all future reports --- django-backend/fecfiler/reports/models.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/django-backend/fecfiler/reports/models.py b/django-backend/fecfiler/reports/models.py index 190e01e8fb..fe518acabf 100644 --- a/django-backend/fecfiler/reports/models.py +++ b/django-backend/fecfiler/reports/models.py @@ -43,9 +43,7 @@ class Report(SoftDeleteModel, CommitteeOwnedModel): date_signed = models.DateField(null=True, blank=True) calculation_status = models.CharField(max_length=255, null=True, blank=True) calculation_token = models.UUIDField( # Prevents race conditions in summary calc. - null=True, - blank=True, - default=None + null=True, blank=True, default=None ) confirmation_email_1 = models.EmailField( @@ -214,7 +212,6 @@ def save(self, *args, **kwargs): reports_to_flag_for_recalculation = Report.objects.filter( committee_account=committee, - coverage_from_date__year=report_year, coverage_from_date__gte=report_date, ) else: From 1f6327a3066b9d68cf65a1c66d15b0cd8e0e1709 Mon Sep 17 00:00:00 2001 From: toddlees Date: Fri, 22 Dec 2023 15:27:53 -0500 Subject: [PATCH 109/109] remove unused year --- django-backend/fecfiler/reports/models.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/django-backend/fecfiler/reports/models.py b/django-backend/fecfiler/reports/models.py index fe518acabf..2586b6f522 100644 --- a/django-backend/fecfiler/reports/models.py +++ b/django-backend/fecfiler/reports/models.py @@ -208,8 +208,6 @@ def save(self, *args, **kwargs): committee = self.report.committee_account report_date = self.report.coverage_from_date if report_date is not None: - report_year = report_date.year - reports_to_flag_for_recalculation = Report.objects.filter( committee_account=committee, coverage_from_date__gte=report_date,