diff --git a/koku/api/report/aws/openshift/query_handler.py b/koku/api/report/aws/openshift/query_handler.py index 5d7717a5b4..d81878f05b 100644 --- a/koku/api/report/aws/openshift/query_handler.py +++ b/koku/api/report/aws/openshift/query_handler.py @@ -38,7 +38,6 @@ def execute_query(self): # noqa: C901 query_group_by = ["date"] + group_by_value query_order_by = ["-date"] query_order_by.extend(self.order) # add implicit ordering - annotations = self._mapper.report_type_map.get("annotations") query_data = query_data.values(*query_group_by).annotate(**annotations) @@ -46,7 +45,6 @@ def execute_query(self): # noqa: C901 query_data = query_data.annotate( account_alias=Coalesce(F(self._mapper.provider_map.get("alias")), "usage_account_id") ) - if self._limit and query_data: query_data = self._group_by_ranks(query, query_data) if not self.parameters.get("order_by"): @@ -94,8 +92,12 @@ def check_if_valid_date_str(date_str): if order_date: sort_term = self._get_group_by()[0] query_order_by.pop(i) - date_filtered_query_data = query_data.filter(usage_start=order_date) - ordered_data = self.order_by(date_filtered_query_data, query_order_by) + filtered_query_data = [] + for index in query_data: + for key, value in index.items(): + if (key == "date") and (value == order_date): + filtered_query_data.append(index) + ordered_data = self.order_by(filtered_query_data, query_order_by) order_of_interest = [] for entry in ordered_data: order_of_interest.append(entry.get(sort_term)) diff --git a/koku/api/report/aws/query_handler.py b/koku/api/report/aws/query_handler.py index 27ecca0b43..5daf42e62b 100644 --- a/koku/api/report/aws/query_handler.py +++ b/koku/api/report/aws/query_handler.py @@ -578,6 +578,7 @@ def execute_individual_query(self, org_unit_applied=False): # noqa: C901 tag_results = None query = query_table.objects.filter(self.query_filter) query_data = query.annotate(**self.annotations) + query_group_by = ["date"] + self._get_group_by() query_order_by = ["-date"] query_order_by.extend(self.order) # add implicit ordering @@ -628,20 +629,23 @@ def check_if_valid_date_str(date_str): break # Remove the date order by as it is not actually used for ordering if order_date: + sort_term = self._get_group_by()[0] query_order_by.pop(i) - date_filtered_query_data = query_data.filter(usage_start=order_date) - ordered_data = self.order_by(date_filtered_query_data, query_order_by) + filtered_query_data = [] + for index in query_data: + for key, value in index.items(): + if (key == "date") and (value == order_date): + filtered_query_data.append(index) + ordered_data = self.order_by(filtered_query_data, query_order_by) order_of_interest = [] for entry in ordered_data: order_of_interest.append(entry.get(sort_term)) # write a special order by function that iterates through the # rest of the days in query_data and puts them in the same order - # return_query_data = [] sorted_data = [item for x in order_of_interest for item in query_data if item.get(sort_term) == x] query_data = self.order_by(sorted_data, ["-date"]) else: - # &order_by[cost]=desc&order_by[date]=2021-08-02 query_data = self.order_by(query_data, query_order_by) # Fetch the data (returning list(dict)) diff --git a/koku/api/report/azure/openshift/query_handler.py b/koku/api/report/azure/openshift/query_handler.py index 92b726f3d7..7cad554ce6 100644 --- a/koku/api/report/azure/openshift/query_handler.py +++ b/koku/api/report/azure/openshift/query_handler.py @@ -81,10 +81,8 @@ def execute_query(self): # noqa: C901 query_group_by = ["date"] + group_by_value query_order_by = ["-date"] query_order_by.extend(self.order) # add implicit ordering - annotations = self._mapper.report_type_map.get("annotations") query_data = query_data.values(*query_group_by).annotate(**annotations) - if self._limit and query_data: query_data = self._group_by_ranks(query, query_data) if not self.parameters.get("order_by"): @@ -122,8 +120,12 @@ def check_if_valid_date_str(date_str): if order_date: sort_term = self._get_group_by()[0] query_order_by.pop(i) - date_filtered_query_data = query_data.filter(usage_start=order_date) - ordered_data = self.order_by(date_filtered_query_data, query_order_by) + filtered_query_data = [] + for index in query_data: + for key, value in index.items(): + if (key == "date") and (value == order_date): + filtered_query_data.append(index) + ordered_data = self.order_by(filtered_query_data, query_order_by) order_of_interest = [] for entry in ordered_data: order_of_interest.append(entry.get(sort_term)) @@ -133,7 +135,6 @@ def check_if_valid_date_str(date_str): sorted_data = [item for x in order_of_interest for item in query_data if item.get(sort_term) == x] query_data = self.order_by(sorted_data, ["-date"]) else: - # &order_by[cost]=desc&order_by[date]=2021-08-02 query_data = self.order_by(query_data, query_order_by) cost_units_value = self._mapper.report_type_map.get("cost_units_fallback", "USD") diff --git a/koku/api/report/azure/query_handler.py b/koku/api/report/azure/query_handler.py index 213a7c90a0..7a6848ae1f 100644 --- a/koku/api/report/azure/query_handler.py +++ b/koku/api/report/azure/query_handler.py @@ -141,7 +141,6 @@ def execute_query(self): # noqa: C901 query_group_by = ["date"] + self._get_group_by() query_order_by = ["-date"] query_order_by.extend(self.order) # add implicit ordering - annotations = self._mapper.report_type_map.get("annotations") query_data = query_data.values(*query_group_by).annotate(**annotations) query_sum = self._build_sum(query) @@ -178,8 +177,12 @@ def check_if_valid_date_str(date_str): if order_date: sort_term = self._get_group_by()[0] query_order_by.pop(i) - date_filtered_query_data = query_data.filter(usage_start=order_date) - ordered_data = self.order_by(date_filtered_query_data, query_order_by) + filtered_query_data = [] + for index in query_data: + for key, value in index.items(): + if (key == "date") and (value == order_date): + filtered_query_data.append(index) + ordered_data = self.order_by(filtered_query_data, query_order_by) order_of_interest = [] for entry in ordered_data: order_of_interest.append(entry.get(sort_term)) diff --git a/koku/api/report/gcp/query_handler.py b/koku/api/report/gcp/query_handler.py index fa8aa6ac29..ef98d6ffd2 100644 --- a/koku/api/report/gcp/query_handler.py +++ b/koku/api/report/gcp/query_handler.py @@ -152,6 +152,7 @@ def execute_query(self): # noqa: C901 query = self.query_table.objects.filter(self.query_filter) query = query.filter(invoice_month__in=self.invoice_months) query_data = query.annotate(**self.annotations) + query_group_by = ["date"] + self._get_group_by() query_order_by = ["-date"] query_order_by.extend(self.order) # add implicit ordering @@ -195,8 +196,12 @@ def check_if_valid_date_str(date_str): if order_date: sort_term = self._get_group_by()[0] query_order_by.pop(i) - date_filtered_query_data = query_data.filter(usage_start=order_date) - ordered_data = self.order_by(date_filtered_query_data, query_order_by) + filtered_query_data = [] + for index in query_data: + for key, value in index.items(): + if (key == "date") and (value == order_date): + filtered_query_data.append(index) + ordered_data = self.order_by(filtered_query_data, query_order_by) order_of_interest = [] for entry in ordered_data: order_of_interest.append(entry.get(sort_term)) diff --git a/koku/api/report/ocp/query_handler.py b/koku/api/report/ocp/query_handler.py index 670dfcdd0d..5101ae6c88 100644 --- a/koku/api/report/ocp/query_handler.py +++ b/koku/api/report/ocp/query_handler.py @@ -172,8 +172,12 @@ def check_if_valid_date_str(date_str): if order_date: sort_term = self._get_group_by()[0] query_order_by.pop(i) - date_filtered_query_data = query_data.filter(usage_start=order_date) - ordered_data = self.order_by(date_filtered_query_data, query_order_by) + filtered_query_data = [] + for index in query_data: + for key, value in index.items(): + if (key == "date") and (value == order_date): + filtered_query_data.append(index) + ordered_data = self.order_by(filtered_query_data, query_order_by) order_of_interest = [] for entry in ordered_data: order_of_interest.append(entry.get(sort_term)) @@ -183,7 +187,6 @@ def check_if_valid_date_str(date_str): sorted_data = [item for x in order_of_interest for item in query_data if item.get(sort_term) == x] query_data = self.order_by(sorted_data, ["-date"]) else: - # &order_by[cost]=desc&order_by[date]=2021-08-02 query_data = self.order_by(query_data, query_order_by) if is_csv_output: diff --git a/koku/api/report/serializers.py b/koku/api/report/serializers.py index fe7b6d3bba..2101246875 100644 --- a/koku/api/report/serializers.py +++ b/koku/api/report/serializers.py @@ -395,7 +395,17 @@ def validate_order_by(self, value): # noqa: C901 continue # sepcial case: we order by date, but we group by an allowed param. if key == "date" and group_keys: - continue + # Checks to make sure the orderby date is allowed + dh = DateHelper() + if ( + value.get("date") >= materialized_view_month_start(dh).date() + and value.get("date") <= dh.today.date() + ): + continue + error = "Order by date must be from {} to {}".format( + materialized_view_month_start(dh).date(), dh.today.date() + ) + raise serializers.ValidationError(error) error[key] = _(f'Order-by "{key}" requires matching Group-by.') raise serializers.ValidationError(error) diff --git a/koku/api/report/test/aws/openshift/test_ocp_aws_query_handler.py b/koku/api/report/test/aws/openshift/test_ocp_aws_query_handler.py index d8a03fdcc7..82119c0b17 100644 --- a/koku/api/report/test/aws/openshift/test_ocp_aws_query_handler.py +++ b/koku/api/report/test/aws/openshift/test_ocp_aws_query_handler.py @@ -452,3 +452,15 @@ def test_ocp_aws_date_incorrect_date(self): url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[service]=*" # noqa: E501 with self.assertRaises(ValidationError): self.mocked_query_params(url, OCPAWSCostView) + + def test_aws_out_of_range_under_date(self): + wrong_date = DateHelper().today.date() - timedelta(days=91) + url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[service]=*" + with self.assertRaises(ValidationError): + self.mocked_query_params(url, OCPAWSCostView) + + def test_aws_out_of_range_over_date(self): + wrong_date = DateHelper().today.date() + timedelta(days=1) + url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[service]=*" + with self.assertRaises(ValidationError): + self.mocked_query_params(url, OCPAWSCostView) diff --git a/koku/api/report/test/aws/tests_queries.py b/koku/api/report/test/aws/tests_queries.py index f924e4f948..a3b76a4e7d 100644 --- a/koku/api/report/test/aws/tests_queries.py +++ b/koku/api/report/test/aws/tests_queries.py @@ -1892,6 +1892,18 @@ def test_aws_date_incorrect_date(self): with self.assertRaises(ValidationError): self.mocked_query_params(url, AWSCostView) + def test_aws_out_of_range_under_date(self): + wrong_date = DateHelper().today.date() - timedelta(days=91) + url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[service]=*" + with self.assertRaises(ValidationError): + self.mocked_query_params(url, AWSCostView) + + def test_aws_out_of_range_over_date(self): + wrong_date = DateHelper().today.date() + timedelta(days=1) + url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[service]=*" + with self.assertRaises(ValidationError): + self.mocked_query_params(url, AWSCostView) + class AWSReportQueryLogicalAndTest(IamTestCase): """Tests the report queries.""" diff --git a/koku/api/report/test/azure/openshift/test_ocp_azure_query_handler.py b/koku/api/report/test/azure/openshift/test_ocp_azure_query_handler.py index 383b7ea501..7fa1936b8b 100644 --- a/koku/api/report/test/azure/openshift/test_ocp_azure_query_handler.py +++ b/koku/api/report/test/azure/openshift/test_ocp_azure_query_handler.py @@ -1066,6 +1066,18 @@ def test_ocp_azure_date_order_by_cost_desc(self): def test_ocp_azure_date_incorrect_date(self): wrong_date = "200BC" - url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[service]=*" # noqa: E501 + url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[service_name]=*" # noqa: E501 + with self.assertRaises(ValidationError): + self.mocked_query_params(url, OCPAzureCostView) + + def test_aws_out_of_range_under_date(self): + wrong_date = DateHelper().today.date() - timedelta(days=91) + url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[service_name]=*" + with self.assertRaises(ValidationError): + self.mocked_query_params(url, OCPAzureCostView) + + def test_aws_out_of_range_over_date(self): + wrong_date = DateHelper().today.date() + timedelta(days=1) + url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[service_name]=*" with self.assertRaises(ValidationError): self.mocked_query_params(url, OCPAzureCostView) diff --git a/koku/api/report/test/azure/tests_azure_query_handler.py b/koku/api/report/test/azure/tests_azure_query_handler.py index e4c721526e..f86cebc7bc 100644 --- a/koku/api/report/test/azure/tests_azure_query_handler.py +++ b/koku/api/report/test/azure/tests_azure_query_handler.py @@ -1262,6 +1262,18 @@ def test_azure_date_order_by_cost_desc(self): def test_azure_date_incorrect_date(self): wrong_date = "200BC" - url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[service]=*" # noqa: E501 + url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[service_name]=*" # noqa: E501 + with self.assertRaises(ValidationError): + self.mocked_query_params(url, AzureCostView) + + def test_aws_out_of_range_under_date(self): + wrong_date = DateHelper().today.date() - timedelta(days=91) + url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[service_name]=*" + with self.assertRaises(ValidationError): + self.mocked_query_params(url, AzureCostView) + + def test_aws_out_of_range_over_date(self): + wrong_date = DateHelper().today.date() + timedelta(days=1) + url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[service_name]=*" with self.assertRaises(ValidationError): self.mocked_query_params(url, AzureCostView) diff --git a/koku/api/report/test/gcp/tests_gcp_query_handler.py b/koku/api/report/test/gcp/tests_gcp_query_handler.py index bddf40c601..536f90b389 100644 --- a/koku/api/report/test/gcp/tests_gcp_query_handler.py +++ b/koku/api/report/test/gcp/tests_gcp_query_handler.py @@ -1121,3 +1121,15 @@ def test_gcp_date_incorrect_date(self): url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[service]=*" # noqa: E501 with self.assertRaises(ValidationError): self.mocked_query_params(url, GCPCostView) + + def test_aws_out_of_range_under_date(self): + wrong_date = DateHelper().today.date() - timedelta(days=91) + url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[service]=*" + with self.assertRaises(ValidationError): + self.mocked_query_params(url, GCPCostView) + + def test_aws_out_of_range_over_date(self): + wrong_date = DateHelper().today.date() + timedelta(days=1) + url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[service]=*" + with self.assertRaises(ValidationError): + self.mocked_query_params(url, GCPCostView) diff --git a/koku/api/report/test/ocp/test_ocp_query_handler.py b/koku/api/report/test/ocp/test_ocp_query_handler.py index 41a9cffa34..29bcb1d252 100644 --- a/koku/api/report/test/ocp/test_ocp_query_handler.py +++ b/koku/api/report/test/ocp/test_ocp_query_handler.py @@ -638,6 +638,18 @@ def test_ocp_date_order_by_cost_desc(self): def test_gcp_date_incorrect_date(self): wrong_date = "200BC" - url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[service]=*" # noqa: E501 + url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[project]=*" # noqa: E501 + with self.assertRaises(ValidationError): + self.mocked_query_params(url, OCPCostView) + + def test_aws_out_of_range_under_date(self): + wrong_date = DateHelper().today.date() - timedelta(days=91) + url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[project]=*" + with self.assertRaises(ValidationError): + self.mocked_query_params(url, OCPCostView) + + def test_aws_out_of_range_over_date(self): + wrong_date = DateHelper().today.date() + timedelta(days=1) + url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[project]=*" with self.assertRaises(ValidationError): self.mocked_query_params(url, OCPCostView)