Cost 1076 update (#3081)

Updating order by date for filter[limit] query parameter

koku/api/report/aws/openshift/query_handler.py

@@ -38,15 +38,13 @@ def execute_query(self):  # noqa: C901
             query_group_by = ["date"] + group_by_value
             query_order_by = ["-date"]
             query_order_by.extend(self.order)  # add implicit ordering
-
             annotations = self._mapper.report_type_map.get("annotations")
             query_data = query_data.values(*query_group_by).annotate(**annotations)
 
             if "account" in query_group_by:
                 query_data = query_data.annotate(
                     account_alias=Coalesce(F(self._mapper.provider_map.get("alias")), "usage_account_id")
                 )
-
             if self._limit and query_data:
                 query_data = self._group_by_ranks(query, query_data)
                 if not self.parameters.get("order_by"):
@@ -94,8 +92,12 @@ def check_if_valid_date_str(date_str):
             if order_date:
                 sort_term = self._get_group_by()[0]
                 query_order_by.pop(i)
-                date_filtered_query_data = query_data.filter(usage_start=order_date)
-                ordered_data = self.order_by(date_filtered_query_data, query_order_by)
+                filtered_query_data = []
+                for index in query_data:
+                    for key, value in index.items():
+                        if (key == "date") and (value == order_date):
+                            filtered_query_data.append(index)
+                ordered_data = self.order_by(filtered_query_data, query_order_by)
                 order_of_interest = []
                 for entry in ordered_data:
                     order_of_interest.append(entry.get(sort_term))

koku/api/report/aws/query_handler.py

@@ -578,6 +578,7 @@ def execute_individual_query(self, org_unit_applied=False):  # noqa: C901
             tag_results = None
             query = query_table.objects.filter(self.query_filter)
             query_data = query.annotate(**self.annotations)
+
             query_group_by = ["date"] + self._get_group_by()
             query_order_by = ["-date"]
             query_order_by.extend(self.order)  # add implicit ordering
@@ -628,20 +629,23 @@ def check_if_valid_date_str(date_str):
                     break
             # Remove the date order by as it is not actually used for ordering
             if order_date:
+
                 sort_term = self._get_group_by()[0]
                 query_order_by.pop(i)
-                date_filtered_query_data = query_data.filter(usage_start=order_date)
-                ordered_data = self.order_by(date_filtered_query_data, query_order_by)
+                filtered_query_data = []
+                for index in query_data:
+                    for key, value in index.items():
+                        if (key == "date") and (value == order_date):
+                            filtered_query_data.append(index)
+                ordered_data = self.order_by(filtered_query_data, query_order_by)
                 order_of_interest = []
                 for entry in ordered_data:
                     order_of_interest.append(entry.get(sort_term))
                 # write a special order by function that iterates through the
                 # rest of the days in query_data and puts them in the same order
-                # return_query_data = []
                 sorted_data = [item for x in order_of_interest for item in query_data if item.get(sort_term) == x]
                 query_data = self.order_by(sorted_data, ["-date"])
             else:
-                # &order_by[cost]=desc&order_by[date]=2021-08-02
                 query_data = self.order_by(query_data, query_order_by)
 
             # Fetch the data (returning list(dict))

koku/api/report/azure/openshift/query_handler.py

@@ -81,10 +81,8 @@ def execute_query(self):  # noqa: C901
             query_group_by = ["date"] + group_by_value
             query_order_by = ["-date"]
             query_order_by.extend(self.order)  # add implicit ordering
-
             annotations = self._mapper.report_type_map.get("annotations")
             query_data = query_data.values(*query_group_by).annotate(**annotations)
-
             if self._limit and query_data:
                 query_data = self._group_by_ranks(query, query_data)
                 if not self.parameters.get("order_by"):
@@ -122,8 +120,12 @@ def check_if_valid_date_str(date_str):
             if order_date:
                 sort_term = self._get_group_by()[0]
                 query_order_by.pop(i)
-                date_filtered_query_data = query_data.filter(usage_start=order_date)
-                ordered_data = self.order_by(date_filtered_query_data, query_order_by)
+                filtered_query_data = []
+                for index in query_data:
+                    for key, value in index.items():
+                        if (key == "date") and (value == order_date):
+                            filtered_query_data.append(index)
+                ordered_data = self.order_by(filtered_query_data, query_order_by)
                 order_of_interest = []
                 for entry in ordered_data:
                     order_of_interest.append(entry.get(sort_term))
@@ -133,7 +135,6 @@ def check_if_valid_date_str(date_str):
                 sorted_data = [item for x in order_of_interest for item in query_data if item.get(sort_term) == x]
                 query_data = self.order_by(sorted_data, ["-date"])
             else:
-                # &order_by[cost]=desc&order_by[date]=2021-08-02
                 query_data = self.order_by(query_data, query_order_by)
 
             cost_units_value = self._mapper.report_type_map.get("cost_units_fallback", "USD")

koku/api/report/azure/query_handler.py

@@ -141,7 +141,6 @@ def execute_query(self):  # noqa: C901
             query_group_by = ["date"] + self._get_group_by()
             query_order_by = ["-date"]
             query_order_by.extend(self.order)  # add implicit ordering
-
             annotations = self._mapper.report_type_map.get("annotations")
             query_data = query_data.values(*query_group_by).annotate(**annotations)
             query_sum = self._build_sum(query)
@@ -178,8 +177,12 @@ def check_if_valid_date_str(date_str):
             if order_date:
                 sort_term = self._get_group_by()[0]
                 query_order_by.pop(i)
-                date_filtered_query_data = query_data.filter(usage_start=order_date)
-                ordered_data = self.order_by(date_filtered_query_data, query_order_by)
+                filtered_query_data = []
+                for index in query_data:
+                    for key, value in index.items():
+                        if (key == "date") and (value == order_date):
+                            filtered_query_data.append(index)
+                ordered_data = self.order_by(filtered_query_data, query_order_by)
                 order_of_interest = []
                 for entry in ordered_data:
                     order_of_interest.append(entry.get(sort_term))

koku/api/report/gcp/query_handler.py

@@ -152,6 +152,7 @@ def execute_query(self):  # noqa: C901
             query = self.query_table.objects.filter(self.query_filter)
             query = query.filter(invoice_month__in=self.invoice_months)
             query_data = query.annotate(**self.annotations)
+
             query_group_by = ["date"] + self._get_group_by()
             query_order_by = ["-date"]
             query_order_by.extend(self.order)  # add implicit ordering
@@ -195,8 +196,12 @@ def check_if_valid_date_str(date_str):
             if order_date:
                 sort_term = self._get_group_by()[0]
                 query_order_by.pop(i)
-                date_filtered_query_data = query_data.filter(usage_start=order_date)
-                ordered_data = self.order_by(date_filtered_query_data, query_order_by)
+                filtered_query_data = []
+                for index in query_data:
+                    for key, value in index.items():
+                        if (key == "date") and (value == order_date):
+                            filtered_query_data.append(index)
+                ordered_data = self.order_by(filtered_query_data, query_order_by)
                 order_of_interest = []
                 for entry in ordered_data:
                     order_of_interest.append(entry.get(sort_term))

koku/api/report/ocp/query_handler.py

@@ -172,8 +172,12 @@ def check_if_valid_date_str(date_str):
             if order_date:
                 sort_term = self._get_group_by()[0]
                 query_order_by.pop(i)
-                date_filtered_query_data = query_data.filter(usage_start=order_date)
-                ordered_data = self.order_by(date_filtered_query_data, query_order_by)
+                filtered_query_data = []
+                for index in query_data:
+                    for key, value in index.items():
+                        if (key == "date") and (value == order_date):
+                            filtered_query_data.append(index)
+                ordered_data = self.order_by(filtered_query_data, query_order_by)
                 order_of_interest = []
                 for entry in ordered_data:
                     order_of_interest.append(entry.get(sort_term))
@@ -183,7 +187,6 @@ def check_if_valid_date_str(date_str):
                 sorted_data = [item for x in order_of_interest for item in query_data if item.get(sort_term) == x]
                 query_data = self.order_by(sorted_data, ["-date"])
             else:
-                # &order_by[cost]=desc&order_by[date]=2021-08-02
                 query_data = self.order_by(query_data, query_order_by)
 
             if is_csv_output:

koku/api/report/serializers.py

@@ -395,7 +395,17 @@ def validate_order_by(self, value):  # noqa: C901
                     continue
                 # sepcial case: we order by date, but we group by an allowed param.
                 if key == "date" and group_keys:
-                    continue
+                    # Checks to make sure the orderby date is allowed
+                    dh = DateHelper()
+                    if (
+                        value.get("date") >= materialized_view_month_start(dh).date()
+                        and value.get("date") <= dh.today.date()
+                    ):
+                        continue
+                    error = "Order by date must be from {} to {}".format(
+                        materialized_view_month_start(dh).date(), dh.today.date()
+                    )
+                    raise serializers.ValidationError(error)
 
             error[key] = _(f'Order-by "{key}" requires matching Group-by.')
             raise serializers.ValidationError(error)

koku/api/report/test/aws/openshift/test_ocp_aws_query_handler.py

@@ -452,3 +452,15 @@ def test_ocp_aws_date_incorrect_date(self):
         url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[service]=*"  # noqa: E501
         with self.assertRaises(ValidationError):
             self.mocked_query_params(url, OCPAWSCostView)
+
+    def test_aws_out_of_range_under_date(self):
+        wrong_date = DateHelper().today.date() - timedelta(days=91)
+        url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[service]=*"
+        with self.assertRaises(ValidationError):
+            self.mocked_query_params(url, OCPAWSCostView)
+
+    def test_aws_out_of_range_over_date(self):
+        wrong_date = DateHelper().today.date() + timedelta(days=1)
+        url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[service]=*"
+        with self.assertRaises(ValidationError):
+            self.mocked_query_params(url, OCPAWSCostView)

koku/api/report/test/aws/tests_queries.py

@@ -1892,6 +1892,18 @@ def test_aws_date_incorrect_date(self):
         with self.assertRaises(ValidationError):
             self.mocked_query_params(url, AWSCostView)
 
+    def test_aws_out_of_range_under_date(self):
+        wrong_date = DateHelper().today.date() - timedelta(days=91)
+        url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[service]=*"
+        with self.assertRaises(ValidationError):
+            self.mocked_query_params(url, AWSCostView)
+
+    def test_aws_out_of_range_over_date(self):
+        wrong_date = DateHelper().today.date() + timedelta(days=1)
+        url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[service]=*"
+        with self.assertRaises(ValidationError):
+            self.mocked_query_params(url, AWSCostView)
+
 
 class AWSReportQueryLogicalAndTest(IamTestCase):
     """Tests the report queries."""

koku/api/report/test/azure/openshift/test_ocp_azure_query_handler.py

@@ -1066,6 +1066,18 @@ def test_ocp_azure_date_order_by_cost_desc(self):
 
     def test_ocp_azure_date_incorrect_date(self):
         wrong_date = "200BC"
-        url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[service]=*"  # noqa: E501
+        url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[service_name]=*"  # noqa: E501
+        with self.assertRaises(ValidationError):
+            self.mocked_query_params(url, OCPAzureCostView)
+
+    def test_aws_out_of_range_under_date(self):
+        wrong_date = DateHelper().today.date() - timedelta(days=91)
+        url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[service_name]=*"
+        with self.assertRaises(ValidationError):
+            self.mocked_query_params(url, OCPAzureCostView)
+
+    def test_aws_out_of_range_over_date(self):
+        wrong_date = DateHelper().today.date() + timedelta(days=1)
+        url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[service_name]=*"
         with self.assertRaises(ValidationError):
             self.mocked_query_params(url, OCPAzureCostView)

koku/api/report/test/azure/tests_azure_query_handler.py

@@ -1262,6 +1262,18 @@ def test_azure_date_order_by_cost_desc(self):
 
     def test_azure_date_incorrect_date(self):
         wrong_date = "200BC"
-        url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[service]=*"  # noqa: E501
+        url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[service_name]=*"  # noqa: E501
+        with self.assertRaises(ValidationError):
+            self.mocked_query_params(url, AzureCostView)
+
+    def test_aws_out_of_range_under_date(self):
+        wrong_date = DateHelper().today.date() - timedelta(days=91)
+        url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[service_name]=*"
+        with self.assertRaises(ValidationError):
+            self.mocked_query_params(url, AzureCostView)
+
+    def test_aws_out_of_range_over_date(self):
+        wrong_date = DateHelper().today.date() + timedelta(days=1)
+        url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[service_name]=*"
         with self.assertRaises(ValidationError):
             self.mocked_query_params(url, AzureCostView)

koku/api/report/test/gcp/tests_gcp_query_handler.py

@@ -1121,3 +1121,15 @@ def test_gcp_date_incorrect_date(self):
         url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[service]=*"  # noqa: E501
         with self.assertRaises(ValidationError):
             self.mocked_query_params(url, GCPCostView)
+
+    def test_aws_out_of_range_under_date(self):
+        wrong_date = DateHelper().today.date() - timedelta(days=91)
+        url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[service]=*"
+        with self.assertRaises(ValidationError):
+            self.mocked_query_params(url, GCPCostView)
+
+    def test_aws_out_of_range_over_date(self):
+        wrong_date = DateHelper().today.date() + timedelta(days=1)
+        url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[service]=*"
+        with self.assertRaises(ValidationError):
+            self.mocked_query_params(url, GCPCostView)

koku/api/report/test/ocp/test_ocp_query_handler.py

@@ -638,6 +638,18 @@ def test_ocp_date_order_by_cost_desc(self):
 
     def test_gcp_date_incorrect_date(self):
         wrong_date = "200BC"
-        url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[service]=*"  # noqa: E501
+        url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[project]=*"  # noqa: E501
+        with self.assertRaises(ValidationError):
+            self.mocked_query_params(url, OCPCostView)
+
+    def test_aws_out_of_range_under_date(self):
+        wrong_date = DateHelper().today.date() - timedelta(days=91)
+        url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[project]=*"
+        with self.assertRaises(ValidationError):
+            self.mocked_query_params(url, OCPCostView)
+
+    def test_aws_out_of_range_over_date(self):
+        wrong_date = DateHelper().today.date() + timedelta(days=1)
+        url = f"?order_by[cost]=desc&order_by[date]={wrong_date}&group_by[project]=*"
         with self.assertRaises(ValidationError):
             self.mocked_query_params(url, OCPCostView)