From cb67842627439ac879649c645f4f9f37c698332e Mon Sep 17 00:00:00 2001 From: Viswas Haridas <37623357+JustARatherRidiculouslyLongUsername@users.noreply.github.com> Date: Tue, 22 Oct 2024 17:15:47 +0530 Subject: [PATCH] fix: travelperk not loading all invoice profiles (#219) * fix: travelperk not loading all invoice profiles integrate with the travelperk pagination API to fetch invoice profiles from all pages * refactor: move `get_all_generator` to `ApiBase` and rename variable * fix: generalise the `get_all_generator` method for all onject types and create an implementation in `InvoiceProfiles` * refactor: clarity --- apps/travelperk/connector.py | 9 +- connectors/travelperk/apis/api_base.py | 88 ++++++++++--------- .../travelperk/apis/invoice_profiles.py | 10 ++- 3 files changed, 61 insertions(+), 46 deletions(-) diff --git a/apps/travelperk/connector.py b/apps/travelperk/connector.py index 4d7ebff2..5d199865 100644 --- a/apps/travelperk/connector.py +++ b/apps/travelperk/connector.py @@ -65,8 +65,11 @@ def sync_invoice_profile(self): :return: Dict """ - response = self.connection.invoice_profiles.get_all() - for invoice_profile in response: + profiles_generator = self.connection.invoice_profiles.get_all_generator() + profiles = [] + + for invoice_profile in profiles_generator: + profiles.append(invoice_profile) country_name = invoice_profile['billing_information']['country_name'] if 'country_name' in invoice_profile['billing_information'] else None currency = invoice_profile['currency'] if 'currency' in invoice_profile else None TravelperkProfileMapping.objects.update_or_create( @@ -79,4 +82,4 @@ def sync_invoice_profile(self): } ) - return response + return profiles diff --git a/connectors/travelperk/apis/api_base.py b/connectors/travelperk/apis/api_base.py index 814b9856..ba7879a9 100644 --- a/connectors/travelperk/apis/api_base.py +++ b/connectors/travelperk/apis/api_base.py @@ -28,7 +28,27 @@ def set_server_url(self, server_url): """ self.__server_url = server_url - def _get_request(self, object_type: str, api_url: str) -> List[Dict] or Dict: + def _get_error(self, status_code: int, response_text: str): + """Get the error object from the response. + + Parameters: + status_code (int): The status code of the response. + response_text (str): The response text. + + Returns: + The error object. + """ + error_map = { + 400: BadRequestError('Something wrong with the request body', response_text), + 401: UnauthorizedClientError('Wrong client secret or/and refresh token', response_text), + 404: NotFoundError('Client ID doesn\'t exist', response_text), + 500: InternalServerError('Internal server error', response_text), + 409: BadRequestError('The webhook already exists', response_text) + } + + return error_map.get(status_code, TravelperkError('Error: {0}'.format(status_code), response_text)) + + def _get_request(self, object_type: str, api_url: str, params: dict = {}) -> List[Dict] or Dict: """Create a HTTP GET request. Parameters: @@ -46,27 +66,38 @@ def _get_request(self, object_type: str, api_url: str) -> List[Dict] or Dict: response = requests.get( '{0}{1}'.format(self.__server_url, api_url), - headers=api_headers + headers=api_headers, + params=params ) if response.status_code == 200: result = json.loads(response.text) return result[object_type] + else: + raise self._get_error(response.status_code, response.text) - elif response.status_code == 400: - raise BadRequestError('Something wrong with the request body', response.text) - - elif response.status_code == 401: - raise UnauthorizedClientError('Wrong client secret or/and refresh token', response.text) + def _get_all_generator(self, object_type: str, api_url: str): + """ + Creates a generator that contains all records of `object_type` across all pages + + Parameters: + object_type (str): The type of object to get + api_url (str): The url for the wanted API + params (dict): The parameters for the request - elif response.status_code == 404: - raise NotFoundError('Client ID doesn\'t exist', response.text) + Returns: + Generator with all objects of type `object_type` + """ - elif response.status_code == 500: - raise InternalServerError('Internal server error', response.text) + limit = 50 + params = {'limit': limit} + total_profiles = self._get_request('total', api_url, params=params) - else: - raise TravelperkError('Error: {0}'.format(response.status_code), response.text) + for offset in range(0, total_profiles, limit): + params['offset'] = offset + profiles = self._get_request(object_type, api_url, params=params) + for profile in profiles: + yield profile def _post_request(self, api_url: str, data: Dict) -> Dict: """Create a HTTP POST request. @@ -95,23 +126,8 @@ def _post_request(self, api_url: str, data: Dict) -> Dict: result = json.loads(response.text) return result - elif response.status_code == 400: - raise BadRequestError('Something wrong with the request body', response.text) - - elif response.status_code == 401: - raise UnauthorizedClientError('Wrong client secret or/and refresh token', response.text) - - elif response.status_code == 404: - raise NotFoundError('Client ID doesn\'t exist', response.text) - - elif response.status_code == 500: - raise InternalServerError('Internal server error', response.text) - - elif response.status_code == 409: - raise BadRequestError('The webhook already exists', response.text) - else: - raise TravelperkError('Error: {0}'.format(response.status_code), response.text) + raise self._get_error(response.status_code, response.text) def _delete_request(self, api_url: str) -> Dict: """Create a HTTP DELETE request. @@ -137,17 +153,5 @@ def _delete_request(self, api_url: str) -> Dict: if response.status_code == 200: return response.text - elif response.status_code == 400: - raise BadRequestError('Something wrong with the request body', response.text) - - elif response.status_code == 401: - raise UnauthorizedClientError('Wrong client secret or/and refresh token', response.text) - - elif response.status_code == 404: - raise NotFoundError('Client ID doesn\'t exist', response.text) - - elif response.status_code == 500: - raise InternalServerError('Internal server error', response.text) - else: - raise TravelperkError('Error: {0}'.format(response.status_code), response.text) \ No newline at end of file + raise self._get_error(response.status_code, response.text) diff --git a/connectors/travelperk/apis/invoice_profiles.py b/connectors/travelperk/apis/invoice_profiles.py index 573d73e5..a28d37c2 100644 --- a/connectors/travelperk/apis/invoice_profiles.py +++ b/connectors/travelperk/apis/invoice_profiles.py @@ -15,4 +15,12 @@ def get_all(self): Returns: List with dicts in Invoice Profile schema. """ - return self._get_request('profiles', InvoiceProfiles.GET_INVOICE_PROFILES) + return [*self.get_all_generator()] + + def get_all_generator(self): + """Create a generator with all the existing Invoice Profiles in the Organization. + + Returns: + Generator with dicts in Invoice Profile schema. + """ + return self._get_all_generator('profiles', self.GET_INVOICE_PROFILES) \ No newline at end of file