From 2f81aea0448f78c7e0f746cb6d79c5d78ee6c231 Mon Sep 17 00:00:00 2001 From: AndreiCautisanu <30831438+AndreiCautisanu@users.noreply.github.com> Date: Thu, 12 Dec 2024 16:23:55 +0200 Subject: [PATCH] basic experiment items tests (#865) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andrei Căutișanu --- .../page_objects/ExperimentItemsPage.py | 53 +++++++++++++ .../page_objects/ExperimentsPage.py | 4 + .../tests/Datasets/datasets_utils.py | 1 + .../tests/Experiments/conftest.py | 3 +- .../test_experiment_items_crud_operations.py | 78 +++++++++++++++++++ tests_end_to_end/tests/sdk_helpers.py | 13 +++- 6 files changed, 150 insertions(+), 2 deletions(-) create mode 100644 tests_end_to_end/page_objects/ExperimentItemsPage.py create mode 100644 tests_end_to_end/tests/Experiments/test_experiment_items_crud_operations.py diff --git a/tests_end_to_end/page_objects/ExperimentItemsPage.py b/tests_end_to_end/page_objects/ExperimentItemsPage.py new file mode 100644 index 0000000000..da6d17bfdd --- /dev/null +++ b/tests_end_to_end/page_objects/ExperimentItemsPage.py @@ -0,0 +1,53 @@ +from playwright.sync_api import Page, expect, Locator +import re + +class ExperimentItemsPage: + + def __init__(self, page: Page): + self.page = page + self.next_page_button_locator = self.page.locator("div:has(> button:nth-of-type(4))").locator('button:nth-of-type(3)') + + def get_pagination_button(self) -> Locator: + return self.page.get_by_role('button', name='Showing') + + def get_total_number_of_items_in_experiment(self): + pagination_button_text = self.get_pagination_button().inner_text() + match = re.search(r'of (\d+)', pagination_button_text) + if match: + return int(match.group(1)) + else: + return 0 + + def get_id_of_nth_experiment_item(self, n: int): + row = self.page.locator('tr').nth(n+1) + cell = row.locator('td').first + cell.hover() + cell.get_by_role('button').nth(1).click() + id = self.page.evaluate('navigator.clipboard.readText()') + return id + + + def get_all_item_ids_on_current_page(self): + ids = [] + rows = self.page.locator('tr').all() + for row_index, row in enumerate(rows[2:]): + item = {} + cells = row.locator('td').all() + cell = row.locator('td').first + cell.hover() + cell.get_by_role('button').nth(1).click() + id = self.page.evaluate('navigator.clipboard.readText()') + ids.append(id) + + return ids + + + def get_all_item_ids_in_experiment(self): + ids = [] + ids.extend(self.get_all_item_ids_on_current_page()) + while self.next_page_button_locator.is_visible() and self.next_page_button_locator.is_enabled(): + self.next_page_button_locator.click() + self.page.wait_for_timeout(500) + ids.extend(self.get_all_dataset_items_on_current_page()) + + return ids \ No newline at end of file diff --git a/tests_end_to_end/page_objects/ExperimentsPage.py b/tests_end_to_end/page_objects/ExperimentsPage.py index 1ac41f313c..64666263ee 100644 --- a/tests_end_to_end/page_objects/ExperimentsPage.py +++ b/tests_end_to_end/page_objects/ExperimentsPage.py @@ -12,6 +12,10 @@ def go_to_page(self): def search_experiment_by_name(self, exp_name: str): self.search_bar.click() self.search_bar.fill(exp_name) + + def click_first_experiment_that_matches_name(self, exp_name: str): + self.search_experiment_by_name(exp_name=exp_name) + self.page.get_by_role('link', name=exp_name).first.click() def check_experiment_exists_by_name(self, exp_name: str): self.search_experiment_by_name(exp_name) diff --git a/tests_end_to_end/tests/Datasets/datasets_utils.py b/tests_end_to_end/tests/Datasets/datasets_utils.py index 4d902dc659..08daea584e 100644 --- a/tests_end_to_end/tests/Datasets/datasets_utils.py +++ b/tests_end_to_end/tests/Datasets/datasets_utils.py @@ -48,6 +48,7 @@ def insert_dataset_items_ui(page: Page, dataset_name, items_list): for item in items_list: dataset_items_page.insert_dataset_item(json.dumps(item)) + time.sleep(0.2) def delete_one_dataset_item_sdk(client: opik.Opik, dataset_name): diff --git a/tests_end_to_end/tests/Experiments/conftest.py b/tests_end_to_end/tests/Experiments/conftest.py index c424b408c7..eebed161c6 100644 --- a/tests_end_to_end/tests/Experiments/conftest.py +++ b/tests_end_to_end/tests/Experiments/conftest.py @@ -30,7 +30,8 @@ def mock_experiment(client: Opik, create_delete_dataset_sdk, insert_dataset_item ) yield { 'id': eval.experiment_id, - 'name': experiment_name + 'name': experiment_name, + 'size': len(dataset.get_items()) } try: delete_experiment_by_id(eval.experiment_id) diff --git a/tests_end_to_end/tests/Experiments/test_experiment_items_crud_operations.py b/tests_end_to_end/tests/Experiments/test_experiment_items_crud_operations.py new file mode 100644 index 0000000000..a1937f44c7 --- /dev/null +++ b/tests_end_to_end/tests/Experiments/test_experiment_items_crud_operations.py @@ -0,0 +1,78 @@ +import pytest +from playwright.sync_api import Page, expect +from page_objects.DatasetsPage import DatasetsPage +from page_objects.ExperimentsPage import ExperimentsPage +from page_objects.ExperimentItemsPage import ExperimentItemsPage +from sdk_helpers import get_experiment_by_id, delete_experiment_by_id, delete_experiment_items_by_id, experiment_items_stream +import opik +import time +from collections import Counter + + +class TestExperimentItemsCrud: + + @pytest.mark.browser_context_args(permissions=['clipboard-read']) + def test_all_experiment_items_created(self, page: Page, mock_experiment): + """ + Creates an experiment with 10 experiment items, then checks that all items are visible in both UI and backend + 1. Create an experiment on a dataset with 10 items (mock_experiment fixture) + 2. Check the item counter on the UI displays the correct total (10 items) + 3. Check the 'trace_count' parameter of the experiment as returned via the v1/private/experiments/{id} endpoint + matches the size of the dataset (10 items) + 4. Check the list of IDs displayed in the UI (currently dataset item IDs) perfectly matches the list of dataset item IDs + as returned from the v1/private/experiments/items/stream endpoint (easy change to grab the items via the SDK if we ever add this) + """ + experiments_page = ExperimentsPage(page) + experiments_page.go_to_page() + experiments_page.click_first_experiment_that_matches_name(exp_name=mock_experiment['name']) + + experiment_items_page = ExperimentItemsPage(page) + items_on_page = experiment_items_page.get_total_number_of_items_in_experiment() + assert items_on_page == mock_experiment['size'] + + experiment_backend = get_experiment_by_id(mock_experiment['id']) + assert experiment_backend.trace_count == mock_experiment['size'] + + ids_on_backend = [item['dataset_item_id'] for item in experiment_items_stream(mock_experiment['name'])] + ids_on_frontend = experiment_items_page.get_all_item_ids_in_experiment() + + assert Counter(ids_on_backend) == Counter(ids_on_frontend) + + + @pytest.mark.browser_context_args(permissions=['clipboard-read']) + def test_delete_experiment_items(self, page: Page, mock_experiment): + """ + Deletes a single experiment item and checks that everything gets updated on both the UI and the backend + 1. Create an experiment on a dataset with 10 items (mock_experiment fixture) + 2. Grabbing an experiment ID from the v1/private/experiments/items/stream endpoint, send a delete request to delete + a single experiment item from the experiment + 3. Check the item counter in the UI is updated (to size(initial_experiment) - 1) + 4. Check the 'trace_count' parameter of the experiment as returned via the v1/private/experiments/{id} endpoint + is updated to the new size (as above) + 5. Check the list of IDs displayed in the UI (currently dataset item IDs) perfectly matches the list of dataset item IDs + as returned from the v1/private/experiments/items/stream endpoint (easy change to grab the items via the SDK if we ever add this) + """ + experiments_page = ExperimentsPage(page) + experiments_page.go_to_page() + experiments_page.click_first_experiment_that_matches_name(exp_name=mock_experiment['name']) + + id_to_delete = experiment_items_stream(exp_name=mock_experiment['name'], limit=1)[0]['id'] + delete_experiment_items_by_id(ids=[id_to_delete]) + + experiment_items_page = ExperimentItemsPage(page) + experiment_items_page.page.reload() + items_on_page = experiment_items_page.get_total_number_of_items_in_experiment() + assert items_on_page == mock_experiment['size'] - 1 + + experiment_sdk = get_experiment_by_id(mock_experiment['id']) + assert experiment_sdk.trace_count == mock_experiment['size'] - 1 + + ids_on_backend = [item['dataset_item_id'] for item in experiment_items_stream(mock_experiment['name'])] + ids_on_frontend = experiment_items_page.get_all_item_ids_in_experiment() + + assert Counter(ids_on_backend) == Counter(ids_on_frontend) + + + + + diff --git a/tests_end_to_end/tests/sdk_helpers.py b/tests_end_to_end/tests/sdk_helpers.py index 7cd483d789..239f2c3c56 100644 --- a/tests_end_to_end/tests/sdk_helpers.py +++ b/tests_end_to_end/tests/sdk_helpers.py @@ -143,4 +143,15 @@ def get_experiment_by_id(exp_id: str): def delete_experiment_by_id(exp_id: str): client = OpikApi() - client.experiments.delete_experiments_by_id(ids=[exp_id]) \ No newline at end of file + client.experiments.delete_experiments_by_id(ids=[exp_id]) + +def delete_experiment_items_by_id(ids: list[str]): + client = OpikApi() + client.experiments.delete_experiment_items(ids=ids) + +def experiment_items_stream(exp_name: str, limit: int = None): + client = OpikApi() + data = b''.join(client.experiments.stream_experiment_items(experiment_name=exp_name, request_options={'chunk_size': 100})) + lines = data.decode('utf-8').split('\r\n') + dict_list = [json.loads(line) for line in lines if line.strip()] + return dict_list \ No newline at end of file