From 6ec030767c32a419205fcb479407611ee5900e3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20C=C4=83uti=C8=99anu?= Date: Wed, 20 Nov 2024 22:02:44 +0200 Subject: [PATCH 1/5] datasets tests, GHA merging, minor fix-ups --- .github/workflows/end2end_suites.yml | 19 ++- .github/workflows/sanity.yml | 63 -------- tests_end_to_end/page_objects/DatasetsPage.py | 23 ++- tests_end_to_end/page_objects/TracesPage.py | 10 +- .../Datasets/test_datasets_crud_operations.py | 144 ++++++++++++++++++ .../Projects/test_projects_crud_operations.py | 1 + tests_end_to_end/tests/Traces/conftest.py | 6 - .../tests/application_sanity/test_sanity.py | 2 +- tests_end_to_end/tests/conftest.py | 53 ++++++- tests_end_to_end/tests/sdk_helpers.py | 31 +++- 10 files changed, 271 insertions(+), 81 deletions(-) delete mode 100644 .github/workflows/sanity.yml create mode 100644 tests_end_to_end/tests/Datasets/test_datasets_crud_operations.py diff --git a/.github/workflows/end2end_suites.yml b/.github/workflows/end2end_suites.yml index 921ed304ff..4b5e3f7515 100644 --- a/.github/workflows/end2end_suites.yml +++ b/.github/workflows/end2end_suites.yml @@ -7,10 +7,13 @@ on: type: choice description: 'Choose which test suite to run' required: true - default: 'projects-CRUD' + default: 'sanity' options: - - projects-CRUD - - traces-CRUD + - all_features + - sanity + - projects + - traces + - datasets jobs: run_suite: @@ -64,10 +67,16 @@ jobs: cd ${{ github.workspace }}/tests_end_to_end export PYTHONPATH='.' - if [ "$SUITE" == "projects-CRUD" ]; then + if [ "$SUITE" == "projects" ]; then pytest -s tests/Projects/test_projects_crud_operations.py --browser chromium --base-url http://localhost:5173 --setup-show - elif [ "$SUITE" == "traces-CRUD" ]; then + elif [ "$SUITE" == "traces" ]; then pytest -s tests/Traces/test_traces_crud_operations.py --browser chromium --base-url http://localhost:5173 --setup-show + elif [ "$SUITE" == "datasets" ]; then + pytest -s tests/Datasets/test_datasets_crud_operations.py --browser chromium --base-url http://localhost:5173 --setup-show + elif [ "$SUITE" == "sanity" ]; then + pytest -s tests/application_sanity/test_sanity.py --browser chromium --base-url http://localhost:5173 --setup-show + elif [ "$SUITE" == "all_features" ]; then + pytest -s tests --browser chromium --base-url http://localhost:5173 --setup-show --ignore=tests/application_sanity fi - name: Stop Opik server diff --git a/.github/workflows/sanity.yml b/.github/workflows/sanity.yml deleted file mode 100644 index c44a21796b..0000000000 --- a/.github/workflows/sanity.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: E2E Sanity tests - -on: - workflow_dispatch: - -jobs: - e2e_sanity: - runs-on: ubuntu-20.04 - - steps: - - name: Checkout repo - uses: actions/checkout@v4 - with: - ref: ${{ github.ref }} - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: 3.12 - - - name: Install Opik - run: pip install ${{ github.workspace }}/sdks/python - - - name: Install Test Dependencies - run: | - pip install -r ${{ github.workspace }}/tests_end_to_end/test_requirements.txt - playwright install - - - name: Install Opik - env: - OPIK_USAGE_REPORT_ENABLED: false - run: | - cd ${{ github.workspace }}/deployment/docker-compose - docker compose up -d --build - - - name: Check Docker pods are up - run: | - chmod +x ./tests_end_to_end/installer_utils/check_docker_compose_pods.sh - ./tests_end_to_end/installer_utils/check_docker_compose_pods.sh - shell: bash - - - name: Check backend health - run: | - chmod +x ./tests_end_to_end/installer_utils/check_backend.sh - ./tests_end_to_end/installer_utils/check_backend.sh - shell: bash - - - name: Check app is up via the UI - run: | - pytest -v -s ${{ github.workspace }}/tests_end_to_end/installer_utils/test_app_status.py - - - name: Run sanity suite - run: | - cd ${{ github.workspace }}/tests_end_to_end - export PYTHONPATH='.' - pytest -s tests/application_sanity/test_sanity.py --browser chromium --base-url http://localhost:5173 --setup-show - - - name: Stop Opik server - if: always() - run: | - cd ${{ github.workspace }}/deployment/docker-compose - docker compose down - cd - diff --git a/tests_end_to_end/page_objects/DatasetsPage.py b/tests_end_to_end/page_objects/DatasetsPage.py index d0509efba4..a3a245439d 100644 --- a/tests_end_to_end/page_objects/DatasetsPage.py +++ b/tests_end_to_end/page_objects/DatasetsPage.py @@ -8,10 +8,29 @@ def __init__(self, page: Page): def go_to_page(self): self.page.goto(self.url) + def create_dataset_by_name(self, dataset_name: str): + self.page.get_by_role('button', name='Create new dataset').click() + self.page.get_by_placeholder('Dataset name').fill(dataset_name) + self.page.get_by_role('button', name='Create dataset').click() + def select_database_by_name(self, name): self.page.get_by_text(name, exact=True).first.click() - def check_dataset_exists_by_name(self, dataset_name): - expect(self.page.get_by_text(dataset_name)).to_be_visible() + def search_dataset(self, dataset_name): + self.page.get_by_test_id("search-input").click() + self.page.get_by_test_id("search-input").fill(dataset_name) + def check_dataset_exists_on_page_by_name(self, dataset_name): + expect(self.page.get_by_text(dataset_name).first).to_be_visible() + + def check_dataset_not_exists_on_page_by_name(self, dataset_name): + expect(self.page.get_by_text(dataset_name).first).not_to_be_visible() + + def delete_dataset_by_name(self, dataset_name): + self.search_dataset(dataset_name) + row = self.page.get_by_role('row').filter(has_text=dataset_name).filter(has=self.page.get_by_role('cell', name=dataset_name, exact=True)) + row.get_by_role("button").click() + self.page.get_by_role("menuitem", name="Delete").click() + self.page.get_by_role("button", name="Delete dataset").click() + \ No newline at end of file diff --git a/tests_end_to_end/page_objects/TracesPage.py b/tests_end_to_end/page_objects/TracesPage.py index 1fecf384cf..a1fff4a866 100644 --- a/tests_end_to_end/page_objects/TracesPage.py +++ b/tests_end_to_end/page_objects/TracesPage.py @@ -83,4 +83,12 @@ def delete_all_traces_that_match_name_contains_filter(self, name: str): pagination_button = self.get_pagination_button() expect(pagination_button).not_to_have_text(f'Showing 1-10 of {total_traces}') - total_traces = self.get_total_number_of_traces_in_project() \ No newline at end of file + total_traces = self.get_total_number_of_traces_in_project() + + + def add_all_traces_to_new_dataset(self, dataset_name: str): + self.page.get_by_label("Select all").click() + self.page.get_by_role("button", name="Add to dataset").click() + self.page.get_by_role("button", name="Create new dataset").click() + self.page.get_by_placeholder("Dataset name").fill(dataset_name) + self.page.get_by_role("button", name="Create dataset").click() \ No newline at end of file diff --git a/tests_end_to_end/tests/Datasets/test_datasets_crud_operations.py b/tests_end_to_end/tests/Datasets/test_datasets_crud_operations.py new file mode 100644 index 0000000000..40d45d5eb0 --- /dev/null +++ b/tests_end_to_end/tests/Datasets/test_datasets_crud_operations.py @@ -0,0 +1,144 @@ +import pytest +from playwright.sync_api import Page, expect +from page_objects.DatasetsPage import DatasetsPage +from page_objects.ProjectsPage import ProjectsPage +from page_objects.TracesPage import TracesPage +from sdk_helpers import delete_dataset_by_name_if_exists, update_dataset_name, get_dataset_by_name +import opik +import time + + +class TestDatasetsCrud: + + def test_create_dataset_ui_datasets_page(self, page: Page): + datasets_page = DatasetsPage(page) + datasets_page.go_to_page() + dataset_name = 'automated_tests_dataset' + try: + datasets_page.create_dataset_by_name(dataset_name=dataset_name) + datasets_page.check_dataset_exists_on_page_by_name(dataset_name=dataset_name) + except Exception as e: + print(f'error during dataset creation: {e}') + raise + finally: + delete_dataset_by_name_if_exists(dataset_name=dataset_name) + + + def test_create_dataset_ui_add_traces_to_new_dataset(self, page: Page, create_delete_project_sdk, create_10_test_traces): + dataset_name = 'automated_tests_dataset' + proj_name = create_delete_project_sdk + projects_page = ProjectsPage(page) + projects_page.go_to_page() + projects_page.click_project(project_name=proj_name) + + traces_page = TracesPage(page) + traces_page.add_all_traces_to_new_dataset(dataset_name=dataset_name) + + try: + datasets_page = DatasetsPage(page) + datasets_page.go_to_page() + datasets_page.check_dataset_exists_on_page_by_name(dataset_name=dataset_name) + except Exception as e: + print(f'error: dataset not created: {e}') + raise + finally: + delete_dataset_by_name_if_exists(dataset_name=dataset_name) + + + def test_create_dataset_sdk_client(self, client: opik.Opik): + dataset_name = 'automated_tests_dataset' + try: + client.create_dataset(name=dataset_name) + time.sleep(0.2) + assert client.get_dataset(name=dataset_name) is not None + except Exception as e: + print(f'error during dataset creation: {e}') + raise + finally: + delete_dataset_by_name_if_exists(dataset_name=dataset_name) + + + @pytest.mark.parametrize('dataset_fixture', ['create_delete_dataset_ui', 'create_delete_dataset_sdk']) + def test_dataset_visibility(self, request, page: Page, client: opik.Opik, dataset_fixture): + dataset_name = request.getfixturevalue(dataset_fixture) + time.sleep(0.5) + + datasets_page = DatasetsPage(page) + datasets_page.go_to_page() + datasets_page.check_dataset_exists_on_page_by_name(dataset_name) + + assert client.get_dataset(dataset_name) is not None + + + @pytest.mark.parametrize('dataset_fixture', ['create_dataset_sdk_no_cleanup', 'create_dataset_ui_no_cleanup']) + def test_dataset_name_update(self, request, page: Page, client: opik.Opik, dataset_fixture): + dataset_name = request.getfixturevalue(dataset_fixture) + time.sleep(0.5) + new_name = 'updated_test_dataset_name' + + name_updated = False + try: + dataset_id = update_dataset_name(name=dataset_name, new_name=new_name) + name_updated = True + + dataset_new_name = get_dataset_by_name(dataset_name=new_name) + + dataset_id_updated_name = dataset_new_name['id'] + assert dataset_id_updated_name == dataset_id + + datasets_page = DatasetsPage(page) + datasets_page.go_to_page() + datasets_page.check_dataset_exists_on_page_by_name(dataset_name=new_name) + datasets_page.check_dataset_not_exists_on_page_by_name(dataset_name=dataset_name) + + except Exception as e: + print(f'Error occured during update of project name: {e}') + raise + + finally: + if name_updated: + delete_dataset_by_name_if_exists(new_name) + else: + delete_dataset_by_name_if_exists(dataset_name) + + + @pytest.mark.parametrize('dataset_fixture', ['create_dataset_sdk_no_cleanup', 'create_dataset_ui_no_cleanup']) + def test_dataset_deletion_in_sdk(self, request, page: Page, client: opik.Opik, dataset_fixture): + dataset_name = request.getfixturevalue(dataset_fixture) + time.sleep(0.5) + client.delete_dataset(name=dataset_name) + dataset_page = DatasetsPage(page) + dataset_page.go_to_page() + dataset_page.check_dataset_not_exists_on_page_by_name(dataset_name=dataset_name) + try: + _ = client.get_dataset(dataset_name) + assert False, f'datasets {dataset_name} somehow still exists after deletion' + except Exception as e: + if '404' in str(e) or 'not found' in str(e).lower(): + pass + else: + raise + + + @pytest.mark.parametrize('dataset_fixture', ['create_dataset_sdk_no_cleanup', 'create_dataset_ui_no_cleanup']) + def test_dataset_deletion_in_ui(self, request, page: Page, client: opik.Opik, dataset_fixture): + dataset_name = request.getfixturevalue(dataset_fixture) + time.sleep(0.5) + datasets_page = DatasetsPage(page) + datasets_page.go_to_page() + datasets_page.delete_dataset_by_name(dataset_name=dataset_name) + time.sleep(1) + + try: + _ = client.get_dataset(dataset_name) + assert False, f'datasets {dataset_name} somehow still exists after deletion' + except Exception as e: + if '404' in str(e) or 'not found' in str(e).lower(): + pass + else: + raise + + dataset_page = DatasetsPage(page) + dataset_page.go_to_page() + dataset_page.check_dataset_not_exists_on_page_by_name(dataset_name=dataset_name) + diff --git a/tests_end_to_end/tests/Projects/test_projects_crud_operations.py b/tests_end_to_end/tests/Projects/test_projects_crud_operations.py index c5334981b5..379f19b22e 100644 --- a/tests_end_to_end/tests/Projects/test_projects_crud_operations.py +++ b/tests_end_to_end/tests/Projects/test_projects_crud_operations.py @@ -112,6 +112,7 @@ def test_project_name_update(self, request, page: Page, project_fixture): except Exception as e: print(f'Error occured during update of project name: {e}') + raise finally: if name_updated: diff --git a/tests_end_to_end/tests/Traces/conftest.py b/tests_end_to_end/tests/Traces/conftest.py index 3b30117867..bc9834e5ca 100644 --- a/tests_end_to_end/tests/Traces/conftest.py +++ b/tests_end_to_end/tests/Traces/conftest.py @@ -44,9 +44,3 @@ def log_x_traces_with_one_span_via_client(client, traces_number): def create_traces(request, traces_number): create = request.getfixturevalue(request.param) yield 0 - - -@pytest.fixture(scope='function') -def create_delete_traces(traces_number): - create_traces_sdk(PREFIX, PROJECT_NAME, traces_number) - yield diff --git a/tests_end_to_end/tests/application_sanity/test_sanity.py b/tests_end_to_end/tests/application_sanity/test_sanity.py index 4558b29165..c8520f3835 100644 --- a/tests_end_to_end/tests/application_sanity/test_sanity.py +++ b/tests_end_to_end/tests/application_sanity/test_sanity.py @@ -105,7 +105,7 @@ def test_dataset_name(datasets_page, config, dataset): ''' Checks that the dataset created via the fixture as defined in sanity_config.yaml is present on the datasets page ''' - datasets_page.check_dataset_exists_by_name(config['dataset']['name']) + datasets_page.check_dataset_exists_on_page_by_name(config['dataset']['name']) def test_dataset_items(page: Page, datasets_page, config, dataset_content): diff --git a/tests_end_to_end/tests/conftest.py b/tests_end_to_end/tests/conftest.py index 0bbe36e06d..569a44a354 100644 --- a/tests_end_to_end/tests/conftest.py +++ b/tests_end_to_end/tests/conftest.py @@ -13,7 +13,7 @@ from page_objects.TracesPage import TracesPage from page_objects.DatasetsPage import DatasetsPage from page_objects.ExperimentsPage import ExperimentsPage -from tests.sdk_helpers import create_project_sdk, delete_project_by_name_sdk +from tests.sdk_helpers import create_project_sdk, delete_project_by_name_sdk, wait_for_number_of_traces_to_be_visible, delete_dataset_by_name_if_exists @pytest.fixture(scope='session', autouse=True) @@ -100,4 +100,53 @@ def create_delete_project_ui(page: Page): projects_page.create_new_project(project_name=proj_name) yield proj_name - delete_project_by_name_sdk(name=proj_name) \ No newline at end of file + delete_project_by_name_sdk(name=proj_name) + + +@pytest.fixture(scope='function') +def create_delete_dataset_sdk(client: opik.Opik): + dataset_name = 'automated_tests_dataset' + client.create_dataset(name=dataset_name) + yield dataset_name + client.delete_dataset(name=dataset_name) + + +@pytest.fixture(scope='function') +def create_delete_dataset_ui(page: Page, client: opik.Opik): + dataset_name = 'automated_tests_dataset' + datasets_page = DatasetsPage(page) + datasets_page.go_to_page() + datasets_page.create_dataset_by_name(dataset_name=dataset_name) + + yield dataset_name + client.delete_dataset(name=dataset_name) + + +@pytest.fixture(scope='function') +def create_dataset_sdk_no_cleanup(client: opik.Opik): + dataset_name = 'automated_tests_dataset' + client.create_dataset(name=dataset_name) + yield dataset_name + + +@pytest.fixture(scope='function') +def create_dataset_ui_no_cleanup(page: Page): + dataset_name = 'automated_tests_dataset' + datasets_page = DatasetsPage(page) + datasets_page.go_to_page() + datasets_page.create_dataset_by_name(dataset_name=dataset_name) + yield dataset_name + + +@pytest.fixture +def create_10_test_traces(page: Page, client, create_delete_project_sdk): + proj_name = create_delete_project_sdk + for i in range(10): + client_trace = client.trace( + name=f'trace{i}', + project_name=proj_name, + input={'input': 'test input'}, + output={'output': 'test output'}, + ) + wait_for_number_of_traces_to_be_visible(project_name=proj_name, number_of_traces=10) + yield \ No newline at end of file diff --git a/tests_end_to_end/tests/sdk_helpers.py b/tests_end_to_end/tests/sdk_helpers.py index fa97e63e15..d45d0e5e33 100644 --- a/tests_end_to_end/tests/sdk_helpers.py +++ b/tests_end_to_end/tests/sdk_helpers.py @@ -88,6 +88,7 @@ def wait_for_number_of_traces_to_be_visible(project_name, number_of_traces, time raise TimeoutError(f'could not get {number_of_traces} traces of project {project_name} via API within {timeout} seconds') + def get_traces_of_project_sdk(project_name: str, size: int): client = OpikApi() traces = client.traces.get_traces_by_project(project_name=project_name, size=size) @@ -101,4 +102,32 @@ def delete_list_of_traces_sdk(ids: list[str]): def update_trace_by_id(id: str): client = OpikApi() - client.traces.update_trace(id=id, ) \ No newline at end of file + client.traces.update_trace(id=id, ) + + +def get_dataset_by_name(dataset_name: str): + client = OpikApi() + dataset = client.datasets.get_dataset_by_identifier(dataset_name=dataset_name) + return dataset.dict() + + +def update_dataset_name(name: str, new_name: str): + client = OpikApi() + dataset = get_dataset_by_name(dataset_name=name) + dataset_id = dataset['id'] + + dataset = client.datasets.update_dataset(id=dataset_id, name=new_name) + + return dataset_id + + +def delete_dataset_by_name_if_exists(dataset_name: str): + client = OpikApi() + dataset = None + try: + dataset = get_dataset_by_name(dataset_name) + except Exception as e: + print(f'Trying to delete dataset {dataset_name}, but it does not exist') + finally: + if dataset: + client.datasets.delete_dataset_by_name(dataset_name=dataset_name) From 9ed6a05e43bda63b9a33d12500fa6d71973dbbad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20C=C4=83uti=C8=99anu?= Date: Thu, 21 Nov 2024 12:50:46 +0200 Subject: [PATCH 2/5] descriptions --- .../Datasets/test_datasets_crud_operations.py | 47 ++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/tests_end_to_end/tests/Datasets/test_datasets_crud_operations.py b/tests_end_to_end/tests/Datasets/test_datasets_crud_operations.py index 40d45d5eb0..6fe1b8af27 100644 --- a/tests_end_to_end/tests/Datasets/test_datasets_crud_operations.py +++ b/tests_end_to_end/tests/Datasets/test_datasets_crud_operations.py @@ -11,6 +11,12 @@ class TestDatasetsCrud: def test_create_dataset_ui_datasets_page(self, page: Page): + """ + Basic test to check dataset creation via UI. Uses the UI after creation to check the dataset exists + 1. Create dataset via UI from the datasets page + 2. Check the dataset exists in the dataset table + 3. If no errors raised, test passes + """ datasets_page = DatasetsPage(page) datasets_page.go_to_page() dataset_name = 'automated_tests_dataset' @@ -25,6 +31,13 @@ def test_create_dataset_ui_datasets_page(self, page: Page): def test_create_dataset_ui_add_traces_to_new_dataset(self, page: Page, create_delete_project_sdk, create_10_test_traces): + """ + Basic test to check dataset creation via "add to new dataset" functionality in the traces page. Uses the UI after creation to check the project exists + 1. Create a project with some traces + 2. Via the UI, select the traces and add them to a new dataset + 3. Switch to the datasets page, check the dataset exists in the dataset table + 4. If no errors raised and dataset exists, test passes + """ dataset_name = 'automated_tests_dataset' proj_name = create_delete_project_sdk projects_page = ProjectsPage(page) @@ -46,6 +59,12 @@ def test_create_dataset_ui_add_traces_to_new_dataset(self, page: Page, create_de def test_create_dataset_sdk_client(self, client: opik.Opik): + """ + Basic test to check dataset creation via SDK. Uses the SDK to fetch the created dataset to check it exists + 1. Create dataset via SDK Opik client + 2. Get the project via SDK OpikAPI client + 3. If dataset creation fails, client.get_dataset will throw an error and the test will fail. + """ dataset_name = 'automated_tests_dataset' try: client.create_dataset(name=dataset_name) @@ -60,6 +79,12 @@ def test_create_dataset_sdk_client(self, client: opik.Opik): @pytest.mark.parametrize('dataset_fixture', ['create_delete_dataset_ui', 'create_delete_dataset_sdk']) def test_dataset_visibility(self, request, page: Page, client: opik.Opik, dataset_fixture): + """ + Checks a created dataset is visible via both the UI and SDK. Test split in 2: checks on datasets created on both UI and SDK + 1. Create a dataset via the UI/the SDK (2 "instances" of the test created for each one) + 2. Fetch the dataset by name using the SDK Opik client and check the dataset exists in the datasets table in the UI + 3. Check that the correct dataset is returned in the SDK and that the name is correct in the UI + """ dataset_name = request.getfixturevalue(dataset_fixture) time.sleep(0.5) @@ -67,11 +92,19 @@ def test_dataset_visibility(self, request, page: Page, client: opik.Opik, datase datasets_page.go_to_page() datasets_page.check_dataset_exists_on_page_by_name(dataset_name) - assert client.get_dataset(dataset_name) is not None + dataset_sdk = client.get_dataset(dataset_name) + assert dataset_sdk.name == dataset_name @pytest.mark.parametrize('dataset_fixture', ['create_dataset_sdk_no_cleanup', 'create_dataset_ui_no_cleanup']) def test_dataset_name_update(self, request, page: Page, client: opik.Opik, dataset_fixture): + """ + Checks using the SDK update method on a dataset. Test split into 2: checks on dataset created on both UI and SDK + 1. Create a dataset via the UI/the SDK (2 "instances" of the test created for each one) + 2. Send a request via the SDK OpikApi client to update the dataset's name + 3. Check on both the SDK and the UI that the dataset has been renamed (on SDK: check dataset ID matches when sending a get by name reequest. on UI: check + dataset with new name appears and no dataset with old name appears) + """ dataset_name = request.getfixturevalue(dataset_fixture) time.sleep(0.5) new_name = 'updated_test_dataset_name' @@ -104,6 +137,12 @@ def test_dataset_name_update(self, request, page: Page, client: opik.Opik, datas @pytest.mark.parametrize('dataset_fixture', ['create_dataset_sdk_no_cleanup', 'create_dataset_ui_no_cleanup']) def test_dataset_deletion_in_sdk(self, request, page: Page, client: opik.Opik, dataset_fixture): + """ + Checks proper deletion of a dataset via the SDK. Test split into 2: checks on datasets created on both UI and SDK + 1. Create a dataset via the UI/the SDK (2 "instances" of the test created for each one) + 2. Send a request via the SDK to delete the dataset + 3. Check on both the SDK and the UI that the dataset no longer exists (client.get_dataset should throw a 404 error, dataset does not appear in datasets table in UI) + """ dataset_name = request.getfixturevalue(dataset_fixture) time.sleep(0.5) client.delete_dataset(name=dataset_name) @@ -122,6 +161,12 @@ def test_dataset_deletion_in_sdk(self, request, page: Page, client: opik.Opik, d @pytest.mark.parametrize('dataset_fixture', ['create_dataset_sdk_no_cleanup', 'create_dataset_ui_no_cleanup']) def test_dataset_deletion_in_ui(self, request, page: Page, client: opik.Opik, dataset_fixture): + """ + Checks proper deletion of a dataset via the SDK. Test split into 2: checks on datasets created on both UI and SDK + 1. Create a dataset via the UI/the SDK (2 "instances" of the test created for each one) + 2. Delete the dataset from the UI using the delete button in the datasets page + 3. Check on both the SDK and the UI that the dataset no longer exists (client.get_dataset should throw a 404 error, dataset does not appear in datasets table in UI) + """ dataset_name = request.getfixturevalue(dataset_fixture) time.sleep(0.5) datasets_page = DatasetsPage(page) From 98efe61e10fae68c9edc75fdbe6f90dc61b4e28c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20C=C4=83uti=C8=99anu?= Date: Thu, 21 Nov 2024 13:07:40 +0200 Subject: [PATCH 3/5] fix dataset creation when no datasets created --- tests_end_to_end/page_objects/DatasetsPage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests_end_to_end/page_objects/DatasetsPage.py b/tests_end_to_end/page_objects/DatasetsPage.py index a3a245439d..d3e2c518d4 100644 --- a/tests_end_to_end/page_objects/DatasetsPage.py +++ b/tests_end_to_end/page_objects/DatasetsPage.py @@ -9,7 +9,7 @@ def go_to_page(self): self.page.goto(self.url) def create_dataset_by_name(self, dataset_name: str): - self.page.get_by_role('button', name='Create new dataset').click() + self.page.get_by_role('button', name='Create new dataset').first.click() self.page.get_by_placeholder('Dataset name').fill(dataset_name) self.page.get_by_role('button', name='Create dataset').click() From 78a4457f949fa62ff62f187530123ee2799217bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20C=C4=83uti=C8=99anu?= Date: Thu, 21 Nov 2024 14:03:17 +0200 Subject: [PATCH 4/5] traces locator fix --- tests_end_to_end/page_objects/TracesPage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests_end_to_end/page_objects/TracesPage.py b/tests_end_to_end/page_objects/TracesPage.py index a1fff4a866..29763e12cf 100644 --- a/tests_end_to_end/page_objects/TracesPage.py +++ b/tests_end_to_end/page_objects/TracesPage.py @@ -5,7 +5,7 @@ class TracesPage: def __init__(self, page: Page): self.page = page self.traces_table = self.page.get_by_role('table') - self.trace_names_selector = 'tr td:nth-child(2) div span' + self.trace_names_selector = 'tr td:nth-child(3) div span' self.next_page_button_locator = self.page.locator("div:has(> button:nth-of-type(4))").locator('button:nth-of-type(3)') self.delete_button_locator = self.page.locator("div").filter(has_text=re.compile(r"^Add to dataset$")).get_by_role("button").nth(2) From 1772e7d1e483ab327ed3308f190f0d60d7392fe6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20C=C4=83uti=C8=99anu?= Date: Thu, 21 Nov 2024 15:23:37 +0200 Subject: [PATCH 5/5] fixed sanity after UI changes too --- tests_end_to_end/page_objects/ProjectsPage.py | 2 +- tests_end_to_end/page_objects/TracesPage.py | 9 +++++++++ tests_end_to_end/tests/application_sanity/test_sanity.py | 5 +++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/tests_end_to_end/page_objects/ProjectsPage.py b/tests_end_to_end/page_objects/ProjectsPage.py index 8be689c968..e65d777fd5 100644 --- a/tests_end_to_end/page_objects/ProjectsPage.py +++ b/tests_end_to_end/page_objects/ProjectsPage.py @@ -11,7 +11,7 @@ def go_to_page(self): self.page.goto(self.url) def click_project(self, project_name): - self.page.get_by_role('cell', name=project_name).click() + self.page.get_by_role('link', name=project_name).click() def search_project(self, project_name): self.page.get_by_test_id("search-input").click() diff --git a/tests_end_to_end/page_objects/TracesPage.py b/tests_end_to_end/page_objects/TracesPage.py index 29763e12cf..f9240c522e 100644 --- a/tests_end_to_end/page_objects/TracesPage.py +++ b/tests_end_to_end/page_objects/TracesPage.py @@ -6,6 +6,7 @@ def __init__(self, page: Page): self.page = page self.traces_table = self.page.get_by_role('table') self.trace_names_selector = 'tr td:nth-child(3) div span' + self.trace_id_selector = 'tr:nth-child({}) > td:nth-child(2) > div'.format self.next_page_button_locator = self.page.locator("div:has(> button:nth-of-type(4))").locator('button:nth-of-type(3)') self.delete_button_locator = self.page.locator("div").filter(has_text=re.compile(r"^Add to dataset$")).get_by_role("button").nth(2) @@ -16,6 +17,14 @@ def get_all_trace_names_on_page(self): names = self.page.locator(self.trace_names_selector).all_inner_texts() return names + + def click_first_trace_that_has_name(self, trace_name: str): + self.page.get_by_role('row').filter(has_text=trace_name).first.get_by_role('button').first.click() + + + def click_nth_trace_on_page(self, n: int): + self.trace_id_selector(n).click() + def get_first_trace_name_on_page(self): self.page.wait_for_selector(self.trace_names_selector) diff --git a/tests_end_to_end/tests/application_sanity/test_sanity.py b/tests_end_to_end/tests/application_sanity/test_sanity.py index c8520f3835..625525e008 100644 --- a/tests_end_to_end/tests/application_sanity/test_sanity.py +++ b/tests_end_to_end/tests/application_sanity/test_sanity.py @@ -44,9 +44,10 @@ def test_spans_of_traces(page, traces_page, config, log_traces_and_spans_low_lev 3. Check that the spans are present in each trace ''' trace_names = traces_page.get_all_trace_names_on_page() + traces_page.click_first_trace_that_has_name('decorator-trace-1') for trace in trace_names: - page.get_by_text(trace).click() + traces_page.click_first_trace_that_has_name(trace) spans_menu = TracesPageSpansMenu(page) trace_type = trace.split('-')[0] # 'client' or 'decorator' for count in range(config['spans']['count']): @@ -66,7 +67,7 @@ def test_trace_and_span_details(page, traces_page, config, log_traces_and_spans_ trace_names = traces_page.get_all_trace_names_on_page() for trace in trace_names: - page.get_by_text(trace).click() + traces_page.click_first_trace_that_has_name(trace) spans_menu = TracesPageSpansMenu(page) trace_type = trace.split('-')[0] tag_names = config['traces'][trace_type]['tags']