Add E2E datasets tests, GHA merging, minor fix-ups (#680)

* datasets tests, GHA merging, minor fix-ups

* descriptions

* fix dataset creation when no datasets created

* traces locator fix

* fixed sanity after UI changes too

---------

Co-authored-by: Andrei Căutișanu <[email protected]>
Co-authored-by: Andrei Căutișanu <[email protected]>

.github/workflows/end2end_suites.yml

@@ -7,10 +7,13 @@ on:
                 type: choice
                 description: 'Choose which test suite to run'
                 required: true
-                default: 'projects-CRUD'
+                default: 'sanity'
                 options:
-                - projects-CRUD
-                - traces-CRUD
+                - all_features
+                - sanity
+                - projects
+                - traces
+                - datasets
 
 jobs:
     run_suite:
@@ -64,10 +67,16 @@ jobs:
                 cd ${{ github.workspace }}/tests_end_to_end
                 export PYTHONPATH='.'
 
-                if [ "$SUITE" == "projects-CRUD" ]; then
+                if [ "$SUITE" == "projects" ]; then
                     pytest -s tests/Projects/test_projects_crud_operations.py --browser chromium --base-url http://localhost:5173 --setup-show
-                elif [ "$SUITE" == "traces-CRUD" ]; then
+                elif [ "$SUITE" == "traces" ]; then
                     pytest -s tests/Traces/test_traces_crud_operations.py --browser chromium --base-url http://localhost:5173 --setup-show
+                elif [ "$SUITE" == "datasets" ]; then
+                    pytest -s tests/Datasets/test_datasets_crud_operations.py --browser chromium --base-url http://localhost:5173 --setup-show
+                elif [ "$SUITE" == "sanity" ]; then
+                    pytest -s tests/application_sanity/test_sanity.py --browser chromium --base-url http://localhost:5173 --setup-show
+                elif [ "$SUITE" == "all_features" ]; then
+                    pytest -s tests --browser chromium --base-url http://localhost:5173 --setup-show --ignore=tests/application_sanity
                 fi
 
             - name: Stop Opik server

tests_end_to_end/page_objects/DatasetsPage.py

@@ -8,10 +8,29 @@ def __init__(self, page: Page):
     def go_to_page(self):
         self.page.goto(self.url)
 
+    def create_dataset_by_name(self, dataset_name: str):
+        self.page.get_by_role('button', name='Create new dataset').first.click()
+        self.page.get_by_placeholder('Dataset name').fill(dataset_name)
+        self.page.get_by_role('button', name='Create dataset').click()
+
     def select_database_by_name(self, name):
         self.page.get_by_text(name, exact=True).first.click()
 
-    def check_dataset_exists_by_name(self, dataset_name):
-        expect(self.page.get_by_text(dataset_name)).to_be_visible()
+    def search_dataset(self, dataset_name):
+        self.page.get_by_test_id("search-input").click()
+        self.page.get_by_test_id("search-input").fill(dataset_name)
 
+    def check_dataset_exists_on_page_by_name(self, dataset_name):
+        expect(self.page.get_by_text(dataset_name).first).to_be_visible()
+
+    def check_dataset_not_exists_on_page_by_name(self, dataset_name):
+        expect(self.page.get_by_text(dataset_name).first).not_to_be_visible()
+
+    def delete_dataset_by_name(self, dataset_name):
+        self.search_dataset(dataset_name)
+        row = self.page.get_by_role('row').filter(has_text=dataset_name).filter(has=self.page.get_by_role('cell', name=dataset_name, exact=True))
+        row.get_by_role("button").click()
+        self.page.get_by_role("menuitem", name="Delete").click()
+        self.page.get_by_role("button", name="Delete dataset").click()
+
 

tests_end_to_end/page_objects/ProjectsPage.py

@@ -11,7 +11,7 @@ def go_to_page(self):
         self.page.goto(self.url)
 
     def click_project(self, project_name):
-        self.page.get_by_role('cell', name=project_name).click()
+        self.page.get_by_role('link', name=project_name).click()
 
     def search_project(self, project_name):
         self.page.get_by_test_id("search-input").click()

tests_end_to_end/page_objects/TracesPage.py

@@ -5,7 +5,8 @@ class TracesPage:
     def __init__(self, page: Page):
         self.page = page
         self.traces_table = self.page.get_by_role('table')
-        self.trace_names_selector = 'tr td:nth-child(2) div span'
+        self.trace_names_selector = 'tr td:nth-child(3) div span'
+        self.trace_id_selector = 'tr:nth-child({}) > td:nth-child(2) > div'.format
         self.next_page_button_locator = self.page.locator("div:has(> button:nth-of-type(4))").locator('button:nth-of-type(3)')
         self.delete_button_locator = self.page.locator("div").filter(has_text=re.compile(r"^Add to dataset$")).get_by_role("button").nth(2)
 
@@ -16,6 +17,14 @@ def get_all_trace_names_on_page(self):
         names = self.page.locator(self.trace_names_selector).all_inner_texts()
         return names
 
+
+    def click_first_trace_that_has_name(self, trace_name: str):
+        self.page.get_by_role('row').filter(has_text=trace_name).first.get_by_role('button').first.click()
+
+
+    def click_nth_trace_on_page(self, n: int):
+        self.trace_id_selector(n).click()
+
 
     def get_first_trace_name_on_page(self):
         self.page.wait_for_selector(self.trace_names_selector)
@@ -83,4 +92,12 @@ def delete_all_traces_that_match_name_contains_filter(self, name: str):
 
             pagination_button = self.get_pagination_button()
             expect(pagination_button).not_to_have_text(f'Showing 1-10 of {total_traces}')
-            total_traces = self.get_total_number_of_traces_in_project()
+            total_traces = self.get_total_number_of_traces_in_project()
+
+
+    def add_all_traces_to_new_dataset(self, dataset_name: str):
+        self.page.get_by_label("Select all").click()
+        self.page.get_by_role("button", name="Add to dataset").click()
+        self.page.get_by_role("button", name="Create new dataset").click()
+        self.page.get_by_placeholder("Dataset name").fill(dataset_name)
+        self.page.get_by_role("button", name="Create dataset").click()

tests_end_to_end/tests/Datasets/test_datasets_crud_operations.py

@@ -0,0 +1,189 @@
+import pytest
+from playwright.sync_api import Page, expect
+from page_objects.DatasetsPage import DatasetsPage
+from page_objects.ProjectsPage import ProjectsPage
+from page_objects.TracesPage import TracesPage
+from sdk_helpers import delete_dataset_by_name_if_exists, update_dataset_name, get_dataset_by_name
+import opik
+import time
+
+
+class TestDatasetsCrud:
+
+    def test_create_dataset_ui_datasets_page(self, page: Page):
+        """
+        Basic test to check dataset creation via UI. Uses the UI after creation to check the dataset exists
+        1. Create dataset via UI from the datasets page
+        2. Check the dataset exists in the dataset table
+        3. If no errors raised, test passes
+        """
+        datasets_page = DatasetsPage(page)
+        datasets_page.go_to_page()
+        dataset_name = 'automated_tests_dataset'
+        try:
+            datasets_page.create_dataset_by_name(dataset_name=dataset_name)
+            datasets_page.check_dataset_exists_on_page_by_name(dataset_name=dataset_name)
+        except Exception as e:
+            print(f'error during dataset creation: {e}')
+            raise
+        finally:
+            delete_dataset_by_name_if_exists(dataset_name=dataset_name)
+
+
+    def test_create_dataset_ui_add_traces_to_new_dataset(self, page: Page, create_delete_project_sdk, create_10_test_traces):
+        """
+        Basic test to check dataset creation via "add to new dataset" functionality in the traces page. Uses the UI after creation to check the project exists
+        1. Create a project with some traces
+        2. Via the UI, select the traces and add them to a new dataset
+        3. Switch to the datasets page, check the dataset exists in the dataset table
+        4. If no errors raised and dataset exists, test passes
+        """
+        dataset_name = 'automated_tests_dataset'
+        proj_name = create_delete_project_sdk
+        projects_page = ProjectsPage(page)
+        projects_page.go_to_page()
+        projects_page.click_project(project_name=proj_name)
+
+        traces_page = TracesPage(page)
+        traces_page.add_all_traces_to_new_dataset(dataset_name=dataset_name)
+
+        try:
+            datasets_page = DatasetsPage(page)
+            datasets_page.go_to_page()
+            datasets_page.check_dataset_exists_on_page_by_name(dataset_name=dataset_name)
+        except Exception as e:
+            print(f'error: dataset not created: {e}')
+            raise
+        finally:
+            delete_dataset_by_name_if_exists(dataset_name=dataset_name)
+
+
+    def test_create_dataset_sdk_client(self, client: opik.Opik):
+        """
+        Basic test to check dataset creation via SDK. Uses the SDK to fetch the created dataset to check it exists
+        1. Create dataset via SDK Opik client
+        2. Get the project via SDK OpikAPI client
+        3. If dataset creation fails, client.get_dataset will throw an error and the test will fail.
+        """
+        dataset_name = 'automated_tests_dataset'
+        try:
+            client.create_dataset(name=dataset_name)
+            time.sleep(0.2)
+            assert client.get_dataset(name=dataset_name) is not None
+        except Exception as e:
+            print(f'error during dataset creation: {e}')
+            raise
+        finally:
+            delete_dataset_by_name_if_exists(dataset_name=dataset_name)
+
+
+    @pytest.mark.parametrize('dataset_fixture', ['create_delete_dataset_ui', 'create_delete_dataset_sdk'])
+    def test_dataset_visibility(self, request, page: Page, client: opik.Opik, dataset_fixture):
+        """
+        Checks a created dataset is visible via both the UI and SDK. Test split in 2: checks on datasets created on both UI and SDK
+        1. Create a dataset via the UI/the SDK (2 "instances" of the test created for each one)
+        2. Fetch the dataset by name using the SDK Opik client and check the dataset exists in the datasets table in the UI
+        3. Check that the correct dataset is returned in the SDK and that the name is correct in the UI
+        """
+        dataset_name = request.getfixturevalue(dataset_fixture)
+        time.sleep(0.5)
+
+        datasets_page = DatasetsPage(page)
+        datasets_page.go_to_page()
+        datasets_page.check_dataset_exists_on_page_by_name(dataset_name)
+
+        dataset_sdk = client.get_dataset(dataset_name)
+        assert dataset_sdk.name == dataset_name
+
+
+    @pytest.mark.parametrize('dataset_fixture', ['create_dataset_sdk_no_cleanup', 'create_dataset_ui_no_cleanup'])
+    def test_dataset_name_update(self, request, page: Page, client: opik.Opik, dataset_fixture):
+        """
+        Checks using the SDK update method on a dataset. Test split into 2: checks on dataset created on both UI and SDK
+        1. Create a dataset via the UI/the SDK (2 "instances" of the test created for each one)
+        2. Send a request via the SDK OpikApi client to update the dataset's name
+        3. Check on both the SDK and the UI that the dataset has been renamed (on SDK: check dataset ID matches when sending a get by name reequest. on UI: check
+        dataset with new name appears and no dataset with old name appears)
+        """
+        dataset_name = request.getfixturevalue(dataset_fixture)
+        time.sleep(0.5)
+        new_name = 'updated_test_dataset_name'
+
+        name_updated = False
+        try:
+            dataset_id = update_dataset_name(name=dataset_name, new_name=new_name)
+            name_updated = True
+
+            dataset_new_name = get_dataset_by_name(dataset_name=new_name)
+
+            dataset_id_updated_name = dataset_new_name['id']
+            assert dataset_id_updated_name == dataset_id
+
+            datasets_page = DatasetsPage(page)
+            datasets_page.go_to_page()
+            datasets_page.check_dataset_exists_on_page_by_name(dataset_name=new_name)
+            datasets_page.check_dataset_not_exists_on_page_by_name(dataset_name=dataset_name)
+
+        except Exception as e:
+            print(f'Error occured during update of project name: {e}')
+            raise
+
+        finally:
+            if name_updated:
+                delete_dataset_by_name_if_exists(new_name)
+            else:
+                delete_dataset_by_name_if_exists(dataset_name)
+
+
+    @pytest.mark.parametrize('dataset_fixture', ['create_dataset_sdk_no_cleanup', 'create_dataset_ui_no_cleanup'])
+    def test_dataset_deletion_in_sdk(self, request, page: Page, client: opik.Opik, dataset_fixture):
+        """
+        Checks proper deletion of a dataset via the SDK. Test split into 2: checks on datasets created on both UI and SDK
+        1. Create a dataset via the UI/the SDK (2 "instances" of the test created for each one)
+        2. Send a request via the SDK to delete the dataset
+        3. Check on both the SDK and the UI that the dataset no longer exists (client.get_dataset should throw a 404 error, dataset does not appear in datasets table in UI)
+        """
+        dataset_name = request.getfixturevalue(dataset_fixture)
+        time.sleep(0.5)
+        client.delete_dataset(name=dataset_name)
+        dataset_page = DatasetsPage(page)
+        dataset_page.go_to_page()
+        dataset_page.check_dataset_not_exists_on_page_by_name(dataset_name=dataset_name)
+        try:
+            _ = client.get_dataset(dataset_name)
+            assert False, f'datasets {dataset_name} somehow still exists after deletion'
+        except Exception as e:
+            if '404' in str(e) or 'not found' in str(e).lower():
+                pass
+            else:
+                raise
+
+
+    @pytest.mark.parametrize('dataset_fixture', ['create_dataset_sdk_no_cleanup', 'create_dataset_ui_no_cleanup'])
+    def test_dataset_deletion_in_ui(self, request, page: Page, client: opik.Opik, dataset_fixture):
+        """
+        Checks proper deletion of a dataset via the SDK. Test split into 2: checks on datasets created on both UI and SDK
+        1. Create a dataset via the UI/the SDK (2 "instances" of the test created for each one)
+        2. Delete the dataset from the UI using the delete button in the datasets page
+        3. Check on both the SDK and the UI that the dataset no longer exists (client.get_dataset should throw a 404 error, dataset does not appear in datasets table in UI)
+        """
+        dataset_name = request.getfixturevalue(dataset_fixture)
+        time.sleep(0.5)
+        datasets_page = DatasetsPage(page)
+        datasets_page.go_to_page()
+        datasets_page.delete_dataset_by_name(dataset_name=dataset_name)
+        time.sleep(1)
+
+        try:
+            _ = client.get_dataset(dataset_name)
+            assert False, f'datasets {dataset_name} somehow still exists after deletion'
+        except Exception as e:
+            if '404' in str(e) or 'not found' in str(e).lower():
+                pass
+            else:
+                raise
+
+        dataset_page = DatasetsPage(page)
+        dataset_page.go_to_page()
+        dataset_page.check_dataset_not_exists_on_page_by_name(dataset_name=dataset_name)
+

tests_end_to_end/tests/Projects/test_projects_crud_operations.py

@@ -112,6 +112,7 @@ def test_project_name_update(self, request, page: Page, project_fixture):
 
         except Exception as e:
             print(f'Error occured during update of project name: {e}')
+            raise
 
         finally:
             if name_updated:

tests_end_to_end/tests/Traces/conftest.py

@@ -44,9 +44,3 @@ def log_x_traces_with_one_span_via_client(client, traces_number):
 def create_traces(request, traces_number):
     create = request.getfixturevalue(request.param)
     yield 0
-
-
-@pytest.fixture(scope='function')
-def create_delete_traces(traces_number):
-    create_traces_sdk(PREFIX, PROJECT_NAME, traces_number)
-    yield