diff --git a/.github/workflows/e2e_tests.yml b/.github/workflows/e2e_tests.yml index bde13e82c..556405339 100644 --- a/.github/workflows/e2e_tests.yml +++ b/.github/workflows/e2e_tests.yml @@ -257,11 +257,9 @@ jobs: timeout-minutes: 60 run: pytest --timeout=600 -ra -sv --color yes --code-highlight yes --durations=0 -vv --ignore tests/e2e/test_notebooks.py tests/e2e env: - KILI_API_CLOUD_VISION: ${{ secrets.KILI_API_CLOUD_VISION }} KILI_API_ENDPOINT: ${{ env.KILI_API_ENDPOINT }} - KILI_API_KEY: ${{ secrets[format('KILI_USER_API_KEY_{0}', env.TEST_AGAINST)] }} - KILI_USER_EMAIL: ${{ secrets[format('KILI_USER_EMAIL_{0}', env.TEST_AGAINST)] }} - KILI_USER_ID: ${{ secrets[format('KILI_USER_ID_{0}', env.TEST_AGAINST)] }} + KILI_API_KEY: ${{ secrets[format('KILI_USER_API_KEY_{0}_USER', env.TEST_AGAINST)] }} + KILI_API_KEY_ADMIN: ${{ secrets[format('KILI_USER_API_KEY_{0}', env.TEST_AGAINST)] }} KILI_TEST_DATA_INTEGRATION_ID: ${{ secrets.KILI_TEST_DATA_INTEGRATION_ID }} - name: Notebook tests diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index c032ab62d..7a7ab58c9 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -1,3 +1,5 @@ +import os + import pytest from kili.client import Kili @@ -5,4 +7,11 @@ @pytest.fixture(scope="session") def kili() -> Kili: + """Kili client with user rights.""" return Kili() + + +@pytest.fixture(scope="session") +def kili_admin() -> Kili: + """Kili client with admin rights.""" + return Kili(api_key=os.environ["KILI_API_KEY_ADMIN"]) diff --git a/tests/e2e/test_cloud_storage.py b/tests/e2e/test_cloud_storage.py index c52db6e33..fcc7927bc 100644 --- a/tests/e2e/test_cloud_storage.py +++ b/tests/e2e/test_cloud_storage.py @@ -11,7 +11,7 @@ @pytest.fixture() -def src_project(kili: Kili): +def src_project(kili_admin: Kili): interface = { "jobs": { "JOB_0": { @@ -32,7 +32,7 @@ def src_project(kili: Kili): } } - project = kili.create_project( + project = kili_admin.create_project( input_type="IMAGE", json_interface=interface, title="test_e2e_synchronize_cloud_storage_connection", @@ -41,7 +41,7 @@ def src_project(kili: Kili): yield project - kili.delete_project(project["id"]) + kili_admin.delete_project(project["id"]) def is_same_endpoint(endpoint_short_name: str, endpoint_url: str) -> bool: @@ -82,7 +82,7 @@ def is_same_endpoint(endpoint_short_name: str, endpoint_url: str) -> bool: ], ) def test_e2e_synchronize_cloud_storage_connection( - kili: Kili, + kili_admin: Kili, src_project: Dict, endpoint_short_name: str, platform_name: str, @@ -91,9 +91,9 @@ def test_e2e_synchronize_cloud_storage_connection( expected_nb_assets_after_sync: int, ) -> None: """E2e test for cloud storage methods.""" - if not is_same_endpoint(endpoint_short_name, kili.api_endpoint): + if not is_same_endpoint(endpoint_short_name, kili_admin.api_endpoint): pytest.skip( - f"Skipping test because endpoint {kili.api_endpoint} does not match" + f"Skipping test because endpoint {kili_admin.api_endpoint} does not match" f" {endpoint_short_name}" ) @@ -113,21 +113,21 @@ def test_e2e_synchronize_cloud_storage_connection( # Check that the data integration exists print("Data integration used:", data_integration_id) - data_integrations = kili.cloud_storage_integrations( + data_integrations = kili_admin.cloud_storage_integrations( status="CONNECTED", cloud_storage_integration_id=data_integration_id ) if len(data_integrations) != 1: raise ValueError(f"Data integration {data_integration_id} not found. Cannot run test.") # Create a data connection - data_connection_id = kili.add_cloud_storage_connection( + data_connection_id = kili_admin.add_cloud_storage_connection( project_id=project_id, cloud_storage_integration_id=data_integration_id, selected_folders=selected_folders, )["id"] # Check that the data connection has been created - data_connections = kili.cloud_storage_connections( + data_connections = kili_admin.cloud_storage_connections( cloud_storage_connection_id=data_connection_id, project_id=project_id, fields=[ @@ -149,15 +149,15 @@ def test_e2e_synchronize_cloud_storage_connection( assert data_connection["dataIntegrationId"] == data_integration_id, data_connection print("Data connection:", data_connection) - nb_assets = kili.count_assets(project_id=project_id) + nb_assets = kili_admin.count_assets(project_id=project_id) assert nb_assets == 0, f"Expected no asset before sync. Got {nb_assets} assets." - kili.synchronize_cloud_storage_connection( + kili_admin.synchronize_cloud_storage_connection( cloud_storage_connection_id=data_connection_id, delete_extraneous_files=True, ) - nb_assets = kili.count_assets(project_id=project_id) + nb_assets = kili_admin.count_assets(project_id=project_id) assert ( nb_assets == expected_nb_assets_after_sync ), f"Expected {expected_nb_assets_after_sync} assets after sync. Got {nb_assets} assets." diff --git a/tests/e2e/test_notebooks.py b/tests/e2e/test_notebooks.py index eb007e225..9d5462fd2 100644 --- a/tests/e2e/test_notebooks.py +++ b/tests/e2e/test_notebooks.py @@ -16,50 +16,62 @@ def process_notebook(notebook_filename: str) -> None: @pytest.mark.parametrize( - "notebook_file", + ("notebook_file", "requires_admin_rights"), [ - "tests/e2e/create_project.ipynb", - "tests/e2e/export_labels.ipynb", - "tests/e2e/import_assets.ipynb", - "tests/e2e/import_predictions.ipynb", + ("tests/e2e/create_project.ipynb", False), + ("tests/e2e/export_labels.ipynb", False), + ("tests/e2e/import_assets.ipynb", False), + ("tests/e2e/import_predictions.ipynb", False), pytest.param( "tests/e2e/plugin_workflow.ipynb", + False, marks=pytest.mark.skipif( "lts.cloud" in os.environ["KILI_API_ENDPOINT"], reason="Feature not available on premise", ), ), - "recipes/basic_project_setup.ipynb", - "recipes/export_a_kili_project.ipynb", - "recipes/frame_dicom_data.ipynb", + ("recipes/basic_project_setup.ipynb", False), + ("recipes/export_a_kili_project.ipynb", True), + ("recipes/frame_dicom_data.ipynb", False), # "recipes/finetuning_dinov2.ipynb", # not testable because requires GPU - "recipes/geojson.ipynb", - "recipes/importing_coco.ipynb", - "recipes/importing_pascalvoc.ipynb", - "recipes/import_text_assets.ipynb", - "recipes/importing_assets_and_metadata.ipynb", - "recipes/importing_pdf_assets.ipynb", - "recipes/importing_labels.ipynb", - "recipes/importing_video_assets.ipynb", - "recipes/inference_labels.ipynb", - "recipes/label_parsing.ipynb", - "recipes/medical_imaging.ipynb", - # "recipes/ner_pre_annotations_openai.ipynb", - "recipes/ocr_pre_annotations.ipynb", - "recipes/pixel_level_masks.ipynb", + ("recipes/geojson.ipynb", False), + ("recipes/importing_coco.ipynb", False), + ("recipes/importing_pascalvoc.ipynb", False), + ("recipes/import_text_assets.ipynb", False), + ("recipes/importing_assets_and_metadata.ipynb", False), + ("recipes/importing_pdf_assets.ipynb", False), + ("recipes/importing_labels.ipynb", False), + ("recipes/importing_video_assets.ipynb", False), + ("recipes/inference_labels.ipynb", False), + ("recipes/label_parsing.ipynb", False), + ("recipes/medical_imaging.ipynb", False), + ("recipes/ner_pre_annotations_openai.ipynb", False), + ("recipes/ocr_pre_annotations.ipynb", False), + ("recipes/pixel_level_masks.ipynb", False), pytest.param( "recipes/plugins_example.ipynb", + False, marks=pytest.mark.skipif( "lts.cloud" in os.environ["KILI_API_ENDPOINT"], reason="Feature not available on premise", ), ), - # "recipes/plugins_development.ipynb" - "recipes/set_up_workflows.ipynb", + # "recipes/plugins_development.ipynb", False + ("recipes/set_up_workflows.ipynb", False), # "recipes/tagtog_to_kili.ipynb", # not testable because data is private - "recipes/webhooks_example.ipynb", + ("recipes/webhooks_example.ipynb", False), ], ) -def test_all_recipes(notebook_file: str): +def test_all_recipes( + notebook_file: str, + requires_admin_rights: bool, # noqa: FBT001 + monkeypatch: pytest.MonkeyPatch, +) -> None: """Runs `process_notebook` on all notebooks in the git repository.""" - process_notebook(notebook_file) + initial_api_key = os.environ["KILI_API_KEY"] + try: + if requires_admin_rights: + monkeypatch.setenv("KILI_API_KEY", os.environ["KILI_API_KEY_ADMIN"]) + process_notebook(notebook_file) + finally: + monkeypatch.setenv("KILI_API_KEY", initial_api_key) diff --git a/tests/e2e/test_query_project_users.py b/tests/e2e/test_query_project_users.py index 3e3469946..ae9bea175 100644 --- a/tests/e2e/test_query_project_users.py +++ b/tests/e2e/test_query_project_users.py @@ -6,41 +6,41 @@ @pytest.fixture() -def project_id_suspended_user_email(kili: Kili): - project = kili.create_project( +def project_id_suspended_user_email(kili_admin: Kili): + project = kili_admin.create_project( input_type="TEXT", title="test_query_project_users.py sdk", json_interface={"jobs": {}} ) # add a user that we desactivate suspended_user_email = f"john.doe{uuid.uuid4()}+desactivated@kili-technology.com" - kili.append_to_roles( + kili_admin.append_to_roles( project_id=project["id"], user_email=suspended_user_email, role="LABELER", ) - kili.update_properties_in_user(email=suspended_user_email, activated=False) + kili_admin.update_properties_in_user(email=suspended_user_email, activated=False) yield project["id"], suspended_user_email - kili.delete_project(project_id=project["id"]) + kili_admin.delete_project(project_id=project["id"]) def test_given_project_when_querying_project_users_it_works( - kili: Kili, project_id_suspended_user_email + kili_admin: Kili, project_id_suspended_user_email ): # Given project_id, suspended_user_email = project_id_suspended_user_email - api_user = kili.get_user() + api_user = kili_admin.get_user() fields = ["activated", "deletedAt", "id", "role", "user.email", "user.id", "status"] # When - all_users = kili.project_users(project_id=project_id, fields=fields, status_in=None) + all_users = kili_admin.project_users(project_id=project_id, fields=fields, status_in=None) # Then assert len(all_users) > 0 # When - activated_users = kili.project_users( + activated_users = kili_admin.project_users( project_id=project_id, fields=fields, status_in=["ACTIVATED"] ) @@ -49,7 +49,9 @@ def test_given_project_when_querying_project_users_it_works( assert activated_users[0]["user"]["email"] == api_user["email"], activated_users # When - admin_users = kili.project_users(project_id=project_id, fields=fields, status_in=["ORG_ADMIN"]) + admin_users = kili_admin.project_users( + project_id=project_id, fields=fields, status_in=["ORG_ADMIN"] + ) # Then, admin users are not api user or disabled user for proj_user in admin_users: @@ -59,7 +61,7 @@ def test_given_project_when_querying_project_users_it_works( }, admin_users # When - disabled_users = kili.project_users( + disabled_users = kili_admin.project_users( project_id=project_id, fields=fields, status_in=["ORG_SUSPENDED"] ) diff --git a/tests/e2e/test_tags.py b/tests/e2e/test_tags.py index 8a9e15669..dd6e7a846 100644 --- a/tests/e2e/test_tags.py +++ b/tests/e2e/test_tags.py @@ -62,21 +62,21 @@ def test_given_project_with_tags_when_i_call_untag_project_then_it_removes_one_t assert not any(tag["label"] == tag_to_delete for tag in project_tags) -def test_given_org_with_tags_when_i_update_tag_then_it_is_updated(kili: Kili): +def test_given_org_with_tags_when_i_update_tag_then_it_is_updated(kili_admin: Kili): # Given - org_tags = kili.tags(fields=("label",)) + org_tags = kili_admin.tags(fields=("label",)) assert len(org_tags) > 0, "Organization has no tags" tag_to_update = next(tag for tag in org_tags if tag["label"]) prev_tag_name = tag_to_update["label"] # When new_tag_name = str(uuid.uuid4()) - kili.update_tag(tag_name=prev_tag_name, new_tag_name=new_tag_name) + kili_admin.update_tag(tag_name=prev_tag_name, new_tag_name=new_tag_name) # Then - org_tags = kili.tags(fields=("label",)) + org_tags = kili_admin.tags(fields=("label",)) assert len(org_tags) > 0, "Organization has no tags" assert new_tag_name in [tag["label"] for tag in org_tags] # Cleanup - kili.update_tag(tag_name=new_tag_name, new_tag_name=prev_tag_name) + kili_admin.update_tag(tag_name=new_tag_name, new_tag_name=prev_tag_name)