From 127661f17481ceba7cf5181fd35f53243d365ad1 Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Thu, 16 Nov 2023 21:34:45 +0530 Subject: [PATCH 01/33] standard tests list generator Signed-off-by: Samhita Alla --- .github/workflows/checks.yml | 24 ++++--- flyte_tests_generator.py | 14 ++++ flyte_tests_manifest.json | 129 ++++++++++++++++++++--------------- 3 files changed, 101 insertions(+), 66 deletions(-) create mode 100644 flyte_tests_generator.py diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index fd715dd3c..e6d214463 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -122,10 +122,17 @@ jobs: name: Publish artifacts to github release runs-on: ubuntu-latest needs: [prerelease] + strategy: + matrix: + python-version: ["3.11"] steps: - uses: actions/checkout@v2 with: fetch-depth: "0" + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} - name: setup download artifact dir run: | mkdir download-artifact @@ -140,6 +147,8 @@ jobs: for i in */; do tar -czvf "../release-snacks/${i%/}.tar.gz" "$i" & done; wait cd .. && sudo rm -rf download-artifact/ cp flyte_tests_manifest.json release-snacks/flyte_tests_manifest.json + python flyte_tests_generator.py >> flyte_tests.txt + cp flyte_tests.txt release-snacks/flyte_tests.txt - name: Release test manifest uses: goreleaser/goreleaser-action@v2 with: @@ -210,18 +219,11 @@ jobs: with: repository: flyteorg/flytesnacks path: flytesnacks + - name: Generate tests + run: python flyte_tests_generator.py >> flyte_tests.txt - name: Register specific tests run: | - for f in \ - basics/basics/hello_world.py \ - basics/basics/workflow.py \ - basics/basics/named_outputs.py \ - advanced_composition/advanced_composition/chain_entities.py \ - advanced_composition/advanced_composition/dynamics.py \ - advanced_composition/advanced_composition/map_task.py \ - advanced_composition/advanced_composition/subworkflows.py \ - data_types_and_io/data_types_and_io/dataclass.py \ - data_types_and_io/data_types_and_io/structured_dataset.py ; + for f in open("flyte_tests.txt").readlines() ; do # TODO: unpin version of flytekit once the FlyteFile bug is fixed pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \ @@ -230,7 +232,7 @@ jobs: --domain development \ --image cr.flyte.org/flyteorg/flytekit:py3.11-1.10.0 \ --version ${{ env.FLYTESNACKS_VERSION }} \ - flytesnacks/examples/$f; + flytesnacks/$f; done - name: End2End run: | diff --git a/flyte_tests_generator.py b/flyte_tests_generator.py new file mode 100644 index 000000000..dc35fb135 --- /dev/null +++ b/flyte_tests_generator.py @@ -0,0 +1,14 @@ +import glob +import itertools + +directories_to_walk = [ + "examples/basics/basics", + "examples/data_types_and_io/data_types_and_io", + "examples/advanced_composition/advanced_composition", +] + +all_tests = [] +for each_directory in directories_to_walk: + all_tests.append(glob.glob(f"{each_directory}/*.py")) + +print("\n".join(list(itertools.chain(*all_tests)))) diff --git a/flyte_tests_manifest.json b/flyte_tests_manifest.json index 91339d19e..b0a756d9d 100644 --- a/flyte_tests_manifest.json +++ b/flyte_tests_manifest.json @@ -1,4 +1,5 @@ -[{ +[ + { "name": "core", "priority": "P0", "path": "core", @@ -6,61 +7,68 @@ "exit_success": true, "exit_message": "" } - },{ + }, + { "name": "integrations-hive", "priority": "P1", "path": "examples/hive_plugin", "exitCondition": { "exit_success": true, - "exit_message": "" + "exit_message": "" } - },{ + }, + { "name": "integrations-k8s-spark", "priority": "P1", "path": "examples/k8s_spark_plugin", "exitCondition": { "exit_success": true, - "exit_message": "" + "exit_message": "" } - },{ + }, + { "name": "integrations-kfpytorch", "priority": "P1", "path": "examples/kfpytorch_plugin", "exitCondition": { "exit_success": true, - "exit_message": "" + "exit_message": "" } - },{ + }, + { "name": "integrations-kftensorflow", "priority": "P1", "path": "examples/kftensorflow_plugin", "exitCondition": { "exit_success": true, - "exit_message": "" + "exit_message": "" } - },{ + }, + { "name": "integrations-pod", "priority": "P1", "path": "examples/k8s_pod_plugin", "exitCondition": { "exit_success": true, - "exit_message": "" + "exit_message": "" } - },{ + }, + { "name": "integrations-pandera", "priority": "P1", "path": "examples/pandera_plugin", "exitCondition": { "exit_success": true, - "exit_message": "" + "exit_message": "" } - },{ + }, + { "name": "integrations-whylogs", "priority": "P1", "path": "examples/whylogs_plugin", "exitCondition": { "exit_success": true, - "exit_message": "" + "exit_message": "" } }, { @@ -69,86 +77,97 @@ "path": "examples/modin_plugin", "exitCondition": { "exit_success": true, - "exit_message": "" + "exit_message": "" } - },{ + }, + { "name": "integrations-papermill", "priority": "P1", "path": "examples/papermill_plugin", "exitCondition": { "exit_success": true, - "exit_message": "" + "exit_message": "" } - },{ + }, + { "name": "integrations-greatexpectations", "priority": "P1", "path": "examples/greatexpectations_plugin", "exitCondition": { "exit_success": true, - "exit_message": "" + "exit_message": "" } - },{ + }, + { "name": "integrations-sagemaker-pytorch", "priority": "P1", "path": "examples/sagemaker_pytorch_plugin", "exitCondition": { "exit_success": true, - "exit_message": "" + "exit_message": "" } - },{ + }, + { "name": "integrations-sagemaker-training", "priority": "P1", "path": "examples/sagemaker_training_plugin", "exitCondition": { "exit_success": true, - "exit_message": "" - } - },{ - "name": "case-studies-house-price-prediction", - "priority": "P2", - "path": "examples/house_price_prediction", - "exitCondition": { - "exit_success": true, - "exit_message": "" - } - },{ - "name": "case-studies-pima-diabetes", - "priority": "P2", - "path": "examples/pima_diabetes", - "exitCondition": { - "exit_success": true, - "exit_message": "" - } - },{ - "name": "case-studies-mnist-classifier", - "priority": "P2", - "path": "examples/mnist_classifier", - "exitCondition": { - "exit_success": true, - "exit_message": "" - } - },{ + "exit_message": "" + } + }, + { + "name": "case-studies-house-price-prediction", + "priority": "P2", + "path": "examples/house_price_prediction", + "exitCondition": { + "exit_success": true, + "exit_message": "" + } + }, + { + "name": "case-studies-pima-diabetes", + "priority": "P2", + "path": "examples/pima_diabetes", + "exitCondition": { + "exit_success": true, + "exit_message": "" + } + }, + { + "name": "case-studies-mnist-classifier", + "priority": "P2", + "path": "examples/mnist_classifier", + "exitCondition": { + "exit_success": true, + "exit_message": "" + } + }, + { "name": "case-studies-eda", "priority": "P2", "path": "examples/eda", "exitCondition": { "exit_success": true, "exit_message": "" - } - },{ + } + }, + { "name": "case-studies-feast-integration", "priority": "P2", "path": "examples/feast_integration", "exitCondition": { "exit_success": true, "exit_message": "" - } - },{ + } + }, + { "name": "case-studies-blast", "priority": "P2", "path": "examples/blast", "exitCondition": { "exit_success": true, "exit_message": "" + } } -}] +] From 27090e99de4470fc1b8520bd7002acc5e3baae75 Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Thu, 16 Nov 2023 21:47:07 +0530 Subject: [PATCH 02/33] bash Signed-off-by: Samhita Alla --- .github/workflows/checks.yml | 4 ++-- flyte_tests.txt | 29 +++++++++++++++++++++++++++++ ft.py | 2 ++ 3 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 flyte_tests.txt create mode 100644 ft.py diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index e6d214463..9950759c6 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -223,7 +223,7 @@ jobs: run: python flyte_tests_generator.py >> flyte_tests.txt - name: Register specific tests run: | - for f in open("flyte_tests.txt").readlines() ; + while read -r line; do # TODO: unpin version of flytekit once the FlyteFile bug is fixed pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \ @@ -233,7 +233,7 @@ jobs: --image cr.flyte.org/flyteorg/flytekit:py3.11-1.10.0 \ --version ${{ env.FLYTESNACKS_VERSION }} \ flytesnacks/$f; - done + done < flyte_tests.txt - name: End2End run: | make end2end_execute diff --git a/flyte_tests.txt b/flyte_tests.txt new file mode 100644 index 000000000..231080ea2 --- /dev/null +++ b/flyte_tests.txt @@ -0,0 +1,29 @@ +examples/basics/basics/task.py +examples/basics/basics/hello_world.py +examples/basics/basics/documenting_workflows.py +examples/basics/basics/shell_task.py +examples/basics/basics/__init__.py +examples/basics/basics/launch_plan.py +examples/basics/basics/imperative_workflow.py +examples/basics/basics/workflow.py +examples/basics/basics/named_outputs.py +examples/data_types_and_io/data_types_and_io/folder.py +examples/data_types_and_io/data_types_and_io/dataclass.py +examples/data_types_and_io/data_types_and_io/pickle_type.py +examples/data_types_and_io/data_types_and_io/__init__.py +examples/data_types_and_io/data_types_and_io/file.py +examples/data_types_and_io/data_types_and_io/structured_dataset.py +examples/data_types_and_io/data_types_and_io/enum_type.py +examples/data_types_and_io/data_types_and_io/pytorch_type.py +examples/advanced_composition/advanced_composition/decorating_tasks.py +examples/advanced_composition/advanced_composition/merge_sort.py +examples/advanced_composition/advanced_composition/checkpoint.py +examples/advanced_composition/advanced_composition/__init__.py +examples/advanced_composition/advanced_composition/chain_entities.py +examples/advanced_composition/advanced_composition/subworkflows.py +examples/advanced_composition/advanced_composition/conditions.py +examples/advanced_composition/advanced_composition/waiting_for_external_inputs.py +examples/advanced_composition/advanced_composition/eager_workflows.py +examples/advanced_composition/advanced_composition/decorating_workflows.py +examples/advanced_composition/advanced_composition/dynamics.py +examples/advanced_composition/advanced_composition/map_task.py diff --git a/ft.py b/ft.py new file mode 100644 index 000000000..edac9ab52 --- /dev/null +++ b/ft.py @@ -0,0 +1,2 @@ +for f in open("flyte_tests.txt").readlines(): + print(f.strip()) From 3f28d92bd1f8d7a370d63c1efbea5b9a83ff974f Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Fri, 17 Nov 2023 22:20:58 +0530 Subject: [PATCH 03/33] nit Signed-off-by: Samhita Alla --- .github/workflows/checks.yml | 2 +- ft.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) delete mode 100644 ft.py diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 9950759c6..886b18838 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -232,7 +232,7 @@ jobs: --domain development \ --image cr.flyte.org/flyteorg/flytekit:py3.11-1.10.0 \ --version ${{ env.FLYTESNACKS_VERSION }} \ - flytesnacks/$f; + flytesnacks/$line; done < flyte_tests.txt - name: End2End run: | diff --git a/ft.py b/ft.py deleted file mode 100644 index edac9ab52..000000000 --- a/ft.py +++ /dev/null @@ -1,2 +0,0 @@ -for f in open("flyte_tests.txt").readlines(): - print(f.strip()) From 330bb1aa9f7cc2d1b2ec5171fba1a78e244e8342 Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Fri, 17 Nov 2023 22:22:08 +0530 Subject: [PATCH 04/33] nit Signed-off-by: Samhita Alla --- flyte_tests.txt | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100644 flyte_tests.txt diff --git a/flyte_tests.txt b/flyte_tests.txt deleted file mode 100644 index 231080ea2..000000000 --- a/flyte_tests.txt +++ /dev/null @@ -1,29 +0,0 @@ -examples/basics/basics/task.py -examples/basics/basics/hello_world.py -examples/basics/basics/documenting_workflows.py -examples/basics/basics/shell_task.py -examples/basics/basics/__init__.py -examples/basics/basics/launch_plan.py -examples/basics/basics/imperative_workflow.py -examples/basics/basics/workflow.py -examples/basics/basics/named_outputs.py -examples/data_types_and_io/data_types_and_io/folder.py -examples/data_types_and_io/data_types_and_io/dataclass.py -examples/data_types_and_io/data_types_and_io/pickle_type.py -examples/data_types_and_io/data_types_and_io/__init__.py -examples/data_types_and_io/data_types_and_io/file.py -examples/data_types_and_io/data_types_and_io/structured_dataset.py -examples/data_types_and_io/data_types_and_io/enum_type.py -examples/data_types_and_io/data_types_and_io/pytorch_type.py -examples/advanced_composition/advanced_composition/decorating_tasks.py -examples/advanced_composition/advanced_composition/merge_sort.py -examples/advanced_composition/advanced_composition/checkpoint.py -examples/advanced_composition/advanced_composition/__init__.py -examples/advanced_composition/advanced_composition/chain_entities.py -examples/advanced_composition/advanced_composition/subworkflows.py -examples/advanced_composition/advanced_composition/conditions.py -examples/advanced_composition/advanced_composition/waiting_for_external_inputs.py -examples/advanced_composition/advanced_composition/eager_workflows.py -examples/advanced_composition/advanced_composition/decorating_workflows.py -examples/advanced_composition/advanced_composition/dynamics.py -examples/advanced_composition/advanced_composition/map_task.py From 8b0cb62f676352aa76721d1394c352cf9b103461 Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Fri, 17 Nov 2023 22:37:36 +0530 Subject: [PATCH 05/33] add torch Signed-off-by: Samhita Alla --- dev-requirements.in | 1 + dev-requirements.txt | 18 ++++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/dev-requirements.in b/dev-requirements.in index 996558ba2..72f0544d7 100644 --- a/dev-requirements.in +++ b/dev-requirements.in @@ -11,3 +11,4 @@ mock pytest mypy mashumaro +torch diff --git a/dev-requirements.txt b/dev-requirements.txt index 174553f41..922882a43 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -101,7 +101,9 @@ docker-image-py==0.1.12 docstring-parser==0.15 # via flytekit filelock==3.13.1 - # via virtualenv + # via + # torch + # virtualenv flake8==6.1.0 # via # -r dev-requirements.in @@ -125,6 +127,7 @@ fsspec==2023.10.0 # flytekit # gcsfs # s3fs + # torch gcsfs==2023.10.0 # via flytekit gitdb==4.0.11 @@ -188,7 +191,9 @@ isort==5.12.0 jaraco-classes==3.3.0 # via keyring jinja2==3.1.2 - # via cookiecutter + # via + # cookiecutter + # torch jmespath==1.0.1 # via botocore joblib==1.3.2 @@ -226,6 +231,8 @@ mock==5.1.0 # via -r dev-requirements.in more-itertools==10.1.0 # via jaraco-classes +mpmath==1.3.0 + # via sympy msal==1.24.1 # via # azure-datalake-store @@ -246,6 +253,8 @@ mypy-extensions==1.0.0 # typing-inspect natsort==8.4.0 # via flytekit +networkx==3.2.1 + # via torch nodeenv==1.8.0 # via pre-commit numpy==1.26.1 @@ -375,8 +384,12 @@ sortedcontainers==2.4.0 # via flytekit statsd==3.3.0 # via flytekit +sympy==1.12 + # via torch text-unidecode==1.3 # via python-slugify +torch==2.1.1 + # via -r dev-requirements.in types-python-dateutil==2.8.19.14 # via arrow typing-extensions==4.8.0 @@ -387,6 +400,7 @@ typing-extensions==4.8.0 # mashumaro # mypy # rich-click + # torch # typing-inspect typing-inspect==0.9.0 # via dataclasses-json From 3bf2d1b31a86232c7eb0e96b90b2c1f33a584ac7 Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Sat, 18 Nov 2023 16:16:04 +0530 Subject: [PATCH 06/33] add torch Signed-off-by: Samhita Alla --- .github/workflows/checks.yml | 2 +- dev-requirements.in | 1 - dev-requirements.txt | 18 ++---------------- 3 files changed, 3 insertions(+), 18 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 886b18838..40fb289a9 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -212,7 +212,7 @@ jobs: - name: Install Python dependencies run: | python -m pip install --upgrade pip - pip install flytekit flytekitplugins-deck-standard + pip install flytekit flytekitplugins-deck-standard torch pip freeze - name: Checkout flytesnacks uses: actions/checkout@v3 diff --git a/dev-requirements.in b/dev-requirements.in index 72f0544d7..996558ba2 100644 --- a/dev-requirements.in +++ b/dev-requirements.in @@ -11,4 +11,3 @@ mock pytest mypy mashumaro -torch diff --git a/dev-requirements.txt b/dev-requirements.txt index 922882a43..174553f41 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -101,9 +101,7 @@ docker-image-py==0.1.12 docstring-parser==0.15 # via flytekit filelock==3.13.1 - # via - # torch - # virtualenv + # via virtualenv flake8==6.1.0 # via # -r dev-requirements.in @@ -127,7 +125,6 @@ fsspec==2023.10.0 # flytekit # gcsfs # s3fs - # torch gcsfs==2023.10.0 # via flytekit gitdb==4.0.11 @@ -191,9 +188,7 @@ isort==5.12.0 jaraco-classes==3.3.0 # via keyring jinja2==3.1.2 - # via - # cookiecutter - # torch + # via cookiecutter jmespath==1.0.1 # via botocore joblib==1.3.2 @@ -231,8 +226,6 @@ mock==5.1.0 # via -r dev-requirements.in more-itertools==10.1.0 # via jaraco-classes -mpmath==1.3.0 - # via sympy msal==1.24.1 # via # azure-datalake-store @@ -253,8 +246,6 @@ mypy-extensions==1.0.0 # typing-inspect natsort==8.4.0 # via flytekit -networkx==3.2.1 - # via torch nodeenv==1.8.0 # via pre-commit numpy==1.26.1 @@ -384,12 +375,8 @@ sortedcontainers==2.4.0 # via flytekit statsd==3.3.0 # via flytekit -sympy==1.12 - # via torch text-unidecode==1.3 # via python-slugify -torch==2.1.1 - # via -r dev-requirements.in types-python-dateutil==2.8.19.14 # via arrow typing-extensions==4.8.0 @@ -400,7 +387,6 @@ typing-extensions==4.8.0 # mashumaro # mypy # rich-click - # torch # typing-inspect typing-inspect==0.9.0 # via dataclasses-json From fbcab15bee469b572c560d9fd0abb766653d3d7a Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Tue, 28 Nov 2023 18:30:28 +0530 Subject: [PATCH 07/33] validate tests Signed-off-by: Samhita Alla --- .github/workflows/checks.yml | 9 +- .../advanced_composition/merge_sort.py | 2 + .../advanced_composition/subworkflows.py | 2 + flyte_tests.txt | 17 +++ flyte_tests_generator.py | 14 --- flyte_tests_manifest.json | 108 ++++++++++++------ flyte_tests_validate.py | 60 ++++++++++ 7 files changed, 156 insertions(+), 56 deletions(-) create mode 100644 flyte_tests.txt delete mode 100644 flyte_tests_generator.py create mode 100644 flyte_tests_validate.py diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 40fb289a9..a2d49afbe 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -147,7 +147,6 @@ jobs: for i in */; do tar -czvf "../release-snacks/${i%/}.tar.gz" "$i" & done; wait cd .. && sudo rm -rf download-artifact/ cp flyte_tests_manifest.json release-snacks/flyte_tests_manifest.json - python flyte_tests_generator.py >> flyte_tests.txt cp flyte_tests.txt release-snacks/flyte_tests.txt - name: Release test manifest uses: goreleaser/goreleaser-action@v2 @@ -219,8 +218,9 @@ jobs: with: repository: flyteorg/flytesnacks path: flytesnacks - - name: Generate tests - run: python flyte_tests_generator.py >> flyte_tests.txt + - name: Validate if the tests exist + run: | + python flyte_tests_validate.py - name: Register specific tests run: | while read -r line; @@ -234,6 +234,3 @@ jobs: --version ${{ env.FLYTESNACKS_VERSION }} \ flytesnacks/$line; done < flyte_tests.txt - - name: End2End - run: | - make end2end_execute diff --git a/examples/advanced_composition/advanced_composition/merge_sort.py b/examples/advanced_composition/advanced_composition/merge_sort.py index 342ff3306..55d28e194 100644 --- a/examples/advanced_composition/advanced_composition/merge_sort.py +++ b/examples/advanced_composition/advanced_composition/merge_sort.py @@ -29,6 +29,7 @@ # %% [markdown] # A simple split function that divides a list into two halves. + # %% @task def split(numbers: typing.List[int]) -> Tuple[typing.List[int], typing.List[int], int, int]: @@ -65,6 +66,7 @@ def merge(sorted_list1: typing.List[int], sorted_list2: typing.List[int]) -> typ # Generally speaking, the algorithm will recurse through the list, splitting it in half until it reaches a size that we # know is efficient enough to run locally. At which point it'll just use the python-builtin sorted function. + # %% [markdown] # This runs the sorting completely locally. It's faster and more efficient to do so if the entire list fits in memory. # %% diff --git a/examples/advanced_composition/advanced_composition/subworkflows.py b/examples/advanced_composition/advanced_composition/subworkflows.py index 07f4efa7e..be4eff792 100644 --- a/examples/advanced_composition/advanced_composition/subworkflows.py +++ b/examples/advanced_composition/advanced_composition/subworkflows.py @@ -87,6 +87,7 @@ def parent_wf(a: int) -> Tuple[int, str, str]: if __name__ == "__main__": print(f"Running parent_wf(a=3) {parent_wf(a=3)}") + # %% [markdown] # Interestingly, we can nest a workflow that has a subworkflow within a workflow. # Workflows can be simply composed from other workflows, even if they are standalone entities. Each of the @@ -164,6 +165,7 @@ def ext_workflow(my_input: str) -> Dict: "parent_workflow_execution", ) + # %% [markdown] # Define another task that returns the repeated keys (in our case, words) from a dictionary. # %% diff --git a/flyte_tests.txt b/flyte_tests.txt new file mode 100644 index 000000000..5de785ff6 --- /dev/null +++ b/flyte_tests.txt @@ -0,0 +1,17 @@ +examples/advanced_composition/advanced_composition/chain_entities.py +examples/advanced_composition/advanced_composition/conditions.py +examples/advanced_composition/advanced_composition/decorating_tasks.py +examples/advanced_composition/advanced_composition/decorating_workflows.py +examples/advanced_composition/advanced_composition/dynamics.py +examples/advanced_composition/advanced_composition/map_task.py +examples/advanced_composition/advanced_composition/waiting_for_external_inputs.py +examples/basics/basics/documenting_workflows.py +examples/basics/basics/hello_world.py +examples/basics/basics/named_outputs.py +examples/basics/basics/shell_task.py +examples/basics/basics/workflow.py +examples/data_types_and_io/data_types_and_io/dataclass.py +examples/data_types_and_io/data_types_and_io/enum_type.py +examples/data_types_and_io/data_types_and_io/file.py +examples/data_types_and_io/data_types_and_io/folder.py +examples/data_types_and_io/data_types_and_io/structured_dataset.py diff --git a/flyte_tests_generator.py b/flyte_tests_generator.py deleted file mode 100644 index dc35fb135..000000000 --- a/flyte_tests_generator.py +++ /dev/null @@ -1,14 +0,0 @@ -import glob -import itertools - -directories_to_walk = [ - "examples/basics/basics", - "examples/data_types_and_io/data_types_and_io", - "examples/advanced_composition/advanced_composition", -] - -all_tests = [] -for each_directory in directories_to_walk: - all_tests.append(glob.glob(f"{each_directory}/*.py")) - -print("\n".join(list(itertools.chain(*all_tests)))) diff --git a/flyte_tests_manifest.json b/flyte_tests_manifest.json index b0a756d9d..8e5590c37 100644 --- a/flyte_tests_manifest.json +++ b/flyte_tests_manifest.json @@ -3,6 +3,57 @@ "name": "core", "priority": "P0", "path": "core", + "examples": [ + ["advanced_composition.chain_entities.chain_workflows_wf", {}], + ["advanced_composition.conditions.consume_outputs", { "my_input": 10.0 }], + ["advanced_composition.decorating_tasks.wf", { "x": 10 }], + ["advanced_composition.decorating_workflows.wf", { "x": 19.8 }], + ["advanced_composition.dynamics.wf", { "s1": "Pear", "s2": "Earth" }], + [ + "advanced_composition.map_task.my_map_workflow", + { "a": [1, 2, 3, 4, 5] } + ], + [ + "advanced_composition.waiting_for_external_inputs.sleep_wf", + { "num": 5 } + ], + ["basics.documenting_workflows.sphinx_docstring_wf", {}], + ["basics.hello_world.hello_world_wf", {}], + ["basics.named_outputs.simple_wf_with_named_outputs", {}], + ["basics.shell_task.shell_task_wf", {}], + ["basics.workflow.simple_wf", { "x": [1, 2, 3], "y": [1, 2, 3] }], + ["data_types_and_io.dataclass.dataclass_wf", { "x": 10, "y": 20 }], + ["data_types_and_io.enum_type.coffee_maker", { "coffee": "latte" }], + [ + "data_types_and_io.file.normalize_csv_file", + { + "csv_url": "https://people.sc.fsu.edu/~jburkardt/data/csv/biostats.csv", + "column_names": [ + "Name", + "Sex", + "Age", + "Heights (in)", + "Weight (lbs)" + ], + "columns_to_normalize": ["Age"] + } + ], + [ + "data_types_and_io.folder.download_and_normalize_csv_files", + { + "csv_urls": [ + "https://people.sc.fsu.edu/~jburkardt/data/csv/biostats.csv", + "https://people.sc.fsu.edu/~jburkardt/data/csv/faithful.csv" + ], + "columns_metadata": [ + ["Name", "Sex", "Age", "Heights (in)", "Weight (lbs)"], + ["Index", "Eruption length (mins)", "Eruption wait (mins)"] + ], + "columns_to_normalize_metadata": [["Age"], ["Eruption length (mins)"]] + } + ], + ["data_types_and_io.structured_dataset.simple_sd_wf", { "a": 42 }] + ], "exitCondition": { "exit_success": true, "exit_message": "" @@ -21,6 +72,12 @@ "name": "integrations-k8s-spark", "priority": "P1", "path": "examples/k8s_spark_plugin", + "examples": [ + [ + "k8s_spark_plugin.pyspark_pi.my_spark", + { "triggered_date": "2023-11-21T18:58:01" } + ] + ], "exitCondition": { "exit_success": true, "exit_message": "" @@ -30,6 +87,7 @@ "name": "integrations-kfpytorch", "priority": "P1", "path": "examples/kfpytorch_plugin", + "examples": [["kfpytorch_plugin.pytorch_mnist.pytorch_training_wf", {}]], "exitCondition": { "exit_success": true, "exit_message": "" @@ -39,15 +97,9 @@ "name": "integrations-kftensorflow", "priority": "P1", "path": "examples/kftensorflow_plugin", - "exitCondition": { - "exit_success": true, - "exit_message": "" - } - }, - { - "name": "integrations-pod", - "priority": "P1", - "path": "examples/k8s_pod_plugin", + "examples": [ + ["kftensorflow_plugin.tf_mnist.mnist_tensorflow_workflow", {}] + ], "exitCondition": { "exit_success": true, "exit_message": "" @@ -57,15 +109,7 @@ "name": "integrations-pandera", "priority": "P1", "path": "examples/pandera_plugin", - "exitCondition": { - "exit_success": true, - "exit_message": "" - } - }, - { - "name": "integrations-whylogs", - "priority": "P1", - "path": "examples/whylogs_plugin", + "examples": [["pandera_plugin.basic_schema_example.process_data", {}]], "exitCondition": { "exit_success": true, "exit_message": "" @@ -75,6 +119,7 @@ "name": "integrations-modin", "priority": "P1", "path": "examples/modin_plugin", + "examples": [["modin_plugin.knn_classifier.pipeline", {}]], "exitCondition": { "exit_success": true, "exit_message": "" @@ -84,6 +129,9 @@ "name": "integrations-papermill", "priority": "P1", "path": "examples/papermill_plugin", + "examples": [ + ["papermill_plugin.simple.nb_to_python_wf", { "f": 3.1415926535 }] + ], "exitCondition": { "exit_success": true, "exit_message": "" @@ -93,24 +141,12 @@ "name": "integrations-greatexpectations", "priority": "P1", "path": "examples/greatexpectations_plugin", - "exitCondition": { - "exit_success": true, - "exit_message": "" - } - }, - { - "name": "integrations-sagemaker-pytorch", - "priority": "P1", - "path": "examples/sagemaker_pytorch_plugin", - "exitCondition": { - "exit_success": true, - "exit_message": "" - } - }, - { - "name": "integrations-sagemaker-training", - "priority": "P1", - "path": "examples/sagemaker_training_plugin", + "examples": [ + ["greatexpectations_plugin.task_example.simple_wf", {}], + ["greatexpectations_plugin.task_example.file_wf", {}], + ["greatexpectations_plugin.task_example.schema_wf", {}], + ["greatexpectations_plugin.task_example.runtime_wf", {}] + ], "exitCondition": { "exit_success": true, "exit_message": "" diff --git a/flyte_tests_validate.py b/flyte_tests_validate.py new file mode 100644 index 000000000..c66c77384 --- /dev/null +++ b/flyte_tests_validate.py @@ -0,0 +1,60 @@ +import json +import os +import subprocess +import re + +file_list = "flyte_tests.txt" + +with open("flyte_tests_manifest.json", "r") as file: + data = json.load(file) + +examples = [(example[0], example[1]) for entry in data for example in entry.get("examples", []) if len(example) >= 1] + +for file_name in open(file_list, "r").readlines(): + file_name = file_name.strip() + print(f"Processing file: {file_name}") + + # Retrieve the file path, including the name of the file and its immediate parent directory + directory_path = os.path.dirname(file_name).split(os.path.sep)[-1:] + file_path = ".".join(directory_path + [os.path.splitext(os.path.basename(file_name))[0]]) + + # Verify if there are any workflows present in the provided file path + workflows = [] + for workflow, params in examples: + if file_path in workflow: + workflows.append((workflow, params)) + else: + raise Exception("No workflows are defined in the given file.") + + for workflow, params in workflows: + # Use the `pyflyte run` command to execute the workflow + output_string = subprocess.run(["pyflyte", "run", file_name], capture_output=True, text=True).stdout + + # Define a regular expression pattern to match tasks/workflows in the pyflyte run output + pattern = re.compile(r"^\│\s+(\w+)\s+", re.MULTILINE) + + # Extract command names using the specified pattern + commands = re.findall(pattern, output_string) + + # Check if the workflow specified is present in the pyflyte run output + just_the_workflow = workflow.split(".")[2] + if just_the_workflow in commands: + print("Workflow found in the pyflyte run output!") + else: + raise Exception("Workflow not found in the pyflyte run output.") + + # Check if the specified parameters are valid + options_output = subprocess.run( + ["pyflyte", "run", file_name, just_the_workflow, "--help"], capture_output=True, text=True + ).stdout + + # Find all matches in the input string + options = [option.replace("--", "") for option in re.compile(r"--\w+").findall(options_output)] + + # Validate if the provided params are a subset of the supported params + if set(params).issubset(set(options)): + print("All parameters found!") + else: + raise Exception( + f"There's a mismatch between the values accepted by the workflow and the ones you provided." + ) From 36ecc7fc77d1eda0787dc0b07b5030ea5a7b5660 Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Wed, 29 Nov 2023 22:18:21 +0530 Subject: [PATCH 08/33] update validation logic Signed-off-by: Samhita Alla --- .github/workflows/checks.yml | 2 +- flyte_tests_validate.py | 21 +++++++++------------ 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index a2d49afbe..9b733b91c 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -218,7 +218,7 @@ jobs: with: repository: flyteorg/flytesnacks path: flytesnacks - - name: Validate if the tests exist + - name: Verify the existence of the tests run: | python flyte_tests_validate.py - name: Register specific tests diff --git a/flyte_tests_validate.py b/flyte_tests_validate.py index c66c77384..295ef577f 100644 --- a/flyte_tests_validate.py +++ b/flyte_tests_validate.py @@ -1,7 +1,7 @@ import json import os -import subprocess import re +import subprocess file_list = "flyte_tests.txt" @@ -18,13 +18,12 @@ directory_path = os.path.dirname(file_name).split(os.path.sep)[-1:] file_path = ".".join(directory_path + [os.path.splitext(os.path.basename(file_name))[0]]) + # Retrieve the workflow(s) + workflows = list(filter(lambda tup: file_path in tup[0], examples)) + # Verify if there are any workflows present in the provided file path - workflows = [] - for workflow, params in examples: - if file_path in workflow: - workflows.append((workflow, params)) - else: - raise Exception("No workflows are defined in the given file.") + if not workflows: + raise Exception("The file does not contain any workflows.") for workflow, params in workflows: # Use the `pyflyte run` command to execute the workflow @@ -39,7 +38,7 @@ # Check if the workflow specified is present in the pyflyte run output just_the_workflow = workflow.split(".")[2] if just_the_workflow in commands: - print("Workflow found in the pyflyte run output!") + print("Workflow found in the pyflyte run output.") else: raise Exception("Workflow not found in the pyflyte run output.") @@ -53,8 +52,6 @@ # Validate if the provided params are a subset of the supported params if set(params).issubset(set(options)): - print("All parameters found!") + print("All parameters found.") else: - raise Exception( - f"There's a mismatch between the values accepted by the workflow and the ones you provided." - ) + raise Exception("There's a mismatch between the values accepted by the workflow and the ones you provided.") From 8bfe2d9d5adfc10c3084cc9dd0028c636093f590 Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Wed, 29 Nov 2023 22:23:03 +0530 Subject: [PATCH 09/33] lint Signed-off-by: Samhita Alla --- docs/getting_started/run_schedule.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/getting_started/run_schedule.md b/docs/getting_started/run_schedule.md index d8d650a7a..3aaca6625 100644 --- a/docs/getting_started/run_schedule.md +++ b/docs/getting_started/run_schedule.md @@ -73,7 +73,7 @@ environment, you can import and execute it directly: Before execute it directly, you need to register the workflow first. ```{prompt} bash $ -pyflyte register wf.py +pyflyte register wf.py ``` ```{code-block} python @@ -347,4 +347,4 @@ In this guide, you learned about how to: - Run tasks, workflows, and launch plans using `FlyteRemote`. - Create a cron schedule to run a launch plan at a specified time interval. -In the next guide, you'll learn how to visualize tasks using Flyte Decks. \ No newline at end of file +In the next guide, you'll learn how to visualize tasks using Flyte Decks. From 910ecb9b008b6cd8ea22f94b8806177ec9bbc643 Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Thu, 30 Nov 2023 21:24:00 +0530 Subject: [PATCH 10/33] debug Signed-off-by: Samhita Alla --- flyte_tests_validate.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flyte_tests_validate.py b/flyte_tests_validate.py index 295ef577f..e11d1b516 100644 --- a/flyte_tests_validate.py +++ b/flyte_tests_validate.py @@ -37,6 +37,9 @@ # Check if the workflow specified is present in the pyflyte run output just_the_workflow = workflow.split(".")[2] + print(workflow) + print(just_the_workflow) + print(commands) if just_the_workflow in commands: print("Workflow found in the pyflyte run output.") else: From e5d1285cb297265e87d652cc524dfc53151a3135 Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Thu, 30 Nov 2023 21:39:12 +0530 Subject: [PATCH 11/33] debug Signed-off-by: Samhita Alla --- flyte_tests_validate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flyte_tests_validate.py b/flyte_tests_validate.py index e11d1b516..81cc31ec8 100644 --- a/flyte_tests_validate.py +++ b/flyte_tests_validate.py @@ -27,7 +27,9 @@ for workflow, params in workflows: # Use the `pyflyte run` command to execute the workflow + print(file_name) output_string = subprocess.run(["pyflyte", "run", file_name], capture_output=True, text=True).stdout + print(output_string) # Define a regular expression pattern to match tasks/workflows in the pyflyte run output pattern = re.compile(r"^\│\s+(\w+)\s+", re.MULTILINE) @@ -37,8 +39,6 @@ # Check if the workflow specified is present in the pyflyte run output just_the_workflow = workflow.split(".")[2] - print(workflow) - print(just_the_workflow) print(commands) if just_the_workflow in commands: print("Workflow found in the pyflyte run output.") From c1e0d087baf2db765d9b7b407ef0945b588956f9 Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Thu, 30 Nov 2023 21:56:18 +0530 Subject: [PATCH 12/33] debug Signed-off-by: Samhita Alla --- flyte_tests_validate.py | 10 +++++++--- test.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 test.py diff --git a/flyte_tests_validate.py b/flyte_tests_validate.py index 81cc31ec8..8dcfbe41a 100644 --- a/flyte_tests_validate.py +++ b/flyte_tests_validate.py @@ -27,9 +27,7 @@ for workflow, params in workflows: # Use the `pyflyte run` command to execute the workflow - print(file_name) output_string = subprocess.run(["pyflyte", "run", file_name], capture_output=True, text=True).stdout - print(output_string) # Define a regular expression pattern to match tasks/workflows in the pyflyte run output pattern = re.compile(r"^\│\s+(\w+)\s+", re.MULTILINE) @@ -37,9 +35,15 @@ # Extract command names using the specified pattern commands = re.findall(pattern, output_string) + print(pattern) + print(output_string) + print(type(output_string)) + print(re.findall(pattern, output_string)) + print(commands) + # Check if the workflow specified is present in the pyflyte run output just_the_workflow = workflow.split(".")[2] - print(commands) + if just_the_workflow in commands: print("Workflow found in the pyflyte run output.") else: diff --git a/test.py b/test.py new file mode 100644 index 000000000..3594b501a --- /dev/null +++ b/test.py @@ -0,0 +1,37 @@ +import re + +s = """ + + Usage: + pyflyte run + examples/advanced_composition/advanced_composition/chain_entities.py + [OPTIONS] COMMAND [ARGS]... + + Run a [workflow|task] from + examples/advanced_composition/advanced_composition/chain_entities.py + +╭─ Options ────────────────────────────────────────────────────────────────────╮ +│ --help Show this message and exit. │ +╰──────────────────────────────────────────────────────────────────────────────╯ +╭─ Commands ───────────────────────────────────────────────────────────────────╮ +│ chain_tasks_wf Workflow │ +│ (advanced_composition.chain_entities.chain_tasks_wf) │ +│ chain_workflows_wf Workflow │ +│ (advanced_composition.chain_entities.chain_workflows_wf) │ +│ sub_workflow_0 Workflow │ +│ (advanced_composition.chain_entities.sub_workflow_0) │ +│ sub_workflow_1 Workflow │ +│ (advanced_composition.chain_entities.sub_workflow_1) │ +│ t0 Task (advanced_composition.chain_entities.t0) │ +│ t1 Task (advanced_composition.chain_entities.t1) │ +│ t2 Task (advanced_composition.chain_entities.t2) │ +╰──────────────────────────────────────────────────────────────────────────────╯ + + +""" +# Define a regular expression pattern to match tasks/workflows in the pyflyte run output +pattern = re.compile(r"^\│\s+(\w+)\s+", re.MULTILINE) + +# Extract command names using the specified pattern +commands = re.findall(pattern, s) +print(commands) From 3d045814dce4de3081da9c4aa9c9ade2ff01ee20 Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Thu, 30 Nov 2023 21:56:34 +0530 Subject: [PATCH 13/33] debug Signed-off-by: Samhita Alla --- test.py | 37 ------------------------------------- 1 file changed, 37 deletions(-) delete mode 100644 test.py diff --git a/test.py b/test.py deleted file mode 100644 index 3594b501a..000000000 --- a/test.py +++ /dev/null @@ -1,37 +0,0 @@ -import re - -s = """ - - Usage: - pyflyte run - examples/advanced_composition/advanced_composition/chain_entities.py - [OPTIONS] COMMAND [ARGS]... - - Run a [workflow|task] from - examples/advanced_composition/advanced_composition/chain_entities.py - -╭─ Options ────────────────────────────────────────────────────────────────────╮ -│ --help Show this message and exit. │ -╰──────────────────────────────────────────────────────────────────────────────╯ -╭─ Commands ───────────────────────────────────────────────────────────────────╮ -│ chain_tasks_wf Workflow │ -│ (advanced_composition.chain_entities.chain_tasks_wf) │ -│ chain_workflows_wf Workflow │ -│ (advanced_composition.chain_entities.chain_workflows_wf) │ -│ sub_workflow_0 Workflow │ -│ (advanced_composition.chain_entities.sub_workflow_0) │ -│ sub_workflow_1 Workflow │ -│ (advanced_composition.chain_entities.sub_workflow_1) │ -│ t0 Task (advanced_composition.chain_entities.t0) │ -│ t1 Task (advanced_composition.chain_entities.t1) │ -│ t2 Task (advanced_composition.chain_entities.t2) │ -╰──────────────────────────────────────────────────────────────────────────────╯ - - -""" -# Define a regular expression pattern to match tasks/workflows in the pyflyte run output -pattern = re.compile(r"^\│\s+(\w+)\s+", re.MULTILINE) - -# Extract command names using the specified pattern -commands = re.findall(pattern, s) -print(commands) From ebe48200926255499da7ce6ce5afe32a74779aba Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Thu, 30 Nov 2023 22:05:17 +0530 Subject: [PATCH 14/33] debug Signed-off-by: Samhita Alla --- .github/workflows/checks.yml | 1 + test.py | 37 ++++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 test.py diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 9b733b91c..b2c187270 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -220,6 +220,7 @@ jobs: path: flytesnacks - name: Verify the existence of the tests run: | + python test.py python flyte_tests_validate.py - name: Register specific tests run: | diff --git a/test.py b/test.py new file mode 100644 index 000000000..aff5a955b --- /dev/null +++ b/test.py @@ -0,0 +1,37 @@ +import re + +s = """ + + Usage: + pyflyte run + examples/advanced_composition/advanced_composition/chain_entities.py + [OPTIONS] COMMAND [ARGS]... + + Run a [workflow|task] from + examples/advanced_composition/advanced_composition/chain_entities.py + +╭─ Options ────────────────────────────────────────────────────────────────────╮ +│ --help Show this message and exit. │ +╰──────────────────────────────────────────────────────────────────────────────╯ +╭─ Commands ───────────────────────────────────────────────────────────────────╮ +│ chain_tasks_wf Workflow │ +│ (advanced_composition.chain_entities.chain_tasks_wf) │ +│ chain_workflows_wf Workflow │ +│ (advanced_composition.chain_entities.chain_workflows_wf) │ +│ sub_workflow_0 Workflow │ +│ (advanced_composition.chain_entities.sub_workflow_0) │ +│ sub_workflow_1 Workflow │ +│ (advanced_composition.chain_entities.sub_workflow_1) │ +│ t0 Task (advanced_composition.chain_entities.t0) │ +│ t1 Task (advanced_composition.chain_entities.t1) │ +│ t2 Task (advanced_composition.chain_entities.t2) │ +╰──────────────────────────────────────────────────────────────────────────────╯ + + +""" +# Define a regular expression pattern to match tasks/workflows in the pyflyte run output +pattern = re.compile(r"^\│\s+(\w+)\s+", re.MULTILINE) + +# Extract command names using the specified pattern +commands = re.findall(pattern, s) +print(commands) From 66087771633c0f80578f99c51e8e57ad396e4cdb Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Thu, 30 Nov 2023 22:20:08 +0530 Subject: [PATCH 15/33] debug Signed-off-by: Samhita Alla --- flyte_tests_validate.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/flyte_tests_validate.py b/flyte_tests_validate.py index 8dcfbe41a..3c330070c 100644 --- a/flyte_tests_validate.py +++ b/flyte_tests_validate.py @@ -27,18 +27,13 @@ for workflow, params in workflows: # Use the `pyflyte run` command to execute the workflow - output_string = subprocess.run(["pyflyte", "run", file_name], capture_output=True, text=True).stdout + output_string = subprocess.run(["pyflyte", "run", file_name], capture_output=True, text=True).stdout.strip() # Define a regular expression pattern to match tasks/workflows in the pyflyte run output pattern = re.compile(r"^\│\s+(\w+)\s+", re.MULTILINE) # Extract command names using the specified pattern commands = re.findall(pattern, output_string) - - print(pattern) - print(output_string) - print(type(output_string)) - print(re.findall(pattern, output_string)) print(commands) # Check if the workflow specified is present in the pyflyte run output From 9711de35fe8be9301ad977cd0a871eee5ecb08cd Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Thu, 30 Nov 2023 22:32:43 +0530 Subject: [PATCH 16/33] modify regex Signed-off-by: Samhita Alla --- flyte_tests_validate.py | 3 ++- test.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/flyte_tests_validate.py b/flyte_tests_validate.py index 3c330070c..4acab437d 100644 --- a/flyte_tests_validate.py +++ b/flyte_tests_validate.py @@ -30,10 +30,11 @@ output_string = subprocess.run(["pyflyte", "run", file_name], capture_output=True, text=True).stdout.strip() # Define a regular expression pattern to match tasks/workflows in the pyflyte run output - pattern = re.compile(r"^\│\s+(\w+)\s+", re.MULTILINE) + pattern = re.compile(r"(?<=│ )(\w+)(?=\s+(?:Workflow|Task))") # Extract command names using the specified pattern commands = re.findall(pattern, output_string) + print(output_string) print(commands) # Check if the workflow specified is present in the pyflyte run output diff --git a/test.py b/test.py index aff5a955b..806d351e2 100644 --- a/test.py +++ b/test.py @@ -30,7 +30,7 @@ """ # Define a regular expression pattern to match tasks/workflows in the pyflyte run output -pattern = re.compile(r"^\│\s+(\w+)\s+", re.MULTILINE) +pattern = re.compile(r"(?<=│ )(\w+)(?=\s+(?:Workflow|Task))") # Extract command names using the specified pattern commands = re.findall(pattern, s) From acf36b3a9b62dab56cb19b0a6d96c565aa870597 Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Fri, 1 Dec 2023 18:31:29 +0530 Subject: [PATCH 17/33] debug Signed-off-by: Samhita Alla --- flyte_tests_validate.py | 18 +++++++---- test.py | 68 ++++++++++++++++++++--------------------- 2 files changed, 45 insertions(+), 41 deletions(-) diff --git a/flyte_tests_validate.py b/flyte_tests_validate.py index 4acab437d..917a0e2a3 100644 --- a/flyte_tests_validate.py +++ b/flyte_tests_validate.py @@ -10,6 +10,16 @@ examples = [(example[0], example[1]) for entry in data for example in entry.get("examples", []) if len(example) >= 1] + +def get_commands(output): + # Find lines starting with '|' + command_lines = re.findall(r"│\s*([a-zA-Z0-9_\-\.]+)\s+", output) + + # Filter out 'Commands' and "--help" lines + commands = [cmd for cmd in command_lines if cmd.lower() != "commands" and cmd.lower() != "--help"] + return commands + + for file_name in open(file_list, "r").readlines(): file_name = file_name.strip() print(f"Processing file: {file_name}") @@ -27,13 +37,9 @@ for workflow, params in workflows: # Use the `pyflyte run` command to execute the workflow - output_string = subprocess.run(["pyflyte", "run", file_name], capture_output=True, text=True).stdout.strip() - - # Define a regular expression pattern to match tasks/workflows in the pyflyte run output - pattern = re.compile(r"(?<=│ )(\w+)(?=\s+(?:Workflow|Task))") + output_string = subprocess.run(["pyflyte", "run", file_name], capture_output=True, text=True).stdout - # Extract command names using the specified pattern - commands = re.findall(pattern, output_string) + commands = get_commands(output_string) print(output_string) print(commands) diff --git a/test.py b/test.py index 806d351e2..2cb570f13 100644 --- a/test.py +++ b/test.py @@ -1,37 +1,35 @@ import re -s = """ - - Usage: - pyflyte run - examples/advanced_composition/advanced_composition/chain_entities.py - [OPTIONS] COMMAND [ARGS]... - - Run a [workflow|task] from - examples/advanced_composition/advanced_composition/chain_entities.py - -╭─ Options ────────────────────────────────────────────────────────────────────╮ -│ --help Show this message and exit. │ -╰──────────────────────────────────────────────────────────────────────────────╯ -╭─ Commands ───────────────────────────────────────────────────────────────────╮ -│ chain_tasks_wf Workflow │ -│ (advanced_composition.chain_entities.chain_tasks_wf) │ -│ chain_workflows_wf Workflow │ -│ (advanced_composition.chain_entities.chain_workflows_wf) │ -│ sub_workflow_0 Workflow │ -│ (advanced_composition.chain_entities.sub_workflow_0) │ -│ sub_workflow_1 Workflow │ -│ (advanced_composition.chain_entities.sub_workflow_1) │ -│ t0 Task (advanced_composition.chain_entities.t0) │ -│ t1 Task (advanced_composition.chain_entities.t1) │ -│ t2 Task (advanced_composition.chain_entities.t2) │ -╰──────────────────────────────────────────────────────────────────────────────╯ - - -""" -# Define a regular expression pattern to match tasks/workflows in the pyflyte run output -pattern = re.compile(r"(?<=│ )(\w+)(?=\s+(?:Workflow|Task))") - -# Extract command names using the specified pattern -commands = re.findall(pattern, s) -print(commands) + +def get_commands(output): + # Find lines starting with '|' + command_lines = re.findall(r"│\s*([a-zA-Z0-9_\-\.]+)\s+", output) + + # Filter out empty strings and 'Commands' line + commands = [cmd for cmd in command_lines if cmd.lower() != "commands" and cmd.lower() != "--help"] + return commands + + +if __name__ == "__main__": + # Example string output + example_output = """ + Usage: pyflyte run examples/advanced_composition/advanced_composition/chain_entities.py [OPTIONS] COMMAND [ARGS]... + + Run a [workflow|task] from examples/advanced_composition/advanced_composition/chain_entities.py + + ╭─ Options ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ + │ --help Show this message and exit. │ + ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + ╭─ Commands ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ + │ chain_tasks_wf Workflow (advanced_composition.chain_entities.chain_tasks_wf) │ + │ chain_workflows_wf Workflow (advanced_composition.chain_entities.chain_workflows_wf) │ + │ sub_workflow_0 Workflow (advanced_composition.chain_entities.sub_workflow_0) │ + │ sub_workflow_1 Workflow (advanced_composition.chain_entities.sub_workflow_1) │ + │ t0 Task (advanced_composition.chain_entities.t0) │ + │ t1 Task (advanced_composition.chain_entities.t1) │ + │ t2 Task (advanced_composition.chain_entities.t2) │ + ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + """ + + commands = get_commands(example_output) + print(commands) From 64f4dac45f2456252fb6fcdbf7ef2c19d058cb7c Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Fri, 1 Dec 2023 18:45:57 +0530 Subject: [PATCH 18/33] remove re Signed-off-by: Samhita Alla --- .github/workflows/checks.yml | 2 +- flyte_tests_validate.py | 7 +------ test.py | 35 ----------------------------------- 3 files changed, 2 insertions(+), 42 deletions(-) delete mode 100644 test.py diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index b2c187270..887eb4a44 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -218,7 +218,7 @@ jobs: with: repository: flyteorg/flytesnacks path: flytesnacks - - name: Verify the existence of the tests + - name: Verify existence of the tests run: | python test.py python flyte_tests_validate.py diff --git a/flyte_tests_validate.py b/flyte_tests_validate.py index 917a0e2a3..7079acb2f 100644 --- a/flyte_tests_validate.py +++ b/flyte_tests_validate.py @@ -39,14 +39,9 @@ def get_commands(output): # Use the `pyflyte run` command to execute the workflow output_string = subprocess.run(["pyflyte", "run", file_name], capture_output=True, text=True).stdout - commands = get_commands(output_string) - print(output_string) - print(commands) - # Check if the workflow specified is present in the pyflyte run output just_the_workflow = workflow.split(".")[2] - - if just_the_workflow in commands: + if " " + just_the_workflow + " " in output_string: print("Workflow found in the pyflyte run output.") else: raise Exception("Workflow not found in the pyflyte run output.") diff --git a/test.py b/test.py deleted file mode 100644 index 2cb570f13..000000000 --- a/test.py +++ /dev/null @@ -1,35 +0,0 @@ -import re - - -def get_commands(output): - # Find lines starting with '|' - command_lines = re.findall(r"│\s*([a-zA-Z0-9_\-\.]+)\s+", output) - - # Filter out empty strings and 'Commands' line - commands = [cmd for cmd in command_lines if cmd.lower() != "commands" and cmd.lower() != "--help"] - return commands - - -if __name__ == "__main__": - # Example string output - example_output = """ - Usage: pyflyte run examples/advanced_composition/advanced_composition/chain_entities.py [OPTIONS] COMMAND [ARGS]... - - Run a [workflow|task] from examples/advanced_composition/advanced_composition/chain_entities.py - - ╭─ Options ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ - │ --help Show this message and exit. │ - ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - ╭─ Commands ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ - │ chain_tasks_wf Workflow (advanced_composition.chain_entities.chain_tasks_wf) │ - │ chain_workflows_wf Workflow (advanced_composition.chain_entities.chain_workflows_wf) │ - │ sub_workflow_0 Workflow (advanced_composition.chain_entities.sub_workflow_0) │ - │ sub_workflow_1 Workflow (advanced_composition.chain_entities.sub_workflow_1) │ - │ t0 Task (advanced_composition.chain_entities.t0) │ - │ t1 Task (advanced_composition.chain_entities.t1) │ - │ t2 Task (advanced_composition.chain_entities.t2) │ - ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - """ - - commands = get_commands(example_output) - print(commands) From 015f40d70a5330fcd76edbf185b38d2572e01bce Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Fri, 1 Dec 2023 18:50:43 +0530 Subject: [PATCH 19/33] remove test.py Signed-off-by: Samhita Alla --- .github/workflows/checks.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 887eb4a44..9fcd5c174 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -220,7 +220,6 @@ jobs: path: flytesnacks - name: Verify existence of the tests run: | - python test.py python flyte_tests_validate.py - name: Register specific tests run: | From 30ce82559060908be2486f14a8d1cd6793efbe6a Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Fri, 1 Dec 2023 18:56:17 +0530 Subject: [PATCH 20/33] debug Signed-off-by: Samhita Alla --- flyte_tests_validate.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flyte_tests_validate.py b/flyte_tests_validate.py index 7079acb2f..20ca13fa6 100644 --- a/flyte_tests_validate.py +++ b/flyte_tests_validate.py @@ -41,6 +41,8 @@ def get_commands(output): # Check if the workflow specified is present in the pyflyte run output just_the_workflow = workflow.split(".")[2] + print(output_string) + print(" " + just_the_workflow + " ") if " " + just_the_workflow + " " in output_string: print("Workflow found in the pyflyte run output.") else: From adc2eec31408e5403fd96e647d937ff4507b0887 Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Fri, 1 Dec 2023 19:06:16 +0530 Subject: [PATCH 21/33] debug Signed-off-by: Samhita Alla --- flyte_tests_validate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flyte_tests_validate.py b/flyte_tests_validate.py index 20ca13fa6..6125a127f 100644 --- a/flyte_tests_validate.py +++ b/flyte_tests_validate.py @@ -37,11 +37,12 @@ def get_commands(output): for workflow, params in workflows: # Use the `pyflyte run` command to execute the workflow - output_string = subprocess.run(["pyflyte", "run", file_name], capture_output=True, text=True).stdout + output_string = str(subprocess.run(["pyflyte", "run", file_name], capture_output=True, text=True).stdout) # Check if the workflow specified is present in the pyflyte run output just_the_workflow = workflow.split(".")[2] print(output_string) + print(len(output_string)) print(" " + just_the_workflow + " ") if " " + just_the_workflow + " " in output_string: print("Workflow found in the pyflyte run output.") From 64272f05a8705a88b067f1dd22774786a1bebbab Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Fri, 1 Dec 2023 19:12:01 +0530 Subject: [PATCH 22/33] debug Signed-off-by: Samhita Alla --- flyte_tests_validate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flyte_tests_validate.py b/flyte_tests_validate.py index 6125a127f..495e20f64 100644 --- a/flyte_tests_validate.py +++ b/flyte_tests_validate.py @@ -42,8 +42,8 @@ def get_commands(output): # Check if the workflow specified is present in the pyflyte run output just_the_workflow = workflow.split(".")[2] print(output_string) - print(len(output_string)) - print(" " + just_the_workflow + " ") + print("Show" in output_string) + print(" " + just_the_workflow + " " in output_string) if " " + just_the_workflow + " " in output_string: print("Workflow found in the pyflyte run output.") else: From 1b112dededcad3cfcaacfb9f73bb072496344719 Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Fri, 1 Dec 2023 19:19:17 +0530 Subject: [PATCH 23/33] remove test.py Signed-off-by: Samhita Alla --- flyte_tests_validate.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/flyte_tests_validate.py b/flyte_tests_validate.py index 495e20f64..0e5a7ecca 100644 --- a/flyte_tests_validate.py +++ b/flyte_tests_validate.py @@ -40,18 +40,14 @@ def get_commands(output): output_string = str(subprocess.run(["pyflyte", "run", file_name], capture_output=True, text=True).stdout) # Check if the workflow specified is present in the pyflyte run output - just_the_workflow = workflow.split(".")[2] - print(output_string) - print("Show" in output_string) - print(" " + just_the_workflow + " " in output_string) - if " " + just_the_workflow + " " in output_string: + if workflow in output_string: print("Workflow found in the pyflyte run output.") else: raise Exception("Workflow not found in the pyflyte run output.") # Check if the specified parameters are valid options_output = subprocess.run( - ["pyflyte", "run", file_name, just_the_workflow, "--help"], capture_output=True, text=True + ["pyflyte", "run", file_name, workflow.split(".")[2], "--help"], capture_output=True, text=True ).stdout # Find all matches in the input string From 6597f983edb9b5bec7478482b74348ad8d8b6ddb Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Fri, 1 Dec 2023 19:26:07 +0530 Subject: [PATCH 24/33] debug Signed-off-by: Samhita Alla --- flyte_tests_validate.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/flyte_tests_validate.py b/flyte_tests_validate.py index 0e5a7ecca..d6c088867 100644 --- a/flyte_tests_validate.py +++ b/flyte_tests_validate.py @@ -53,6 +53,10 @@ def get_commands(output): # Find all matches in the input string options = [option.replace("--", "") for option in re.compile(r"--\w+").findall(options_output)] + print(params) + print(options_output) + print(options) + # Validate if the provided params are a subset of the supported params if set(params).issubset(set(options)): print("All parameters found.") From e31c28595f9067dda98b5aac8c4a4ed44a6b5395 Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Fri, 1 Dec 2023 20:58:50 +0530 Subject: [PATCH 25/33] change regex Signed-off-by: Samhita Alla --- flyte_tests_validate.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/flyte_tests_validate.py b/flyte_tests_validate.py index d6c088867..6819fedfc 100644 --- a/flyte_tests_validate.py +++ b/flyte_tests_validate.py @@ -35,7 +35,7 @@ def get_commands(output): if not workflows: raise Exception("The file does not contain any workflows.") - for workflow, params in workflows: + for workflow, params_dict in workflows: # Use the `pyflyte run` command to execute the workflow output_string = str(subprocess.run(["pyflyte", "run", file_name], capture_output=True, text=True).stdout) @@ -51,14 +51,10 @@ def get_commands(output): ).stdout # Find all matches in the input string - options = [option.replace("--", "") for option in re.compile(r"--\w+").findall(options_output)] - - print(params) - print(options_output) - print(options) - - # Validate if the provided params are a subset of the supported params - if set(params).issubset(set(options)): + params = params_dict.keys() + if not params: + print("No parameters found.") + elif any(re.findall(r"|".join(params), options_output, re.IGNORECASE)): print("All parameters found.") else: raise Exception("There's a mismatch between the values accepted by the workflow and the ones you provided.") From 1ac7e02b15032d5dea1775107c9bccdd1491bd85 Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Fri, 1 Dec 2023 21:28:01 +0530 Subject: [PATCH 26/33] debug Signed-off-by: Samhita Alla --- flyte_tests_validate.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/flyte_tests_validate.py b/flyte_tests_validate.py index 6819fedfc..dae48780b 100644 --- a/flyte_tests_validate.py +++ b/flyte_tests_validate.py @@ -39,6 +39,10 @@ def get_commands(output): # Use the `pyflyte run` command to execute the workflow output_string = str(subprocess.run(["pyflyte", "run", file_name], capture_output=True, text=True).stdout) + print(output_string) + print(workflow) + print(workflow in output_string) + # Check if the workflow specified is present in the pyflyte run output if workflow in output_string: print("Workflow found in the pyflyte run output.") From 04fe6d55b13847da44160f33147f4e462209c48d Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Fri, 1 Dec 2023 21:57:53 +0530 Subject: [PATCH 27/33] add env Signed-off-by: Samhita Alla --- flyte_tests_validate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flyte_tests_validate.py b/flyte_tests_validate.py index dae48780b..65a9753fb 100644 --- a/flyte_tests_validate.py +++ b/flyte_tests_validate.py @@ -51,7 +51,7 @@ def get_commands(output): # Check if the specified parameters are valid options_output = subprocess.run( - ["pyflyte", "run", file_name, workflow.split(".")[2], "--help"], capture_output=True, text=True + ["pyflyte", "run", file_name, workflow.split(".")[2], "--help"], capture_output=True, text=True, env={} ).stdout # Find all matches in the input string From b35a1911f7e54e1d283bdbcb9eb865ed20cca850 Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Fri, 1 Dec 2023 22:08:09 +0530 Subject: [PATCH 28/33] do not truncate output Signed-off-by: Samhita Alla --- flyte_tests_validate.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/flyte_tests_validate.py b/flyte_tests_validate.py index 65a9753fb..a0352b9cc 100644 --- a/flyte_tests_validate.py +++ b/flyte_tests_validate.py @@ -51,7 +51,10 @@ def get_commands(output): # Check if the specified parameters are valid options_output = subprocess.run( - ["pyflyte", "run", file_name, workflow.split(".")[2], "--help"], capture_output=True, text=True, env={} + ["pyflyte", "run", file_name, workflow.split(".")[2], "--help"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, ).stdout # Find all matches in the input string From 6069c0f5ab52247a54e9ee5a38d302531cad240d Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Fri, 1 Dec 2023 22:18:13 +0530 Subject: [PATCH 29/33] search Signed-off-by: Samhita Alla --- flyte_tests_validate.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/flyte_tests_validate.py b/flyte_tests_validate.py index a0352b9cc..aa5f1e3ea 100644 --- a/flyte_tests_validate.py +++ b/flyte_tests_validate.py @@ -39,21 +39,18 @@ def get_commands(output): # Use the `pyflyte run` command to execute the workflow output_string = str(subprocess.run(["pyflyte", "run", file_name], capture_output=True, text=True).stdout) - print(output_string) - print(workflow) - print(workflow in output_string) - # Check if the workflow specified is present in the pyflyte run output - if workflow in output_string: + print(output_string.split()) + just_the_workflow = workflow.split(".")[2] + if just_the_workflow in output_string.split(): print("Workflow found in the pyflyte run output.") else: raise Exception("Workflow not found in the pyflyte run output.") # Check if the specified parameters are valid options_output = subprocess.run( - ["pyflyte", "run", file_name, workflow.split(".")[2], "--help"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + ["pyflyte", "run", file_name, just_the_workflow, "--help"], + capture_output=True, text=True, ).stdout From 5f345eabccacf2c33a8600deb7f321e45f85fbf6 Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Sat, 2 Dec 2023 22:12:26 +0530 Subject: [PATCH 30/33] clean string Signed-off-by: Samhita Alla --- flyte_tests_validate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flyte_tests_validate.py b/flyte_tests_validate.py index aa5f1e3ea..4bd79214b 100644 --- a/flyte_tests_validate.py +++ b/flyte_tests_validate.py @@ -40,9 +40,9 @@ def get_commands(output): output_string = str(subprocess.run(["pyflyte", "run", file_name], capture_output=True, text=True).stdout) # Check if the workflow specified is present in the pyflyte run output - print(output_string.split()) + cleaned_string = re.sub(r"\x1b\[[0-9;]*[mG]", "", output_string) just_the_workflow = workflow.split(".")[2] - if just_the_workflow in output_string.split(): + if just_the_workflow in cleaned_string.split(): print("Workflow found in the pyflyte run output.") else: raise Exception("Workflow not found in the pyflyte run output.") From 8668e131a6bf5b4fe62f70a110215b92de4f3e86 Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Sat, 2 Dec 2023 22:39:36 +0530 Subject: [PATCH 31/33] cleanup Signed-off-by: Samhita Alla --- flyte_tests_validate.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/flyte_tests_validate.py b/flyte_tests_validate.py index 4bd79214b..2d21d5a6b 100644 --- a/flyte_tests_validate.py +++ b/flyte_tests_validate.py @@ -10,16 +10,6 @@ examples = [(example[0], example[1]) for entry in data for example in entry.get("examples", []) if len(example) >= 1] - -def get_commands(output): - # Find lines starting with '|' - command_lines = re.findall(r"│\s*([a-zA-Z0-9_\-\.]+)\s+", output) - - # Filter out 'Commands' and "--help" lines - commands = [cmd for cmd in command_lines if cmd.lower() != "commands" and cmd.lower() != "--help"] - return commands - - for file_name in open(file_list, "r").readlines(): file_name = file_name.strip() print(f"Processing file: {file_name}") @@ -54,7 +44,6 @@ def get_commands(output): text=True, ).stdout - # Find all matches in the input string params = params_dict.keys() if not params: print("No parameters found.") From 21c7fb2860bb31d7aa70b8b85d8be2185e0e90b9 Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Wed, 6 Dec 2023 17:30:19 +0530 Subject: [PATCH 32/33] incorporate suggestions by Eduardo Signed-off-by: Samhita Alla --- .github/workflows/checks.yml | 2 +- flyte_tests_validate.py | 101 ++++++++++++++++++----------------- 2 files changed, 54 insertions(+), 49 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 9fcd5c174..e6b7a036c 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -230,7 +230,7 @@ jobs: register \ --project flytesnacks \ --domain development \ - --image cr.flyte.org/flyteorg/flytekit:py3.11-1.10.0 \ + --image cr.flyte.org/flyteorg/flytekit:py3.11-latest \ --version ${{ env.FLYTESNACKS_VERSION }} \ flytesnacks/$line; done < flyte_tests.txt diff --git a/flyte_tests_validate.py b/flyte_tests_validate.py index 2d21d5a6b..12db7b33c 100644 --- a/flyte_tests_validate.py +++ b/flyte_tests_validate.py @@ -3,51 +3,56 @@ import re import subprocess -file_list = "flyte_tests.txt" - -with open("flyte_tests_manifest.json", "r") as file: - data = json.load(file) - -examples = [(example[0], example[1]) for entry in data for example in entry.get("examples", []) if len(example) >= 1] - -for file_name in open(file_list, "r").readlines(): - file_name = file_name.strip() - print(f"Processing file: {file_name}") - - # Retrieve the file path, including the name of the file and its immediate parent directory - directory_path = os.path.dirname(file_name).split(os.path.sep)[-1:] - file_path = ".".join(directory_path + [os.path.splitext(os.path.basename(file_name))[0]]) - - # Retrieve the workflow(s) - workflows = list(filter(lambda tup: file_path in tup[0], examples)) - - # Verify if there are any workflows present in the provided file path - if not workflows: - raise Exception("The file does not contain any workflows.") - - for workflow, params_dict in workflows: - # Use the `pyflyte run` command to execute the workflow - output_string = str(subprocess.run(["pyflyte", "run", file_name], capture_output=True, text=True).stdout) - - # Check if the workflow specified is present in the pyflyte run output - cleaned_string = re.sub(r"\x1b\[[0-9;]*[mG]", "", output_string) - just_the_workflow = workflow.split(".")[2] - if just_the_workflow in cleaned_string.split(): - print("Workflow found in the pyflyte run output.") - else: - raise Exception("Workflow not found in the pyflyte run output.") - - # Check if the specified parameters are valid - options_output = subprocess.run( - ["pyflyte", "run", file_name, just_the_workflow, "--help"], - capture_output=True, - text=True, - ).stdout - - params = params_dict.keys() - if not params: - print("No parameters found.") - elif any(re.findall(r"|".join(params), options_output, re.IGNORECASE)): - print("All parameters found.") - else: - raise Exception("There's a mismatch between the values accepted by the workflow and the ones you provided.") +if __name__ == "__main__": + file_list = "flyte_tests.txt" + + with open("flyte_tests_manifest.json", "r") as file: + data = json.load(file) + + examples = [ + (example[0], example[1]) for entry in data for example in entry.get("examples", []) if len(example) >= 1 + ] + + for file_name in open(file_list, "r").readlines(): + file_name = file_name.strip() + print(f"Processing file: {file_name}") + + # Retrieve the file path, including the name of the file and its immediate parent directory + directory_path = os.path.dirname(file_name).split(os.path.sep)[-1:] + file_path = ".".join(directory_path + [os.path.splitext(os.path.basename(file_name))[0]]) + + # Retrieve the workflow(s) + workflows = list(filter(lambda tup: file_path in tup[0], examples)) + + # Verify if there are any workflows present in the provided file path + if not workflows: + raise Exception("The file does not contain any workflows.") + + for workflow, params_dict in workflows: + # Use the `pyflyte run` command to execute the workflow + output_string = str(subprocess.run(["pyflyte", "run", file_name], capture_output=True, text=True).stdout) + + # Check if the workflow specified is present in the pyflyte run output + cleaned_string = re.sub(r"\x1b\[[0-9;]*[mG]", "", output_string) + just_the_workflow = workflow.split(".")[2] + if just_the_workflow in cleaned_string.split(): + print("Workflow found in the pyflyte run output.") + else: + raise Exception("Workflow not found in the pyflyte run output.") + + # Check if the specified parameters are valid + options_output = subprocess.run( + ["pyflyte", "run", file_name, just_the_workflow, "--help"], + capture_output=True, + text=True, + ).stdout + + params = params_dict.keys() + if not params: + print("No parameters found.") + elif any(re.findall(r"|".join(params), options_output, re.IGNORECASE)): + print("All parameters found.") + else: + raise Exception( + "There's a mismatch between the values accepted by the workflow and the ones you provided." + ) From fe80603c885fa0a268eee368d9881a560fe0a9ff Mon Sep 17 00:00:00 2001 From: Samhita Alla Date: Wed, 6 Dec 2023 17:59:43 +0530 Subject: [PATCH 33/33] remove comment Signed-off-by: Samhita Alla --- .github/workflows/checks.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index e6b7a036c..dd9502ade 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -225,7 +225,6 @@ jobs: run: | while read -r line; do - # TODO: unpin version of flytekit once the FlyteFile bug is fixed pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \ register \ --project flytesnacks \