From 0df57efcda35d7fc1fbf6a0fb93716c8a208e886 Mon Sep 17 00:00:00 2001 From: Ilya Matiach Date: Fri, 25 Aug 2023 17:55:50 -0400 Subject: [PATCH] add e2e UI notebook tests to covid19 event multilabel text classification notebook --- .../CI-e2e-notebooks-text-vision.yml | 1 + .../datasetExplorer.spec.ts | 10 ++++ .../errorAnalysis.spec.ts | 11 ++++ .../modelOverview.spec.ts | 10 ++++ .../modelAssessment/IModelAssessmentData.ts | 3 +- .../CovidTextClassificationModelDebugging.ts | 52 +++++++++++++++++++ .../datasets/modelAssessmentDatasets.ts | 2 + scripts/e2e-widget.js | 5 +- 8 files changed, 90 insertions(+), 4 deletions(-) create mode 100644 apps/widget-e2e/src/integration/modelAssessment/responsibleaidashboardCovid19EventMultilabelTextClassificationModelDebugging/datasetExplorer.spec.ts create mode 100644 apps/widget-e2e/src/integration/modelAssessment/responsibleaidashboardCovid19EventMultilabelTextClassificationModelDebugging/errorAnalysis.spec.ts create mode 100644 apps/widget-e2e/src/integration/modelAssessment/responsibleaidashboardCovid19EventMultilabelTextClassificationModelDebugging/modelOverview.spec.ts create mode 100644 libs/e2e/src/lib/describer/modelAssessment/datasets/CovidTextClassificationModelDebugging.ts diff --git a/.github/workflows/CI-e2e-notebooks-text-vision.yml b/.github/workflows/CI-e2e-notebooks-text-vision.yml index eb210f459a..fedc75d112 100644 --- a/.github/workflows/CI-e2e-notebooks-text-vision.yml +++ b/.github/workflows/CI-e2e-notebooks-text-vision.yml @@ -156,6 +156,7 @@ jobs: run: | yarn e2e-widget -n "responsibleaidashboard-DBPedia-text-classification-model-debugging" -f ${{ matrix.flights }} yarn e2e-widget -n "responsibleaidashboard-blbooksgenre-binary-text-classification-model-debugging" -f ${{ matrix.flights }} + yarn e2e-widget -n "responsibleaidashboard-covid19-event-multilabel-text-classification-model-debugging" -f ${{ matrix.flights }} - name: Upload e2e test screen shot if: always() diff --git a/apps/widget-e2e/src/integration/modelAssessment/responsibleaidashboardCovid19EventMultilabelTextClassificationModelDebugging/datasetExplorer.spec.ts b/apps/widget-e2e/src/integration/modelAssessment/responsibleaidashboardCovid19EventMultilabelTextClassificationModelDebugging/datasetExplorer.spec.ts new file mode 100644 index 0000000000..d4afa2753b --- /dev/null +++ b/apps/widget-e2e/src/integration/modelAssessment/responsibleaidashboardCovid19EventMultilabelTextClassificationModelDebugging/datasetExplorer.spec.ts @@ -0,0 +1,10 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { + describeDatasetExplorer, + modelAssessmentDatasets +} from "@responsible-ai/e2e"; +const datasetShape = + modelAssessmentDatasets.CovidTextClassificationModelDebugging; +describeDatasetExplorer(datasetShape, "CovidTextClassificationModelDebugging"); diff --git a/apps/widget-e2e/src/integration/modelAssessment/responsibleaidashboardCovid19EventMultilabelTextClassificationModelDebugging/errorAnalysis.spec.ts b/apps/widget-e2e/src/integration/modelAssessment/responsibleaidashboardCovid19EventMultilabelTextClassificationModelDebugging/errorAnalysis.spec.ts new file mode 100644 index 0000000000..78edf0f01f --- /dev/null +++ b/apps/widget-e2e/src/integration/modelAssessment/responsibleaidashboardCovid19EventMultilabelTextClassificationModelDebugging/errorAnalysis.spec.ts @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { + describeErrorAnalysis, + modelAssessmentDatasets +} from "@responsible-ai/e2e"; + +const datasetShape = + modelAssessmentDatasets.CovidTextClassificationModelDebugging; +describeErrorAnalysis(datasetShape, "CovidTextClassificationModelDebugging"); diff --git a/apps/widget-e2e/src/integration/modelAssessment/responsibleaidashboardCovid19EventMultilabelTextClassificationModelDebugging/modelOverview.spec.ts b/apps/widget-e2e/src/integration/modelAssessment/responsibleaidashboardCovid19EventMultilabelTextClassificationModelDebugging/modelOverview.spec.ts new file mode 100644 index 0000000000..f5eca4333e --- /dev/null +++ b/apps/widget-e2e/src/integration/modelAssessment/responsibleaidashboardCovid19EventMultilabelTextClassificationModelDebugging/modelOverview.spec.ts @@ -0,0 +1,10 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { + describeModelOverview, + modelAssessmentDatasets +} from "@responsible-ai/e2e"; +const datasetShape = + modelAssessmentDatasets.CovidTextClassificationModelDebugging; +describeModelOverview(datasetShape, "CovidTextClassificationModelDebugging"); diff --git a/libs/e2e/src/lib/describer/modelAssessment/IModelAssessmentData.ts b/libs/e2e/src/lib/describer/modelAssessment/IModelAssessmentData.ts index 28c67d5e98..b732373fe9 100644 --- a/libs/e2e/src/lib/describer/modelAssessment/IModelAssessmentData.ts +++ b/libs/e2e/src/lib/describer/modelAssessment/IModelAssessmentData.ts @@ -207,5 +207,6 @@ export enum RAINotebookNames { "FridgeMultilabelModelDebugging" = "responsibleaidashboard-fridge-multilabel-image-classification-model-debugging.py", "FridgeObjectDetectionModelDebugging" = "responsibleaidashboard-fridge-object-detection-model-debugging.py", "DBPediaTextClassificationModelDebugging" = "responsibleaidashboard-DBPedia-text-classification-model-debugging.py", - "BlbooksgenreTextClassificationModelDebugging" = "responsibleaidashboard-blbooksgenre-binary-text-classification-model-debugging.py" + "BlbooksgenreTextClassificationModelDebugging" = "responsibleaidashboard-blbooksgenre-binary-text-classification-model-debugging.py", + "CovidTextClassificationModelDebugging" = "responsibleaidashboard-covid19-event-multilabel-text-classification-model-debugging.py" } diff --git a/libs/e2e/src/lib/describer/modelAssessment/datasets/CovidTextClassificationModelDebugging.ts b/libs/e2e/src/lib/describer/modelAssessment/datasets/CovidTextClassificationModelDebugging.ts new file mode 100644 index 0000000000..75cb04761a --- /dev/null +++ b/libs/e2e/src/lib/describer/modelAssessment/datasets/CovidTextClassificationModelDebugging.ts @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +export const CovidTextClassificationModelDebugging = { + causalAnalysisData: { + hasCausalAnalysisComponent: false + }, + checkDupCohort: true, + cohortDefaultName: "All data", + dataBalanceData: { + aggregateBalanceMeasuresComputed: false, + distributionBalanceMeasuresComputed: false, + featureBalanceMeasuresComputed: false + }, + errorAnalysisData: { + hasErrorAnalysisComponent: true + }, + featureImportanceData: { + hasFeatureImportanceComponent: false + }, + featureNames: ["text"], + isTextClassification: true, + modelOverviewData: { + featureCohortView: { + firstFeatureToSelect: "positive_words", + multiFeatureCohorts: 3, + secondFeatureToSelect: "negative_words", + singleFeatureCohorts: 3 + }, + hasModelOverviewComponent: true, + initialCohorts: [ + { + metrics: { + accuracy: "0.95", + precision: "0.944", + recall: "1" + }, + name: "All data", + sampleSize: "20" + } + ], + newCohort: { + metrics: { + accuracy: "0.947", + precision: "0.941", + recall: "1" + }, + name: "CohortCreateE2E-text-classification", + sampleSize: "19" + } + } +}; diff --git a/libs/e2e/src/lib/describer/modelAssessment/datasets/modelAssessmentDatasets.ts b/libs/e2e/src/lib/describer/modelAssessment/datasets/modelAssessmentDatasets.ts index 5be53fa03b..69053a9a8b 100644 --- a/libs/e2e/src/lib/describer/modelAssessment/datasets/modelAssessmentDatasets.ts +++ b/libs/e2e/src/lib/describer/modelAssessment/datasets/modelAssessmentDatasets.ts @@ -7,6 +7,7 @@ import { IModelAssessmentData } from "../IModelAssessmentData"; import { BlbooksgenreTextClassificationModelDebugging } from "./BlbooksgenreTextClassificationModelDebugging"; import { CensusClassificationModelDebugging } from "./CensusClassificationModelDebugging"; +import { CovidTextClassificationModelDebugging } from "./CovidTextClassificationModelDebugging"; import { DBPediaTextClassificationModelDebugging } from "./DBPediaTextClassificationModelDebugging"; import { DiabetesDecisionMaking } from "./DiabetesDecisionMaking"; import { DiabetesRegressionModelDebugging } from "./DiabetesRegressionModelDebugging"; @@ -24,6 +25,7 @@ export const regExForNumbersWithBrackets = /^\((\d+)\)$/; // Ex: (60) const modelAssessmentDatasets: { [name: string]: IModelAssessmentData } = { BlbooksgenreTextClassificationModelDebugging, CensusClassificationModelDebugging, + CovidTextClassificationModelDebugging, DBPediaTextClassificationModelDebugging, DiabetesDecisionMaking, DiabetesRegressionModelDebugging, diff --git a/scripts/e2e-widget.js b/scripts/e2e-widget.js index 802b381ad4..cfa3fd45ac 100644 --- a/scripts/e2e-widget.js +++ b/scripts/e2e-widget.js @@ -29,11 +29,10 @@ const visionFileNames = [ ]; const textFileNames = [ "responsibleaidashboard-DBPedia-text-classification-model-debugging", - "responsibleaidashboard-blbooksgenre-binary-text-classification-model-debugging" -]; -const ignoredFiles = [ + "responsibleaidashboard-blbooksgenre-binary-text-classification-model-debugging", "responsibleaidashboard-covid19-event-multilabel-text-classification-model-debugging" ]; +const ignoredFiles = []; const fileNames = tabularFileNames .concat(visionFileNames) .concat(textFileNames);