Skip to content

Commit

Permalink
Add AXE testing to test suite (#3764)
Browse files Browse the repository at this point in the history
* Initial commit

* Add docs, add more controls

* CR feedback 1

* Small fixes

* Add comments explaining decision making.

* Fix TAEF bug (hopefully)

* Fix nuget reference

* update win32.registry

* fix failing win32.registry version disrepancy

* add missing generatePathProperty

* add missing dlls to helixPayload

* fix formatting

* add error message and output .a11yTest file

* fix failing axe test

* add missing dlls

* add missing dll

* temp fix for axe test failures

Co-authored-by: Karen Lai <[email protected]>
  • Loading branch information
marcelwgn and karkarl authored Dec 15, 2021
1 parent a1ace79 commit fa28e62
Show file tree
Hide file tree
Showing 40 changed files with 428 additions and 21 deletions.
13 changes: 13 additions & 0 deletions build/CopyFilesToStagingDir.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,19 @@ PublishFile -IfExists $FullBuildOutput\Microsoft.UI.Xaml.Design\Microsoft.UI.Xam
PublishFile -IfExists $BuildOutputDir\$Configuration\AnyCPU\Microsoft.UI.Xaml.FrameworkPackagePRI\Microsoft.UI.Xaml.pri $FullPublishDir\Microsoft.UI.Xaml.FrameworkPackagePRI\
PublishFile -IfExists $BuildOutputDir\$Configuration\AnyCPU\MUXControls.Test.TAEF\MUXControls.Test.dll $FullPublishDir\Test\
PublishFile -IfExists $BuildOutputDir\$Configuration\AnyCPU\MUXControls.Test.TAEF\MUXTestInfra.TAEF.dll $FullPublishDir\Test\
PublishFile -IfExists $BuildOutputDir\$Configuration\AnyCPU\MUXControls.Test.TAEF\Microsoft.Win32.Registry.dll $FullPublishDir\Test\
PublishFile -IfExists $BuildOutputDir\$Configuration\AnyCPU\MUXControls.Test.TAEF\Axe.Windows.Actions.dll $FullPublishDir\Test\
PublishFile -IfExists $BuildOutputDir\$Configuration\AnyCPU\MUXControls.Test.TAEF\Axe.Windows.Automation.dll $FullPublishDir\Test\
PublishFile -IfExists $BuildOutputDir\$Configuration\AnyCPU\MUXControls.Test.TAEF\Axe.Windows.Core.dll $FullPublishDir\Test\
PublishFile -IfExists $BuildOutputDir\$Configuration\AnyCPU\MUXControls.Test.TAEF\Axe.Windows.Desktop.dll $FullPublishDir\Test\
PublishFile -IfExists $BuildOutputDir\$Configuration\AnyCPU\MUXControls.Test.TAEF\Axe.Windows.Rules.dll $FullPublishDir\Test\
PublishFile -IfExists $BuildOutputDir\$Configuration\AnyCPU\MUXControls.Test.TAEF\Axe.Windows.RuleSelection.dll $FullPublishDir\Test\
PublishFile -IfExists $BuildOutputDir\$Configuration\AnyCPU\MUXControls.Test.TAEF\Axe.Windows.SystemAbstractions.dll $FullPublishDir\Test\
PublishFile -IfExists $BuildOutputDir\$Configuration\AnyCPU\MUXControls.Test.TAEF\Axe.Windows.Telemetry.dll $FullPublishDir\Test\
PublishFile -IfExists $BuildOutputDir\$Configuration\AnyCPU\MUXControls.Test.TAEF\Axe.Windows.Win32.dll $FullPublishDir\Test\
PublishFile -IfExists $BuildOutputDir\$Configuration\AnyCPU\MUXControls.Test.TAEF\Newtonsoft.Json.dll $FullPublishDir\Test\
PublishFile -IfExists $BuildOutputDir\$Configuration\AnyCPU\MUXControls.Test.TAEF\System.Drawing.Common.dll $FullPublishDir\Test\
PublishFile -IfExists $BuildOutputDir\$Configuration\AnyCPU\MUXControls.Test.TAEF\System.IO.Packaging.dll $FullPublishDir\Test\
PublishFile -IfExists $FullBuildOutput\Microsoft.UI.Xaml\WebView2Loader.dll $FullPublishDir\Test\
PublishFile -IfExists $BuildOutputDir\$Configuration\AnyCPU\MUXControls.ReleaseTest.TAEF\MUXControls.ReleaseTest.dll $FullPublishDir\Test\
PublishFile -IfExists $BuildOutputDir\$Configuration\AnyCPU\MUXControls.ReleaseTest.TAEF\WebView2Loader.dll $FullPublishDir\Test\
Expand Down
13 changes: 13 additions & 0 deletions build/Helix/PrepareHelixPayload.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,19 @@ function Copy-Recursively-If-Exists

# Copy files from the 'drop' artifact dir
Copy-Item "$repoDirectory\Artifacts\$ArtifactName\$Configuration\$Platform\Test\MUXControls.Test.dll" $payloadDir
Copy-Item "$repoDirectory\Artifacts\$ArtifactName\$Configuration\$Platform\Test\Microsoft.Win32.Registry.dll" "$payloadDir\.NETCoreApp2.1\"
Copy-Item "$repoDirectory\Artifacts\$ArtifactName\$Configuration\$Platform\Test\Axe.Windows.Actions.dll" "$payloadDir\.NETCoreApp2.1\"
Copy-Item "$repoDirectory\Artifacts\$ArtifactName\$Configuration\$Platform\Test\Axe.Windows.Automation.dll" "$payloadDir\.NETCoreApp2.1\"
Copy-Item "$repoDirectory\Artifacts\$ArtifactName\$Configuration\$Platform\Test\Axe.Windows.Core.dll" "$payloadDir\.NETCoreApp2.1\"
Copy-Item "$repoDirectory\Artifacts\$ArtifactName\$Configuration\$Platform\Test\Axe.Windows.Desktop.dll" "$payloadDir\.NETCoreApp2.1\"
Copy-Item "$repoDirectory\Artifacts\$ArtifactName\$Configuration\$Platform\Test\Axe.Windows.Rules.dll" "$payloadDir\.NETCoreApp2.1\"
Copy-Item "$repoDirectory\Artifacts\$ArtifactName\$Configuration\$Platform\Test\Axe.Windows.RuleSelection.dll" "$payloadDir\.NETCoreApp2.1\"
Copy-Item "$repoDirectory\Artifacts\$ArtifactName\$Configuration\$Platform\Test\Axe.Windows.SystemAbstractions.dll" "$payloadDir\.NETCoreApp2.1\"
Copy-Item "$repoDirectory\Artifacts\$ArtifactName\$Configuration\$Platform\Test\Axe.Windows.Telemetry.dll" "$payloadDir\.NETCoreApp2.1\"
Copy-Item "$repoDirectory\Artifacts\$ArtifactName\$Configuration\$Platform\Test\Axe.Windows.Win32.dll" "$payloadDir\.NETCoreApp2.1\"
Copy-Item "$repoDirectory\Artifacts\$ArtifactName\$Configuration\$Platform\Test\Newtonsoft.Json.dll" "$payloadDir\.NETCoreApp2.1\"
Copy-Item "$repoDirectory\Artifacts\$ArtifactName\$Configuration\$Platform\Test\System.Drawing.Common.dll" "$payloadDir\.NETCoreApp2.1\"
Copy-Item "$repoDirectory\Artifacts\$ArtifactName\$Configuration\$Platform\Test\System.IO.Packaging.dll" "$payloadDir\.NETCoreApp2.1\"
Copy-Item "$repoDirectory\Artifacts\$ArtifactName\$Configuration\$Platform\Test\MUXTestInfra.TAEF.dll" $payloadDir
Copy-If-Exists "$repoDirectory\Artifacts\$ArtifactName\$Configuration\$Platform\Test\MUXExperimental.Test.dll" $payloadDir
Copy-If-Exists "$repoDirectory\Artifacts\$ArtifactName\$Configuration\$Platform\Test\WebView2Loader.dll" $payloadDir
Expand Down
10 changes: 10 additions & 0 deletions dev/AutoSuggestBox/InteractionTests/AutoSuggestBoxTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

using Microsoft.Windows.Apps.Test.Foundation.Controls;
using Microsoft.Windows.Apps.Test.Foundation.Patterns;
using MUXTestInfra.Shared.Infra;

namespace Windows.UI.Xaml.Tests.MUXControls.InteractionTests
{
Expand Down Expand Up @@ -58,5 +59,14 @@ public void CanSelectSuggestion()
Verify.AreEqual("dolor", autoSuggestBoxTextBox.Value);
}
}

[TestMethod]
public void VerifyAxeScanPasses()
{
using (var setup = new TestSetupHelper("AutoSuggestBox-Axe"))
{
AxeTestHelper.TestForAxeIssues();
}
}
}
}
7 changes: 5 additions & 2 deletions dev/AutoSuggestBox/TestUI/AutoSuggestBoxPage.xaml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@

<StackPanel Margin="12">
<TextBlock Text="Sample AutoSuggestBoxes" Style="{ThemeResource StandardGroupHeader}"/>
<AutoSuggestBox QueryIcon="Find"/>
<AutoSuggestBox AutomationProperties.Name="Sample AutoSuggestBox"/>
<AutoSuggestBox Header="AutoSuggestBox with header"/>
<AutoSuggestBox AutomationProperties.Name="Sample AutoSuggestBox with QueryIcon" QueryIcon="Find"/>
<AutoSuggestBox Header="AutoSuggestBox with header" FontSize="12">
<AutoSuggestBox.QueryIcon>
<controls:AnimatedIcon>
Expand All @@ -24,12 +26,13 @@
</controls:AnimatedIcon>
</AutoSuggestBox.QueryIcon>
</AutoSuggestBox>

<AutoSuggestBox Header="AutoSuggestBox with suggestions" QuerySubmitted="AutoSuggestBox_QuerySubmitted" />
<StackPanel x:Name="stackPanelSearch"
Margin="5"
Orientation="Horizontal">
<AutoSuggestBox Height="33" MinWidth="230"
PlaceholderText="Search Stuff" QueryIcon="Find" />
Header="Search Stuff" QueryIcon="Find" />
</StackPanel>
</StackPanel>

Expand Down
1 change: 1 addition & 0 deletions dev/AutoSuggestBox/TestUI/AutoSuggestBoxPage.xaml.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
namespace MUXControlsTestApp
{
[TopLevelTestPage(Name = "AutoSuggestBox", Icon = "AutoSuggestBox.png")]
[AxeScanTestPage(Name = "AutoSuggestBox-Axe")]
public sealed partial class AutoSuggestBoxPage : TestPage
{
string[] suggestions =
Expand Down
10 changes: 10 additions & 0 deletions dev/Expander/InteractionTests/ExpanderTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
using Microsoft.Windows.Apps.Test.Foundation.Controls;
using Microsoft.Windows.Apps.Test.Foundation.Patterns;
using Microsoft.Windows.Apps.Test.Foundation.Waiters;
using MUXTestInfra.Shared.Infra;

namespace Windows.UI.Xaml.Tests.MUXControls.InteractionTests
{
Expand Down Expand Up @@ -126,6 +127,15 @@ public void TestCleanup()
TestCleanupHelper.Cleanup();
}

[TestMethod]
public void VerifyAxeScanPasses()
{
using (var setup = new TestSetupHelper("Expander-Axe"))
{
AxeTestHelper.TestForAxeIssues();
}
}

[TestMethod]
public void ExpandCollapseAutomationTests()
{
Expand Down
2 changes: 1 addition & 1 deletion dev/Expander/TestUI/ExpanderPage.xaml
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@

<controls:Expander AutomationProperties.Name="ExpanderWithButtons" IsExpanded="False" Margin="12" MaxWidth="{ThemeResource FlyoutThemeMaxWidth}">
<controls:Expander.Header>
<ToggleButton>This is a toggle button in the header</ToggleButton>
<ToggleButton>This is a Togglebutton in the header</ToggleButton>
</controls:Expander.Header>
<StackPanel>
<TextBlock TextWrapping="Wrap" Text="Content. This is long content to test wrapping on the content section, this content should be wrapping after the control expands to max width."/>
Expand Down
7 changes: 7 additions & 0 deletions dev/Expander/TestUI/ExpanderPage.xaml.cs
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ protected override AutomationPeer OnCreateAutomationPeer()
{
return new TestControlAutomationPeer(this);
}

}

public sealed class TestControlAutomationPeer : FrameworkElementAutomationPeer
Expand All @@ -32,9 +33,15 @@ protected override AutomationControlType GetAutomationControlTypeCore()
{
return AutomationControlType.Custom;
}

protected override string GetLocalizedControlTypeCore()
{
return "TestControl";
}
}

[TopLevelTestPage(Name = "Expander")]
[AxeScanTestPage(Name = "Expander-Axe")]
public sealed partial class ExpanderPage : TestPage
{
public ExpanderPage()
Expand Down
12 changes: 11 additions & 1 deletion dev/MenuBar/MenuBar_InteractionTests/MenuBarTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,13 @@
using Microsoft.Windows.Apps.Test.Foundation.Controls;
using Microsoft.Windows.Apps.Test.Foundation.Patterns;
using Microsoft.Windows.Apps.Test.Foundation.Waiters;
using MUXTestInfra.Shared.Infra;

namespace Windows.UI.Xaml.Tests.MUXControls.InteractionTests
{
[TestClass]
public class MenuBarTests
{
{
[ClassInitialize]
[TestProperty("RunAs", "User")]
[TestProperty("Classification", "Integration")]
Expand All @@ -36,6 +37,15 @@ public static void ClassInitialize(TestContext testContext)
TestEnvironment.Initialize(testContext);
}

[TestMethod]
public void VerifyAxeScanPasses()
{
using (var setup = new TestSetupHelper("MenuBar-Axe"))
{
AxeTestHelper.TestForAxeIssues();
}
}

[TestCleanup]
public void TestCleanup()
{
Expand Down
1 change: 1 addition & 0 deletions dev/MenuBar/MenuBar_TestUI/MenuBarPage.xaml.cs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
namespace MUXControlsTestApp
{
[TopLevelTestPage(Name = "MenuBar", Icon = "MenuBar.png")]
[AxeScanTestPage(Name = "MenuBar-Axe")]
public sealed partial class MenuBarPage : TestPage
{
public MenuBarPage()
Expand Down
10 changes: 10 additions & 0 deletions dev/NavigationView/NavigationView_InteractionTests/CommonTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
using Microsoft.Windows.Apps.Test.Foundation;
using Microsoft.Windows.Apps.Test.Foundation.Controls;
using Microsoft.Windows.Apps.Test.Foundation.Waiters;
using MUXTestInfra.Shared.Infra;

namespace Windows.UI.Xaml.Tests.MUXControls.InteractionTests.NavigationViewTests
{
Expand All @@ -38,6 +39,15 @@ public static void ClassInitialize(TestContext testContext)
TestEnvironment.Initialize(testContext);
}

[TestMethod]
public void VerifyAxeScanPasses()
{
using (var setup = new TestSetupHelper("NavigationView-Axe"))
{
AxeTestHelper.TestForAxeIssues();
}
}

[TestMethod]
public void DisplayModeTest()
{
Expand Down
35 changes: 35 additions & 0 deletions dev/NavigationView/TestUI/NavigationViewAxeTestPage.xaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
<!-- Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT License. See LICENSE in the project root for license information. -->
<local:TestPage
x:Class="MUXControlsTestApp.NavigationViewAxeTestPage"
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:local="using:MUXControlsTestApp"
xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
mc:Ignorable="d"
xmlns:muxc="using:Microsoft.UI.Xaml.Controls"
Background="{ThemeResource ApplicationPageBackgroundThemeBrush}">

<StackPanel>
<muxc:NavigationView PaneDisplayMode="Left">
<muxc:NavigationView.MenuItems>
<muxc:NavigationViewItem Content="Item 1"/>
<muxc:NavigationViewItem AutomationProperties.Name="Item 2"/>
</muxc:NavigationView.MenuItems>
</muxc:NavigationView>

<muxc:NavigationView PaneDisplayMode="LeftCompact">
<muxc:NavigationView.MenuItems>
<muxc:NavigationViewItem Content="Item 1"/>
<muxc:NavigationViewItem AutomationProperties.Name="Item 2"/>
</muxc:NavigationView.MenuItems>
</muxc:NavigationView>

<muxc:NavigationView PaneDisplayMode="Top">
<muxc:NavigationView.MenuItems>
<muxc:NavigationViewItem Content="Item 1"/>
<muxc:NavigationViewItem AutomationProperties.Name="Item 2"/>
</muxc:NavigationView.MenuItems>
</muxc:NavigationView>
</StackPanel>
</local:TestPage>
29 changes: 29 additions & 0 deletions dev/NavigationView/TestUI/NavigationViewAxeTestPage.xaml.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See LICENSE in the project root for license information.

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices.WindowsRuntime;
using Windows.Foundation;
using Windows.Foundation.Collections;
using Windows.UI.Xaml;
using Windows.UI.Xaml.Controls;
using Windows.UI.Xaml.Controls.Primitives;
using Windows.UI.Xaml.Data;
using Windows.UI.Xaml.Input;
using Windows.UI.Xaml.Media;
using Windows.UI.Xaml.Navigation;

namespace MUXControlsTestApp
{
[AxeScanTestPage(Name = "NavigationView-Axe")]
public sealed partial class NavigationViewAxeTestPage : TestPage
{
public NavigationViewAxeTestPage()
{
this.InitializeComponent();
}
}
}
9 changes: 8 additions & 1 deletion dev/NavigationView/TestUI/NavigationView_TestUI.projitems
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,10 @@
<SubType>Designer</SubType>
<Generator>MSBuild:Compile</Generator>
</Page>
<Page Include="$(MSBuildThisFileDirectory)NavigationViewAxeTestPage.xaml">
<SubType>Designer</SubType>
<Generator>MSBuild:Compile</Generator>
</Page>
<Page Include="$(MSBuildThisFileDirectory)CustomResources\NavigationViewCustomThemeResourcesPage.xaml">
<SubType>Designer</SubType>
<Generator>MSBuild:Compile</Generator>
Expand Down Expand Up @@ -168,7 +172,10 @@
<DependentUpon>NavigationViewItemTemplatePage.xaml</DependentUpon>
</Compile>
<Compile Include="$(MSBuildThisFileDirectory)Common\NavigationViewPage.xaml.cs">
<DependentUpon>NavigationViewPage.xaml</DependentUpon>
<DependentUpon>NavigationViewPage.xaml</DependentUpon>
</Compile>
<Compile Include="$(MSBuildThisFileDirectory)NavigationViewAxeTestPage.xaml.cs">
<DependentUpon>NavigationViewAxeTestPage.xaml</DependentUpon>
</Compile>
<Compile Include="$(MSBuildThisFileDirectory)CustomResources\NavigationViewCustomThemeResourcesPage.xaml.cs">
<DependentUpon>NavigationViewCustomThemeResourcesPage.xaml</DependentUpon>
Expand Down
9 changes: 9 additions & 0 deletions dev/NumberBox/InteractionTests/NumberBoxTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
using Microsoft.Windows.Apps.Test.Foundation.Controls;
using Microsoft.Windows.Apps.Test.Foundation.Patterns;
using Microsoft.Windows.Apps.Test.Foundation.Waiters;
using MUXTestInfra.Shared.Infra;

namespace Windows.UI.Xaml.Tests.MUXControls.InteractionTests
{
Expand Down Expand Up @@ -605,6 +606,14 @@ public void VerifyRightClickForContextMenuDoesNotDeselectText()
}
}

[TestMethod]
public void VerifyAxeScanPasses()
{
using (var setup = new TestSetupHelper("NumberBox-Axe"))
{
AxeTestHelper.TestForAxeIssues();
}
}

Button FindButton(UIObject parent, string buttonName)
{
Expand Down
16 changes: 16 additions & 0 deletions dev/NumberBox/TestUI/NumberBoxAxeTestPage.xaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
<!-- Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT License. See LICENSE in the project root for license information. -->
<local:TestPage
x:Class="MUXControlsTestApp.NumberBoxAxeTestPage"
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:local="using:MUXControlsTestApp"
xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
xmlns:muxc="using:Microsoft.UI.Xaml.Controls"
mc:Ignorable="d"
Background="{ThemeResource ApplicationPageBackgroundThemeBrush}">

<StackPanel>
<muxc:NumberBox Header="Sample NumberBox"/>
</StackPanel>
</local:TestPage>
13 changes: 13 additions & 0 deletions dev/NumberBox/TestUI/NumberBoxAxeTestPage.xaml.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See LICENSE in the project root for license information.
namespace MUXControlsTestApp
{
[AxeScanTestPage(Name = "NumberBox-Axe")]
public sealed partial class NumberBoxAxeTestPage : TestPage
{
public NumberBoxAxeTestPage()
{
this.InitializeComponent();
}
}
}
7 changes: 7 additions & 0 deletions dev/NumberBox/TestUI/NumberBox_TestUI.projitems
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,19 @@
<Import_RootNamespace>NumberBox_TestUI</Import_RootNamespace>
</PropertyGroup>
<ItemGroup>
<Page Include="$(MSBuildThisFileDirectory)NumberBoxAxeTestPage.xaml">
<SubType>Designer</SubType>
<Generator>MSBuild:Compile</Generator>
</Page>
<Page Include="$(MSBuildThisFileDirectory)NumberBoxPage.xaml">
<SubType>Designer</SubType>
<Generator>MSBuild:Compile</Generator>
</Page>
</ItemGroup>
<ItemGroup>
<Compile Include="$(MSBuildThisFileDirectory)NumberBoxAxeTestPage.xaml.cs">
<DependentUpon>NumberBoxAxeTestPage.xaml</DependentUpon>
</Compile>
<Compile Include="$(MSBuildThisFileDirectory)NumberBoxPage.xaml.cs">
<DependentUpon>NumberBoxPage.xaml</DependentUpon>
</Compile>
Expand Down
Loading

0 comments on commit fa28e62

Please sign in to comment.