Execute tests directly on Pipeline (#8439)

(cherry picked from commit 0a2d1ce)
microsoft · May 1, 2023 · e22d00e · e22d00e
1 parent e75d9a4
commit e22d00e
Show file tree

Hide file tree

Showing 24 changed files with 621 additions and 481 deletions.
diff --git a/PGO.runtime.props b/PGO.runtime.props
@@ -23,5 +23,9 @@
             <Link>%(Filename)%(Extension)</Link>
             <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
         </None>
+        <None Include="$(VCToolsRedistDir)\vc_redist.$(PlatformShortName).exe" Condition="'$(PlatformShortName)'!='ARM'">
+            <Link>%(Filename)%(Extension)</Link>
+            <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+        </None>
     </ItemGroup>
 </Project>
diff --git a/build/AzurePipelinesTemplates/MUX-BuildAndPublishPGONuGet-Job.yml b/build/AzurePipelinesTemplates/MUX-BuildAndPublishPGONuGet-Job.yml
@@ -6,8 +6,8 @@ parameters:
 jobs:
 - job: BuildAndPublishPGONuGet
   dependsOn: ${{ parameters.dependsOn }}
-  pool:
-    vmImage: 'windows-2019'
+  pool: 
+    name: WinDevPool-S
   variables:
     artifactsPath: $(Build.SourcesDirectory)\Artifacts
     pgoToolsPath: $(Build.SourcesDirectory)\tools\MUXPGODatabase

diff --git a/build/AzurePipelinesTemplates/MUX-BuildDevProject-Steps.yml b/build/AzurePipelinesTemplates/MUX-BuildDevProject-Steps.yml
@@ -36,7 +36,7 @@ steps:
       vsVersion: 16.0
       platform: '$(buildPlatform)'
       configuration: '$(buildConfiguration)'
-      msbuildArgs: '/restore /p:UseInsiderSDK=$(UseInsiderSDK) /p:AppxPackageDir="$(appxPackageDir)" /p:AppxBundle=Never /p:AppxSymbolPackageEnabled=false /binaryLogger:$(Build.SourcesDirectory)/MUXExperimental.sln.$(buildPlatform).$(buildConfiguration).binlog /p:MUXVersionBuild=$(builddate_yymm) /p:MUXVersionRevision=$(builddate_dd)$(buildrevision) /p:VCToolsInstallDir="$(VCToolsInstallDir)\" /p:PGOBuildMode=$(PGOBuildMode)'
+      msbuildArgs: '/restore /p:UseInsiderSDK=$(UseInsiderSDK) /p:AppxPackageDir="$(appxPackageDir)" /p:AppxBundle=Never /p:AppxSymbolPackageEnabled=false /binaryLogger:$(Build.SourcesDirectory)/MUXExperimental.sln.$(buildPlatform).$(buildConfiguration).binlog /p:MUXVersionBuild=$(builddate_yymm) /p:MUXVersionRevision=$(builddate_dd)$(buildrevision) /p:VCToolsInstallDir="$(VCToolsInstallDir)\" /p:VCToolsRedistDir="$(VCToolsRedistDir)\"'
 
   - task: PublishBuildArtifacts@1
     displayName: 'Publish MUXExperimental.sln binlog'

diff --git a/build/AzurePipelinesTemplates/MUX-BuildProject-Steps.yml b/build/AzurePipelinesTemplates/MUX-BuildProject-Steps.yml
@@ -34,7 +34,7 @@ steps:
       vsVersion: 16.0
       platform: '$(buildPlatform)'
       configuration: '$(buildConfiguration)'
-      msbuildArgs: '/restore /p:UseInsiderSDK=$(UseInsiderSDK) /p:AppxPackageDir="${{ parameters.appxPackageDir }}" /p:AppxBundle=Never /p:AppxSymbolPackageEnabled=false /binaryLogger:$(Build.SourcesDirectory)/${{ parameters.solutionPath }}.$(buildPlatform).$(buildConfiguration).binlog /p:MUXVersionBuild=$(builddate_yymm) /p:MUXVersionRevision=$(builddate_dd)$(buildrevision) /p:VCToolsInstallDir="$(VCToolsInstallDir)\" /p:PGOBuildMode=$(PGOBuildMode)'
+      msbuildArgs: '/restore /p:UseInsiderSDK=$(UseInsiderSDK) /p:AppxPackageDir="${{ parameters.appxPackageDir }}" /p:AppxBundle=Never /p:AppxSymbolPackageEnabled=false /binaryLogger:$(Build.SourcesDirectory)/${{ parameters.solutionPath }}.$(buildPlatform).$(buildConfiguration).binlog /p:MUXVersionBuild=$(builddate_yymm) /p:MUXVersionRevision=$(builddate_dd)$(buildrevision) /p:VCToolsInstallDir="$(VCToolsInstallDir)\" /p:VCToolsRedistDir="$(VCToolsRedistDir)\" /p:PGOBuildMode=$(PGOBuildMode)'
 
   - task: PublishBuildArtifacts@1
     displayName: 'Publish ${{ parameters.solutionPath }} binlog'

diff --git a/build/AzurePipelinesTemplates/MUX-CreateHelixProjFile-Steps.yml b/build/AzurePipelinesTemplates/MUX-CreateHelixProjFile-Steps.yml
@@ -2,6 +2,7 @@ parameters:
   condition: ''
   testBinaryDirectoryPath: '$(Build.SourcesDirectory)\HelixPayload'
   testFilePattern: ''
+  outputProjDir: $(Build.SourcesDirectory)\helixworkitems
   outputProjFileName: ''
   taefQuery: ''
 
@@ -12,4 +13,4 @@ steps:
     inputs:
       targetType: filePath
       filePath: build\Helix\GenerateTestProjFile.ps1
-      arguments: -TestFilePattern '${{ parameters.testFilePattern }}' -TestBinaryDirectoryPath '${{ parameters.testBinaryDirectoryPath }}' -OutputProjFile '$(Build.ArtifactStagingDirectory)\${{ parameters.outputProjFileName }}' -TaefQuery "${{ parameters.taefQuery }}" -TestNamePrefix $(buildConfiguration).$(buildPlatform)
+      arguments: -TestFilePattern '${{ parameters.testFilePattern }}' -TestBinaryDirectoryPath '${{ parameters.testBinaryDirectoryPath }}' -OutputProjFile '${{ parameters.outputProjDir }}\${{ parameters.outputProjFileName }}' -TaefQuery "${{ parameters.taefQuery }}" -TestNamePrefix $(buildConfiguration).$(buildPlatform)
diff --git a/build/AzurePipelinesTemplates/MUX-CreateNugetPackage-Job.yml b/build/AzurePipelinesTemplates/MUX-CreateNugetPackage-Job.yml
@@ -1,7 +1,6 @@
 parameters:
 # Extract the build revision number from Build.BuildNumber. This is needed to pass to build-nupkg
   jobName: ''
-  dependsOn: ''
   buildOutputDir: '$(Build.SourcesDirectory)\BuildOutput'
   nupkgdir: '$(build.artifactStagingDirectory)'
 # The "primary" build arch is the one that the nuspec gets its winmd, pri, and other neutral files from
@@ -13,11 +12,11 @@ parameters:
 
 jobs:
 - job: ${{ parameters.jobName }}
-  dependsOn:
-    - ${{ parameters.dependsOn }}
-
-  pool:
-    vmImage: 'windows-2019'
+  pool: 
+    ${{ if eq(variables['System.CollectionUri'], 'https://dev.azure.com/ms/') }}:
+      name: WinDevPoolOSS-S
+    ${{ if ne(variables['System.CollectionUri'], 'https://dev.azure.com/ms/') }}:
+      name: WinDevPool-S
 
   steps:
 

diff --git a/build/AzurePipelinesTemplates/MUX-MergePGD-Job.yml b/build/AzurePipelinesTemplates/MUX-MergePGD-Job.yml
@@ -6,8 +6,8 @@ parameters:
 jobs:
 - job: MergePGD
   dependsOn: ${{ parameters.dependsOn }}
-  pool:
-    vmImage: 'windows-2019'
+  pool: 
+    name: WinDevPool-S
   variables:
     artifactsPath: $(Build.SourcesDirectory)\Artifacts
     pgoArtifactsPath: $(artifactsPath)\${{ parameters.pgoArtifact }}
@@ -25,11 +25,23 @@ jobs:
       echo ##vso[task.setvariable variable=VCToolsInstallDir]%VCToolsInstallDir%
     displayName: 'Retrieve VC tools directory'
 
+  - task: DownloadBuildArtifacts@0
+    inputs:
+      artifactName: TestOutput-fwPkg
+      downloadPath: $(artifactsPath)\pgo
+
   - task: DownloadBuildArtifacts@0
     inputs:
       artifactName: ${{ parameters.pgoArtifact }}
       downloadPath: $(artifactsPath)
 
+  - task: powershell@2
+    displayName: CopyPGOFiles.ps1
+    inputs:
+      targetType: filePath
+      filePath: build\Helix\CopyPGOFiles.ps1
+      arguments: -SourceFolder '$(artifactsPath)\pgo\TestOutput-fwPkg\Win10-RS5\$(buildPlatform)$(buildConfiguration)' -OutputFolder '$(artifactsPath)'
+
   - script: |
       cd $(buildPlatform)
       "%VCToolsInstallDir%\bin\hostx64\x64\pgomgr.exe" /merge *.pgc ${{ parameters.pgdFile }}

diff --git a/build/AzurePipelinesTemplates/MUX-NugetReleaseTest-Job.yml b/build/AzurePipelinesTemplates/MUX-NugetReleaseTest-Job.yml
@@ -4,7 +4,6 @@ parameters:
   runTestJobName: 'RunNugetPkgTestsInHelix'
   dependsOn: ''
   runTests: 'true'
-  helixType: 'test/nuget'
   test_matrix: ''
   useFrameworkPkg: false
   taefQuery: ''
@@ -102,7 +101,9 @@ jobs:
       del %TEMP%\vsinstalldir.txt
       call "%_VSINSTALLDIR15%\Common7\Tools\VsDevCmd.bat"
       echo VCToolsInstallDir = %VCToolsInstallDir%
+      echo VCToolsRedistDir = %VCToolsRedistDir%
       echo ##vso[task.setvariable variable=VCToolsInstallDir]%VCToolsInstallDir%
+      echo ##vso[task.setvariable variable=VCToolsRedistDir]%VCToolsRedistDir%
     displayName: 'Retrieve VC tools directory'
 
   - template: MUX-BuildProject-Steps.yml
@@ -122,13 +123,13 @@ jobs:
 - ${{if eq(parameters.runTests, 'true')}}:
   - template: MUX-RunHelixTests-Job.yml
     parameters:
-      name: ${{ parameters.runTestJobName }}
       dependsOn: ${{ parameters.buildJobName }}
-      testSuite: 'NugetTestSuite'
-      helixType: ${{ parameters.helixType }}
+      ${{if eq(parameters.useFrameworkPkg, 'false')}}:
+        testSuite: 'nupkg'
+      ${{if eq(parameters.useFrameworkPkg, 'true')}}:
+        testSuite: 'fwPkg'
       artifactName: ${{ parameters.buildArtifactName }}
       ${{if eq(parameters.useFrameworkPkg, 'true')}}:
         taefQuery: "(not (@NugetPkgTestsOnly = 'true'))"
       rerunPassesRequiredToAvoidFailure: 5
-      ${{if ne(parameters.test_matrix, '') }}:
-        matrix: ${{ parameters.test_matrix }}
+      maxParallel: 2
diff --git a/build/AzurePipelinesTemplates/MUX-ProcessTestResults-Job.yml b/build/AzurePipelinesTemplates/MUX-ProcessTestResults-Job.yml
@@ -14,7 +14,7 @@ jobs:
   timeoutInMinutes: 120
   variables:
     helixOutputFolder: $(Build.SourcesDirectory)\HelixOutput
-    winuiHelixVersion: 0.0.2.6
+    winuiHelixVersion: 0.5.8
     helixScriptPath: $(Build.SourcesDirectory)\build\Helix\packages\Microsoft.Internal.WinUI.Helix.$(winuiHelixVersion)\scripts\pipeline
     ${{if eq(parameters.pgoArtifact, '') }}:
       processHelixFilesExtraArgs: ''

diff --git a/build/AzurePipelinesTemplates/MUX-RestorePgo-Steps.yml b/build/AzurePipelinesTemplates/MUX-RestorePgo-Steps.yml
@@ -6,7 +6,9 @@ steps:
       del %TEMP%\vsinstalldir.txt
       call "%_VSINSTALLDIR15%\Common7\Tools\VsDevCmd.bat"
       echo VCToolsInstallDir = %VCToolsInstallDir%
+      echo VCToolsRedistDir = %VCToolsRedistDir%
       echo ##vso[task.setvariable variable=VCToolsInstallDir]%VCToolsInstallDir%
+      echo ##vso[task.setvariable variable=VCToolsRedistDir]%VCToolsRedistDir%
     displayName: 'Retrieve VC tools directory'
 
   - task: powershell@2

diff --git a/build/AzurePipelinesTemplates/MUX-RunHelixTests-Job.yml b/build/AzurePipelinesTemplates/MUX-RunHelixTests-Job.yml
@@ -1,44 +1,41 @@
 parameters:
-  name: 'RunTestsInHelix'
   dependsOn: ''
-  condition: ''
-  testSuite: DevTestSuite
-  helixType: test/devtest
+  condition: succeeded()
+  testSuite: devtest 
   # If a Pipeline runs this template more than once, this parameter should be unique per build flavor to differentiate the
   # the different test runs:
   artifactName: 'drop'
-  maxParallel: 4
+  maxParallel: 10
   rerunPassesRequiredToAvoidFailure: 5
   taefQuery: ''
   # if 'useBuildOutputFromBuildId' is set, we will default to using a build from this pipeline:
-  useBuildOutputFromPipeline: $(System.DefinitionId)
+
   matrix: 
     Release_x86:
       buildPlatform: 'x86'
       buildConfiguration: 'release'
-      openHelixTargetQueues: 'windows.10.amd64.clientrs5.open.xaml'
-      closedHelixTargetQueues: 'windows.10.amd64.clientrs5.xaml'
     Release_x64:
       buildPlatform: 'x64'
       buildConfiguration: 'release'
-      openHelixTargetQueues: 'windows.10.amd64.client.open.reunion'
-      closedHelixTargetQueues: 'windows.10.amd64.client20h2.xaml'
 
 jobs:
-- job: ${{ parameters.name }}
+- job: CreateTestPayload_${{ parameters.testSuite }}
   dependsOn: ${{ parameters.dependsOn }}
   condition: ${{ parameters.condition }}
-  pool:
-    vmImage: 'windows-2019'
+  pool: 
+    ${{ if eq(variables['System.CollectionUri'], 'https://dev.azure.com/ms/') }}:
+      name: WinDevPoolOSS-S
+    ${{ if ne(variables['System.CollectionUri'], 'https://dev.azure.com/ms/') }}:
+      name: WinDevPool-S
   timeoutInMinutes: 120
   strategy:
-    maxParallel: ${{ parameters.maxParallel }}
+    maxParallel: 10
     matrix: ${{ parameters.matrix }}
   variables:
     artifactsDir: $(Build.SourcesDirectory)\Artifacts
-    helixCommonArgs: '/binaryLogger:$(Build.SourcesDirectory)/${{parameters.name}}.$(buildPlatform).$(buildConfiguration).binlog /p:HelixBuild=$(Build.BuildId).$(buildPlatform).$(buildConfiguration) /p:Platform=$(buildPlatform) /p:Configuration=$(buildConfiguration) /p:HelixType=${{parameters.helixType}}$(buildPlatform)$(buildConfiguration) /p:TestSuite=${{parameters.testSuite}} /p:ProjFilesPath=$(Build.ArtifactStagingDirectory) /p:rerunPassesRequiredToAvoidFailure=${{parameters.rerunPassesRequiredToAvoidFailure}}'
+    useBuildOutputFromPipeline: $(System.DefinitionId)
+    useBuildOutputFromBuildId: ''
 
-
   steps:
   - task: CmdLine@1
     displayName: 'Display build machine environment variables'
@@ -69,7 +66,7 @@ jobs:
       buildType: specific
       buildVersionToDownload: specific
       project: $(System.TeamProjectId)
-      pipeline: ${{ parameters.useBuildOutputFromPipeline }}
+      pipeline: $(useBuildOutputFromPipeline)
       buildId: $(useBuildOutputFromBuildId)
       artifactName: ${{ parameters.artifactName }} 
       downloadPath: '$(artifactsDir)'
@@ -87,63 +84,69 @@ jobs:
       filename: 'dir'
       arguments: '/s $(Build.SourcesDirectory)\HelixPayload'
 
-  - template: MUX-CreateHelixProjFile-Steps.yml
-    parameters:
-      condition: and(succeeded(),ne('${{ parameters.testSuite }}','NugetTestSuite'))
-      testBinaryDirectoryPath: '$(Build.SourcesDirectory)\HelixPayload\$(buildConfiguration)\$(buildPlatform)'
-      testFilePattern: 'MUXControls.Test.dll'
-      outputProjFileName: 'RunTestsInHelix-InteractionTests.proj'
-      taefQuery: ${{ parameters.taefQuery }}
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish artifact: TestPayload-${{ parameters.testSuite}}'
+    inputs:
+      PathtoPublish: $(Build.SourcesDirectory)\HelixPayload
+      artifactName: TestPayload-${{ parameters.testSuite}}
 
-  - template: MUX-CreateHelixProjFile-Steps.yml
-    parameters:
-      condition: and(succeeded(),ne('${{ parameters.testSuite }}','NugetTestSuite'))
-      testBinaryDirectoryPath: '$(Build.SourcesDirectory)\HelixPayload\$(buildConfiguration)\$(buildPlatform)'
-      testFilePattern: 'MUXControlsTestApp.msix'
-      outputProjFileName: 'RunTestsInHelix-ApiTests.proj'
-      taefQuery: ${{ parameters.taefQuery }}
 
-  - template: MUX-CreateHelixProjFile-Steps.yml
-    parameters:
-      condition: and(succeeded(),ne('${{ parameters.testSuite }}','NugetTestSuite'))
-      testBinaryDirectoryPath: '$(Build.SourcesDirectory)\HelixPayload\$(buildConfiguration)\$(buildPlatform)'
-      testFilePattern: 'IXMPTestApp.msix'
-      outputProjFileName: 'RunTestsInHelix-IXMPTestAppTests.proj'
-      taefQuery: ${{ parameters.taefQuery }}
+  - ${{ if eq(parameters.testSuite, 'devtest') }}:
+    - template: MUX-CreateHelixProjFile-Steps.yml
+      parameters:
+        testBinaryDirectoryPath: '$(Build.SourcesDirectory)\HelixPayload\$(buildConfiguration)\$(buildPlatform)'
+        testFilePattern: 'MUXControls.Test.dll'
+        outputProjFileName: 'RunTestsInHelix-InteractionTests.proj'
+        taefQuery: ${{ parameters.taefQuery }}
+    - template: MUX-CreateHelixProjFile-Steps.yml
+      parameters:
+        testBinaryDirectoryPath: '$(Build.SourcesDirectory)\HelixPayload\$(buildConfiguration)\$(buildPlatform)'
+        testFilePattern: 'MUXControlsTestApp.msix'
+        outputProjFileName: 'RunTestsInHelix-ApiTests.proj'
+        taefQuery: ${{ parameters.taefQuery }}
+    - template: MUX-CreateHelixProjFile-Steps.yml
+      parameters:
+        testBinaryDirectoryPath: '$(Build.SourcesDirectory)\HelixPayload\$(buildConfiguration)\$(buildPlatform)'
+        testFilePattern: 'IXMPTestApp.msix'
+        outputProjFileName: 'RunTestsInHelix-IXMPTestAppTests.proj'
+        taefQuery: ${{ parameters.taefQuery }}
 
-  - template: MUX-CreateHelixProjFile-Steps.yml
-    parameters:
-      condition: and(succeeded(),eq('${{ parameters.testSuite }}','NugetTestSuite'))
-      testBinaryDirectoryPath: '$(Build.SourcesDirectory)\HelixPayload\$(buildConfiguration)\$(buildPlatform)'
-      testFilePattern: 'MUXControls.ReleaseTest.dll'
-      outputProjFileName: 'RunTestsInHelix-NugetTests.proj'
-      taefQuery: ${{ parameters.taefQuery }}
+  - ${{ if in(parameters.testSuite, 'nupkg', 'fwpkg') }}:
+    - template: MUX-CreateHelixProjFile-Steps.yml
+      parameters:
+        testBinaryDirectoryPath: '$(Build.SourcesDirectory)\HelixPayload\$(buildConfiguration)\$(buildPlatform)'
+        testFilePattern: 'MUXControls.ReleaseTest.dll'
+        outputProjFileName: 'RunTestsInHelix-NugetTests.proj'
+        taefQuery: ${{ parameters.taefQuery }}
 
+  - task: CopyFiles@2
+    displayName: 'Copy helix work item proj files'
+    inputs:
+      SourceFolder: '$(Build.SourcesDirectory)\helixworkitems'
+      TargetFolder: '$(Build.ArtifactStagingDirectory)\$(buildPlatform)$(buildConfiguration)\'
+      Contents: '*.proj'
   - task: PublishBuildArtifacts@1
     displayName: 'Publish generated .proj files'
     inputs:
       PathtoPublish: $(Build.ArtifactStagingDirectory)
-      artifactName: ${{ parameters.artifactName }}
-
-  - task: DotNetCoreCLI@2
-    displayName: 'Run tests in Helix (open queues)'
-    condition: and(succeeded(),eq(variables['System.CollectionUri'],'https://dev.azure.com/ms/'))
-    env:
-      SYSTEM_ACCESSTOKEN: $(System.AccessToken)
-    inputs:
-      command: custom
-      projects: build\Helix\RunTestsInHelix.proj
-      custom: msbuild
-      arguments: '$(helixCommonArgs) /p:IsExternal=true /p:Creator=WinUI /p:HelixTargetQueues=$(openHelixTargetQueues)'
+      artifactName: TestWorkItems-${{ parameters.testSuite}}
 
-  - task: DotNetCoreCLI@2
-    displayName: 'Run tests in Helix (closed queues)'
-    condition: and(succeeded(),ne(variables['System.CollectionUri'],'https://dev.azure.com/ms/'))
-    env:
-      SYSTEM_ACCESSTOKEN: $(System.AccessToken)
-      HelixAccessToken: $(HelixApiAccessToken)
-    inputs:
-      command: custom
-      projects: build\Helix\RunTestsInHelix.proj
-      custom: msbuild
-      arguments: '$(helixCommonArgs) /p:HelixTargetQueues=$(closedHelixTargetQueues)'
+- template: MUX-RunTestsOnPipeline-Job.yml
+  parameters:
+    runTestJobName: RunTestsRS5x86_${{ parameters.testSuite }}
+    dependsOn: CreateTestPayload_${{ parameters.testSuite }}
+    parallel: ${{ parameters.maxParallel }}
+    testSuite: ${{ parameters.testSuite }}
+    testOS: Win10-RS5
+    buildPlatform: x86
+    buildConfiguration: release
+
+- template: MUX-RunTestsOnPipeline-Job.yml
+  parameters:
+    runTestJobName: RunTests22H2x64_${{ parameters.testSuite }}
+    dependsOn: CreateTestPayload_${{ parameters.testSuite }}
+    parallel: ${{ parameters.maxParallel }}
+    testSuite: ${{ parameters.testSuite }}
+    testOS: Win10-22H2
+    buildPlatform: x64
+    buildConfiguration: release