diff --git a/eng/ci/official-build.yml b/eng/ci/official-build.yml index 1455ec986c..640808112d 100644 --- a/eng/ci/official-build.yml +++ b/eng/ci/official-build.yml @@ -48,6 +48,8 @@ extends: codeql: # Move codeql for source languages to source analysis stage runSourceLanguagesInSourceAnalysis: true + spotBugs: + enabled: false # Explicitly disabling as our JavaWorker package causes this to light up and subsequently fail. stages: - stage: Build diff --git a/eng/ci/templates/official/jobs/build-artifacts-linux.yml b/eng/ci/templates/official/jobs/build-artifacts-linux.yml index ef6cf4b728..1e321d910a 100644 --- a/eng/ci/templates/official/jobs/build-artifacts-linux.yml +++ b/eng/ci/templates/official/jobs/build-artifacts-linux.yml @@ -6,24 +6,24 @@ jobs: project: src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj configuration: release runtime: linux-x64 - artifacts_path: $(Build.ArtifactStagingDirectory) - log_dir: $(artifacts_path)/log - zip_artifacts_path: $(artifacts_path)/Linux + drop_path: $(Build.ArtifactStagingDirectory) + log_path: $(drop_path)/logs + linux_drop_path: $(drop_path)/linux build_args: '-v m -c $(configuration) -r $(runtime) --self-contained true' templateContext: - outputParentDirectory: $(artifacts_path) + outputParentDirectory: $(drop_path) outputs: # TODO: onboard to Azure Artifacts Drops to allow accessing this from docker linux pipeline in msazure # https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/azure-artifacts/artifact-services-onboarding - output: pipelineArtifact displayName: Publish linux artifacts - path: $(zip_artifacts_path) - artifact: Linux + path: $(linux_drop_path) + artifact: drop_linux - output: pipelineArtifact displayName: Publish logs - path: $(log_dir) - artifact: Linux_Log + path: $(log_path) + artifact: logs_linux sbomEnabled: false condition: always() @@ -41,7 +41,7 @@ jobs: command: custom custom: restore projects: $(project) - arguments: '-v m -r $(runtime) -p:PublishReadyToRun=true -bl:$(log_dir)/restore.binlog' # add PublishReadyToRun=true for https://github.com/dotnet/sdk/issues/20701 + arguments: '-v m -r $(runtime) -p:PublishReadyToRun=true -bl:$(log_path)/restore.binlog' # add PublishReadyToRun=true for https://github.com/dotnet/sdk/issues/20701 - task: DotNetCoreCLI@2 displayName: Build @@ -49,7 +49,7 @@ jobs: command: custom custom: build projects: $(project) - arguments: '$(build_args) --no-restore -bl:$(log_dir)/build.binlog' + arguments: '$(build_args) --no-restore -bl:$(log_path)/build.binlog' - task: DotNetCoreCLI@2 displayName: Publish @@ -58,5 +58,6 @@ jobs: custom: publish publishWebProjects: false # we use our own publish logic zipAfterPublish: false # we use our own zip logic + modifyOutputPath: false projects: $(project) - arguments: '$(build_args) --no-build -p:ZipArtifactsPath=$(zip_artifacts_path) -bl:$(log_dir)/publish.binlog' + arguments: '$(build_args) --no-build -bl:$(log_path)/publish.binlog -o $(linux_drop_path)/host' diff --git a/eng/ci/templates/official/jobs/build-artifacts-windows.yml b/eng/ci/templates/official/jobs/build-artifacts-windows.yml index 7b3252da87..6747a05e4e 100644 --- a/eng/ci/templates/official/jobs/build-artifacts-windows.yml +++ b/eng/ci/templates/official/jobs/build-artifacts-windows.yml @@ -8,45 +8,41 @@ jobs: os: windows variables: - nuget_package_path: $(Build.ArtifactStagingDirectory)/NugetPackages - log_dir: $(Build.ArtifactStagingDirectory)/log + drop_path: $(Build.ArtifactStagingDirectory) + nuget_drop_path: $(drop_path)/nuget + windows_drop_path: $(drop_path)/windows + log_path: $(drop_path)/log templateContext: - outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputParentDirectory: $(drop_path) outputs: - output: pipelineArtifact displayName: Publish site extension - path: $(Build.ArtifactStagingDirectory)/SiteExtension - artifact: SiteExtension - - output: pipelineArtifact - displayName: Publish private site extension - path: $(Build.ArtifactStagingDirectory)/PrivateSiteExtension - artifact: PrivateSiteExtension - - output: pipelineArtifact - displayName: Publish site extension symbols - path: $(Build.ArtifactStagingDirectory)/SiteExtensionSymbols - artifact: Symbols + path: $(windows_drop_path) + artifact: drop_windows - output: pipelineArtifact displayName: Publish nuget packages - path: $(Build.ArtifactStagingDirectory)/NugetPackages - artifact: NugetPackages + path: $(nuget_drop_path) + artifact: drop_nuget - output: pipelineArtifact displayName: Publish logs - path: $(log_dir) - artifact: Windows_Log + path: $(log_path) + artifact: logs_windows sbomEnabled: false condition: always() steps: - template: /eng/ci/templates/install-dotnet.yml@self - template: /eng/ci/templates/steps/build-site-ext.yml@self + parameters: + publishDir: $(windows_drop_path) - task: DotNetCoreCLI@2 displayName: Build host packages inputs: command: custom custom: pack - arguments: -c release -o $(nuget_package_path) + arguments: -c release -o $(nuget_drop_path) projects: | **/WebJobs.Script.csproj **/WebJobs.Script.WebHost.csproj @@ -73,7 +69,7 @@ jobs: inputs: command: custom custom: pack - arguments: '--no-build -c release -o $(nuget_package_path)' + arguments: '--no-build -c release -o $(nuget_drop_path)' projects: | **/WebJobs.Script.Abstractions.csproj @@ -89,14 +85,14 @@ jobs: inputs: command: custom custom: pack - arguments: '--no-build -c release -o $(nuget_package_path)' + arguments: '--no-build -c release -o $(nuget_drop_path)' projects: | **/ExtensionsMetadataGenerator.csproj - template: ci/sign-files.yml@eng parameters: displayName: Sign NugetPackages - folderPath: $(nuget_package_path) + folderPath: $(nuget_drop_path) pattern: '*.nupkg' signType: nuget @@ -108,4 +104,4 @@ jobs: - task: DeleteFiles@1 displayName: Delete CodeSignSummary files inputs: - contents: '$(nuget_package_path)/**/CodeSignSummary-*.md' + contents: '$(nuget_drop_path)/**/CodeSignSummary-*.md' diff --git a/eng/ci/templates/steps/build-site-ext.yml b/eng/ci/templates/steps/build-site-ext.yml index a3c7c1d65a..53bcfaacc7 100644 --- a/eng/ci/templates/steps/build-site-ext.yml +++ b/eng/ci/templates/steps/build-site-ext.yml @@ -2,6 +2,9 @@ parameters: - name: project type: string default: src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.csproj +- name: publishDir + type: string + default: '' steps: # Restore must be a separate step so we can pass in 'PublishReadyToRun=true' @@ -11,7 +14,7 @@ steps: command: custom custom: restore projects: ${{ parameters.project }} - arguments: '-v m -p:PublishReadyToRun=true -bl:$(log_dir)/site_ext.restore.binlog' + arguments: '-v m -p:PublishReadyToRun=true -bl:$(log_path)/site_ext.restore.binlog' - task: DotNetCoreCLI@2 displayName: Build site extension @@ -19,7 +22,7 @@ steps: command: custom custom: build projects: ${{ parameters.project }} - arguments: '--no-restore -v m -c release -bl:$(log_dir)/site_ext.build.binlog' + arguments: '--no-restore -v m -c release -bl:$(log_path)/site_ext.build.binlog' - task: DotNetCoreCLI@2 displayName: Publish site extension @@ -28,5 +31,9 @@ steps: custom: publish publishWebProjects: false # we use our own publish logic zipAfterPublish: false # we use our own zip logic + modifyOutputPath: false projects: ${{ parameters.project }} - arguments: '--no-build -v m -c release -p:ZipArtifactsPath=$(Build.ArtifactStagingDirectory) -bl:$(log_dir)/site_ext.publish.binlog' + ${{ if eq(parameters.publishDir, '') }}: + arguments: '--no-build -v m -c release -bl:$(log_path)/site_ext.publish.binlog' + ${{ else }}: + arguments: '--no-build -v m -c release -bl:$(log_path)/site_ext.publish.binlog -o ${{ parameters.publishDir }}' diff --git a/src/WebJobs.Script.SiteExtension/New-PrivateSiteExtension.ps1 b/src/WebJobs.Script.SiteExtension/New-PrivateSiteExtension.ps1 new file mode 100644 index 0000000000..3cc1f79894 --- /dev/null +++ b/src/WebJobs.Script.SiteExtension/New-PrivateSiteExtension.ps1 @@ -0,0 +1,130 @@ +<# + .SYNOPSIS + Produces a private site extension. + + .DESCRIPTION + Takes in a published site extension and produces a private site extension. + + .PARAMETER InputPath + The path of the published 'SiteExtension'. Leave null to scan for root in a child from here. + + .PARAMETER OutputPath + The path to produce the private site extension to. When zipping, this is the name of the zip file. Leave null to compute this name. + + .PARAMETER Bitness + The bitness to produce the private site extension with. Default is '32bit'. + + .PARAMETER Zip + [Switch] Include to produce site extension as a zip. + + .PARAMETER Force + [Switch] Include to overwrite existing files. + + .INPUTS + None. You can't pipe objects to Update-Month.ps1. + + .OUTPUTS + None. Update-Month.ps1 doesn't generate any output. +#> + +param ( + [string] $InputPath = $null, + [string] $OutputPath = $null, + [ValidateSet('x64', '64bit', 'x86', '32bit')][string] $Bitness = '32bit', + [switch] $Zip, + [switch] $Force +) + +$normalizeBitness = @{ + 'x64' = '64bit' + '64bit' = '64bit' + 'x86' = '64bit' + '32bit' = '32bit' +} + +$Bitness = $normalizeBitness[$Bitness] + +if (-not $InputPath) +{ + $InputPath = (Get-ChildItem -Path . -Filter "extension.xml" -Recurse).Directory.FullName +} + +if (Test-Path (Join-Path $InputPath "WebJobs.Script.SiteExtension.csproj")) +{ + Write-Error "This script should not be ran in the WebJobs.Script.SiteExtension project folder. Run this script in the root of the published site extension folder." + exit 1 +} + +if (-not (Join-Path $InputPath "extension.xml" | Test-Path)) +{ + Write-Error "Unable to find published site extension." + exit 1 +} + +if (-not $OutputPath) +{ + $runtime = $Bitness -eq '32bit' ? 'win-x86' : 'win-x64' + $leaf = (Split-Path $InputPath -Leaf) + $split = $leaf.IndexOf('.') + $OutputPath = "$($leaf.Substring(0, $split)).Private.$($leaf.Substring($split + 1)).$runtime" +} + +function New-TemporaryDirectory { + $tmp = [System.IO.Path]::GetTempPath() + $name = (New-Guid).ToString("N") + return New-Item -ItemType Directory -Path (Join-Path $tmp $name) +} + +function Write-Zip ($outputPath) +{ + if (Test-Path $outputPath) { + if ($Force) { + Remove-Item -Path $outputPath -Recurse -Force + } else { + Write-Error "OutputPath already exists. Use -Force to overwrite." + exit 1 + } + } + + $tempDir = New-TemporaryDirectory + Write-Folder $tempDir + + Compress-Archive -Path "$tempDir/*" -DestinationPath $outputPath + Remove-Item -Path $tempDir -Recurse -Force +} + +function Write-Folder ($outputPath) +{ + if (Test-Path "$outputPath/*") { + if ($Force) { + Remove-Item -Path $outputPath -Recurse -Force + } else { + Write-Error "OutputPath already exists. Use -Force to overwrite." + exit 1 + } + } + + $inputPath = Get-ChildItem -Path $InputPath -Attributes Directory + $outputPath = Join-Path $outputPath "SiteExtensions" "Functions" + New-Item -ItemType Directory -Path $outputPath | Out-Null + + Copy-Item "$inputPath/applicationHost.xdt" -Destination $outputPath | Out-Null + + $filesDest = (Join-Path $outputPath "$Bitness") + Copy-Item "$inputPath/$Bitness/" -Destination "$filesDest/" -Container -Recurse | Out-Null + + $workerDest = (Join-Path $filesDest "workers") + Copy-Item "$inputPath/workers/" -Destination "$workerDest/" -Container -Recurse | Out-Null +} + +if ($Zip) { + if (-not $OutputPath.EndsWith(".zip")) { + $OutputPath = "$OutputPath.zip" + } + + Write-Zip $OutputPath +} else { + Write-Folder $OutputPath +} + +Write-Host "Published private site extension to $OutputPath" diff --git a/src/WebJobs.Script.SiteExtension/New-ReleaseJson.ps1 b/src/WebJobs.Script.SiteExtension/New-ReleaseJson.ps1 new file mode 100644 index 0000000000..c5c30f9a18 --- /dev/null +++ b/src/WebJobs.Script.SiteExtension/New-ReleaseJson.ps1 @@ -0,0 +1,41 @@ +param( + [string] $Version = $null, + [string[]] $Artifacts = $null, + [string] $CommitId = $null, + [string] $OutputPath +) + +if (-not $Version) { + $Version = dotnet build $PSScriptRoot --getProperty:Version +} + +if (-Not $Artifacts) { + $Artifacts = @( + "Functions.$Version" + "Functions.Symbols.$Version.win-x64" + "Functions.Symbols.$Version.win-x86" + ) +} + +if (-not $CommitId) { + $CommitId = $env:BUILD_SOURCEVERSION +} + +if (-not $CommitId) { + $CommitId = (git rev-parse HEAD).Trim() +} + +$obj = @{ + name = $Version + artifacts = $Artifacts + tag = "v$Version" + commitId = $CommitId + releaseNotesFile = "release_notes.md" +} + +if ($OutputPath) { + Write-Host "Writing $obj to $OutputPath" + $obj | ConvertTo-Json | Out-File $OutputPath +} + +return $obj diff --git a/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets b/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets index 7137585c65..010ba608ed 100644 --- a/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets +++ b/src/WebJobs.Script.SiteExtension/Publish.SingleTFM.targets @@ -10,11 +10,10 @@ ValidatePublishSettings; PublishProjectReferences; - PublishPrivateProjectReferences; RemoveUnneededRuntimes; MoveSymbols; - DeletePrivateSymbols; WriteHardLinkHashes; + WriteReleaseJson; ZipAfterPublish;ZipArtifactsPath @@ -27,15 +26,17 @@ - SiteExtension/$(Version)/ - $([MSBuild]::NormalizePath('$(PublishDir)$(SiteExtensionRelativeDir)')) - PrivateSiteExtension/SiteExtensions/Functions/ - $([MSBuild]::NormalizePath('$(PublishDir)$(PrivateSiteExtensionRelativeDir)')) + $(SiteExtensionName).$(Version)/ + $(SiteExtensionBaseDir)$(Version)/ + $([MSBuild]::NormalizePath($([System.IO.Path]::Combine($(PublishDir), $(SiteExtensionRelativeDir))))) + + + + - - + @@ -58,8 +59,6 @@ %(PublishRuntimeIdentifier.Identity) $(SiteExtensionDir)%(PublishRuntimeIdentifier.PublishDir)/ - $(PrivateSiteExtensionDir)%(PublishRuntimeIdentifier.PublishDir)/ - %(PublishRuntimeIdentifier.PrivateExtension) false @@ -75,7 +74,8 @@ <_PublishProjectReferenceExistent Include="@(_MSBuildProjectReferenceExistent)"> %(AdditionalProperties);PublishDir=%(PublishDir);_IsPublishing=true;PublishWorkers=false - + + - - - - <_PublishPrivateProjectReferenceExistent Include="@(_MSBuildProjectReferenceExistent)" Condition="%(PublishPrivate)"> - %(AdditionalProperties);PublishDir=%(PublishPrivateDir);_IsPublishing=true - - - - - - - <_RuntimesToRemove Include="@(PublishRuntimeIdentifier->'$(SiteExtensionDir)%(PublishDir)/runtimes/linux')" /> <_RuntimesToRemove Include="@(PublishRuntimeIdentifier->'$(SiteExtensionDir)%(PublishDir)/runtimes/osx')" /> - <_RuntimesToRemove Include="@(PublishRuntimeIdentifier->'$(PrivateSiteExtensionDir)%(PublishDir)/runtimes/linux')" Condition="%(PrivateExtension)" /> - <_RuntimesToRemove Include="@(PublishRuntimeIdentifier->'$(PrivateSiteExtensionDir)%(PublishDir)/runtimes/osx')" Condition="%(PrivateExtension)" /> @@ -125,33 +105,27 @@ <_SymbolDirs Include="@(PublishRuntimeIdentifier->'$(SiteExtensionDir)%(PublishDir)')"> - $(PublishDir)Symbols/$(SiteExtensionName).Symbols.$(Version).%(Identity) + $(PublishDir)$(SiteExtensionName).Symbols.$(Version).%(Identity) + $(SiteExtensionName).Symbols.$(Version).%(Identity) $(SiteExtensionName).Symbols.$(Version).%(Identity).zip - <_WorkerSymbols Include="$(SiteExtensionDir)workers/**/*.pdb" Destination="$(PublishDir)Symbols/$(SiteExtensionName).Symbols.$(Version).%(PublishRuntimeIdentifier.Identity)/workers" /> - + <_WorkerSymbols Include="$(SiteExtensionDir)workers/**/*.pdb" Destination="$(PublishDir)$(SiteExtensionName).Symbols.$(Version).%(PublishRuntimeIdentifier.Identity)/workers" /> + + - - - - <_PrivateSymbolsToRemove Include="$(PrivateSiteExtensionDir)/**/*.pdb" /> - - - - <_FilesToHash Include="$(SiteExtensionDir)**" /> - + - + @@ -161,8 +135,17 @@ + + + <_ReleaseArtifactsArg>@(ReleaseArtifact->'"%(Identity)"', ', ') + + + + + + diff --git a/src/WebJobs.Script.SiteExtension/Tasks.targets b/src/WebJobs.Script.SiteExtension/Tasks.targets index 67f3ef84c7..d3c43a65e8 100644 --- a/src/WebJobs.Script.SiteExtension/Tasks.targets +++ b/src/WebJobs.Script.SiteExtension/Tasks.targets @@ -97,4 +97,46 @@ for (int i = 0; i < Directories.Length; i++) + + + + + + + + + + + + + + { + try + { + var file = Files[i]; + using (var stream = File.OpenRead(file.ItemSpec)) + using (var algorithm = SHA256.Create()) + { + byte[] hash = algorithm.ComputeHash(stream); + file.SetMetadata("FileHash", Convert.ToBase64String(hash)); + } + } + catch (Exception ex) + { + Log.LogErrorFromException(ex); + } + }); + + Items = Files; + + if (Log.HasLoggedErrors) + { + return false; + } +]]> + + + + diff --git a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.csproj b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.csproj index b1e34b916c..85e55247ec 100644 --- a/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.csproj +++ b/src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.csproj @@ -18,8 +18,13 @@ - - + + + + + + +