From 133cfb33c8eda62e34fc527337f76c5396998a2d Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Wed, 27 Nov 2019 00:03:23 +0000 Subject: [PATCH 001/161] [master] Update dependencies from dotnet/arcade (#496) * Update dependencies from https://github.com/dotnet/arcade build 20191117.2 - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19567.2 * Update dependencies from https://github.com/dotnet/arcade build 20191124.1 - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19574.1 --- eng/common/templates/post-build/post-build.yml | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 8a8d84f2..7a4252a4 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -207,17 +207,4 @@ stages: channelId: 551 transportFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'NETCore_Experimental_Publishing' - channelName: '.NET Core Experimental' - channelId: 562 - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental-symbols/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal-symbols/nuget/v3/index.json' \ No newline at end of file From 347cfac279151a5ea1e282160a53461caca386f7 Mon Sep 17 00:00:00 2001 From: Tomas Matousek Date: Tue, 26 Nov 2019 15:47:26 -0800 Subject: [PATCH 002/161] Bump version to 1.1.0 --- eng/Versions.props | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Versions.props b/eng/Versions.props index 6df5c27f..1955a904 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -2,8 +2,8 @@ - 1.0.0 - beta2 + 1.1.0 + beta true true From 281402c8d34814a7a87de751e28c5d483bef3bfe Mon Sep 17 00:00:00 2001 From: Tomas Matousek Date: Tue, 26 Nov 2019 16:06:16 -0800 Subject: [PATCH 003/161] Publish all packages to ADO artifacts --- azure-pipelines.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 57459f59..489d957c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -91,6 +91,14 @@ stages: - script: eng\cibuild.cmd -configuration $(_BuildConfig) -prepareMachine $(_BuildArgs) displayName: Build and Test + # Archive NuGet packages to DevOps (workaround for https://github.com/dotnet/arcade/issues/4444) + - task: PublishBuildArtifacts@1 + displayName: Publish Artifact Packages + inputs: + PathtoPublish: 'artifacts\packages\$(BuildConfiguration)' + ArtifactName: 'Packages' + condition: succeeded() + - job: MacOS displayName: 'MacOS' pool: Hosted macOS From 48b11f62d80ebee10b79e6d35fa05069513de05e Mon Sep 17 00:00:00 2001 From: Tomas Matousek Date: Tue, 26 Nov 2019 15:13:16 -0800 Subject: [PATCH 004/161] Update readme to release 1.0.0 --- README.md | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index d4a2865d..aa286f53 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Source Link (preview[\*](#preview-status)) +# Source Link Source Link is a language- and source-control agnostic system for providing first-class source debugging experiences for binaries. The goal of the project is to enable anyone building [NuGet libraries to provide source debugging](https://github.com/dotnet/designs/blob/master/accepted/diagnostics/debugging-with-symbols-and-sources.md) for their users with almost no effort. Microsoft libraries, such as .NET Core and Roslyn have enabled Source Link. Source Link is supported by Microsoft. @@ -10,10 +10,6 @@ The [original Source Link implementation](https://github.com/ctaggart/SourceLink > If you arrived here from the original Source Link documentation - you do not need to use `SourceLink.Create.CommandLine`. You only need to install the packages listed below. -#### Preview Status - -We recommend using Source Link preview for building production packages. Although the implementation details and the public APIs of the Source Link pre-release packages are still in flux, there will be no difference in the format of the data it generates between preview and 1.0.0. The only changes in the generated data we expect are due to bug fixes that target scenario that previously did not work correctly. - ## Using Source Link in .NET projects You can enable Source Link experience in your own .NET project by setting a few properties and adding a PackageReference to a Source Link package: @@ -52,7 +48,7 @@ For projects hosted by [GitHub](http://github.com) or [GitHub Enterprise](https: ```xml - + ``` @@ -62,7 +58,7 @@ For projects hosted by [Azure Repos](https://azure.microsoft.com/en-us/services/ ```xml - + ``` @@ -73,7 +69,7 @@ For projects hosted by on-prem [Azure DevOps Server](https://azure.microsoft.com ```xml - + ``` @@ -93,7 +89,7 @@ For projects hosted by [GitLab](https://gitlab.com) reference [Microsoft.SourceL ```xml - + ``` @@ -103,7 +99,7 @@ For projects in git repositories hosted on [Bitbucket.org](https://bitbucket.org ```xml - + ``` @@ -130,9 +126,9 @@ To add Source Link support to your native project add package references corresp ```xml - - - + + + ``` From 14b08107f792a8341c08c399ecbf9fa6f683b105 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Matou=C5=A1ek?= Date: Tue, 26 Nov 2019 16:29:07 -0800 Subject: [PATCH 005/161] Update azure-pipelines.yml --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 489d957c..7bd4424f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -95,7 +95,7 @@ stages: - task: PublishBuildArtifacts@1 displayName: Publish Artifact Packages inputs: - PathtoPublish: 'artifacts\packages\$(BuildConfiguration)' + PathtoPublish: 'artifacts\packages\$(_BuildConfig)' ArtifactName: 'Packages' condition: succeeded() From 152df5690c8b751993369093074de920d72e3136 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Wed, 27 Nov 2019 13:27:32 +0000 Subject: [PATCH 006/161] Update dependencies from https://github.com/dotnet/arcade build 20191126.2 (#512) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19576.2 --- eng/Version.Details.xml | 4 ++-- eng/common/templates/post-build/post-build.yml | 15 ++++++++++++++- eng/common/tools.ps1 | 2 +- global.json | 2 +- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 976b5b1f..53aa923a 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 549475dff607986e6d9626f0dc8678e9736b7d0c + e1f099bf18a14e8ef5dc50f1a90078839aa102c8 diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 7a4252a4..8a8d84f2 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -207,4 +207,17 @@ stages: channelId: 551 transportFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal-symbols/nuget/v3/index.json' \ No newline at end of file + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal-symbols/nuget/v3/index.json' + +- template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NETCore_Experimental_Publishing' + channelName: '.NET Core Experimental' + channelId: 562 + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental-symbols/nuget/v3/index.json' diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 26a6d03c..d762c9f0 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -247,7 +247,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = if ($msbuildCmd -ne $null) { # Workaround for https://github.com/dotnet/roslyn/issues/35793 # Due to this issue $msbuildCmd.Version returns 0.0.0.0 for msbuild.exe 16.2+ - $msbuildVersion = [Version]::new((Get-Item $msbuildCmd.Path).VersionInfo.ProductVersion.Split(@('-', '+'))[0]) + $msbuildVersion = [Version]::new((Get-Item $msbuildCmd.Path).VersionInfo.ProductVersion.Split([char[]]@('-', '+'))[0]) if ($msbuildVersion -ge $vsMinVersion) { return $global:_MSBuildExe = $msbuildCmd.Path diff --git a/global.json b/global.json index c3494674..2d38a3b6 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19575.7" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19576.2" } } From 5467a64fb985433f589a79ed2a91d361f35cbb85 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 28 Nov 2019 13:28:06 +0000 Subject: [PATCH 007/161] Update dependencies from https://github.com/dotnet/arcade build 20191127.4 (#514) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19577.4 --- eng/Version.Details.xml | 4 ++-- eng/common/build.ps1 | 19 ++++++++++--------- global.json | 2 +- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 53aa923a..5d788b2d 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - e1f099bf18a14e8ef5dc50f1a90078839aa102c8 + e103549927549c5b5b8cf02479f8e3b021037001 diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index ce6882ee..0fd4d9c7 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -112,17 +112,18 @@ function Build { @properties } -if ($clean) { - if(Test-Path $ArtifactsDir) { - Remove-Item -Recurse -Force $ArtifactsDir - Write-Host 'Artifacts directory deleted.' - } - exit 0 -} - try { . $PSScriptRoot\tools.ps1 - If ((Test-Path variable:LastExitCode) -And ($LastExitCode -ne 0)) { + + if ($clean) { + if (Test-Path $ArtifactsDir) { + Remove-Item -Recurse -Force $ArtifactsDir + Write-Host 'Artifacts directory deleted.' + } + exit 0 + } + + if ((Test-Path variable:LastExitCode) -And ($LastExitCode -ne 0)) { Write-PipelineTelemetryError -Category 'InitializeToolset' -Message 'Eng/common/tools.ps1 returned a non-zero exit code.' ExitWithExitCode $LastExitCode } diff --git a/global.json b/global.json index 2d38a3b6..f59fce88 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19576.2" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19577.4" } } From 30650587bbadab9d47881f3cd7e108002a04156c Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 29 Nov 2019 13:38:02 +0000 Subject: [PATCH 008/161] Update dependencies from https://github.com/dotnet/arcade build 20191128.1 (#516) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19578.1 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 5d788b2d..f3226364 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - e103549927549c5b5b8cf02479f8e3b021037001 + 7d8af3c802523cd7d9df9f6cb4ce49236851ca0f diff --git a/global.json b/global.json index f59fce88..0553d805 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19577.4" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19578.1" } } From 37dee44acff489c9762e656a403f607ecedc474a Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Sat, 30 Nov 2019 13:37:57 +0000 Subject: [PATCH 009/161] Update dependencies from https://github.com/dotnet/arcade build 20191129.1 (#518) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19579.1 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index f3226364..638c439f 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 7d8af3c802523cd7d9df9f6cb4ce49236851ca0f + ab04a0fb86ba6bcb63b59317833c47996facafdf diff --git a/global.json b/global.json index 0553d805..7fb01db3 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19578.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19579.1" } } From 50ce5ce432df1cbd902644b2926bb5df5d4b6005 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Sun, 1 Dec 2019 13:37:33 +0000 Subject: [PATCH 010/161] Update dependencies from https://github.com/dotnet/arcade build 20191130.1 (#520) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19580.1 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 638c439f..6162f0cf 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - ab04a0fb86ba6bcb63b59317833c47996facafdf + d58525128d68de745027d7eaf83b3a9e178e23df diff --git a/global.json b/global.json index 7fb01db3..8dff9823 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19579.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19580.1" } } From 7fe93a7639354d89c9d9899bcee91e897c18642d Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2019 13:35:48 +0000 Subject: [PATCH 011/161] Update dependencies from https://github.com/dotnet/arcade build 20191201.1 (#521) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19601.1 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 6162f0cf..8bd6f3d7 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - d58525128d68de745027d7eaf83b3a9e178e23df + c0b56ff3569e3c7475070486c40543ea4c6f6dc7 diff --git a/global.json b/global.json index 8dff9823..7ebe7404 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19580.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19601.1" } } From 604d4b233f966070682217a0a9a8d6892610771e Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2019 13:39:06 +0000 Subject: [PATCH 012/161] Update dependencies from https://github.com/dotnet/arcade build 20191201.1 (#522) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19601.1 From 66352cbc688d59d36a6a229a2418874579bf0052 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Tue, 3 Dec 2019 13:28:10 +0000 Subject: [PATCH 013/161] Update dependencies from https://github.com/dotnet/arcade build 20191202.4 (#523) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19602.4 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 8bd6f3d7..0288911f 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - c0b56ff3569e3c7475070486c40543ea4c6f6dc7 + 9d34fd008e754e1ada35c8b6bc3694e7a90b4ed7 diff --git a/global.json b/global.json index 7ebe7404..8551add7 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19601.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19602.4" } } From d51002eaa4e4ce7ea2f9ee7ca30ccbdc44bba9ca Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Wed, 4 Dec 2019 13:38:06 +0000 Subject: [PATCH 014/161] Update dependencies from https://github.com/dotnet/arcade build 20191203.17 (#525) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19603.17 --- eng/Version.Details.xml | 4 +- eng/common/SetupNugetSources.ps1 | 65 ++++++++++--------- eng/common/SetupNugetSources.sh | 78 ++++++++++++++++------- eng/common/build.ps1 | 11 +--- eng/common/init-tools-native.ps1 | 6 +- eng/common/native/install-tool.ps1 | 2 +- eng/common/pipeline-logging-functions.ps1 | 2 +- global.json | 2 +- 8 files changed, 103 insertions(+), 67 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 0288911f..c39e4f27 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 9d34fd008e754e1ada35c8b6bc3694e7a90b4ed7 + b902fd6b6948e689a5128fa6d94dc7de13e6af84 diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index 6662e4bc..a5a1e711 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -16,7 +16,7 @@ # condition: eq(variables['Agent.OS'], 'Windows_NT') # inputs: # filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 -# arguments: -ConfigFile ${Env:BUILD_SOURCESDIRECTORY}/NuGet.config -Password $Env:Token +# arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token # env: # Token: $(dn-bot-dnceng-artifact-feeds-rw) @@ -94,41 +94,48 @@ function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Password) { } } -try { - if (!(Test-Path $ConfigFile -PathType Leaf)) { - Write-PipelineTelemetryError -Category 'Build' -Message "Couldn't find the file NuGet config file: $ConfigFile" +if (!(Test-Path $ConfigFile -PathType Leaf)) { + Write-PipelineTelemetryError -Category 'Build' -Message "Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. Couldn't find the NuGet config file: $ConfigFile" + ExitWithExitCode 1 +} + +if (!$Password) { + Write-PipelineTelemetryError -Category 'Build' -Message 'Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. Please supply a valid PAT' ExitWithExitCode 1 - } +} - # Load NuGet.config - $doc = New-Object System.Xml.XmlDocument - $filename = (Get-Item $ConfigFile).FullName - $doc.Load($filename) +# Load NuGet.config +$doc = New-Object System.Xml.XmlDocument +$filename = (Get-Item $ConfigFile).FullName +$doc.Load($filename) - # Get reference to or create one if none exist already - $sources = $doc.DocumentElement.SelectSingleNode("packageSources") - if ($sources -eq $null) { - $sources = $doc.CreateElement("packageSources") - $doc.DocumentElement.AppendChild($sources) | Out-Null - } +# Get reference to or create one if none exist already +$sources = $doc.DocumentElement.SelectSingleNode("packageSources") +if ($sources -eq $null) { + $sources = $doc.CreateElement("packageSources") + $doc.DocumentElement.AppendChild($sources) | Out-Null +} - # Looks for a node. Create it if none is found. - $creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials") - if ($creds -eq $null) { - $creds = $doc.CreateElement("packageSourceCredentials") - $doc.DocumentElement.AppendChild($creds) | Out-Null - } +# Looks for a node. Create it if none is found. +$creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials") +if ($creds -eq $null) { + $creds = $doc.CreateElement("packageSourceCredentials") + $doc.DocumentElement.AppendChild($creds) | Out-Null +} - # Insert credential nodes for Maestro's private feeds - InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Password $Password +# Insert credential nodes for Maestro's private feeds +InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Password $Password +$dotnet3Source = $sources.SelectSingleNode("add[@key='dotnet3']") +if ($dotnet3Source -ne $null) { AddPackageSource -Sources $sources -SourceName "dotnet3-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password AddPackageSource -Sources $sources -SourceName "dotnet3-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal-transport/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password - - $doc.Save($filename) } -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_ - ExitWithExitCode 1 + +$dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']") +if ($dotnet31Source -ne $null) { + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password } + +$doc.Save($filename) diff --git a/eng/common/SetupNugetSources.sh b/eng/common/SetupNugetSources.sh index 55ad70e3..7d6fef27 100644 --- a/eng/common/SetupNugetSources.sh +++ b/eng/common/SetupNugetSources.sh @@ -17,7 +17,7 @@ # displayName: Setup Private Feeds Credentials # inputs: # filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh -# arguments: $BUILD_SOURCESDIRECTORY/NuGet.config $Token +# arguments: $(Build.SourcesDirectory)/NuGet.config $Token # condition: ne(variables['Agent.OS'], 'Windows_NT') # env: # Token: $(dn-bot-dnceng-artifact-feeds-rw) @@ -42,7 +42,12 @@ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" . "$scriptroot/tools.sh" if [ ! -f "$ConfigFile" ]; then - Write-PipelineTelemetryError -Category 'Build' -Message "Couldn't find the file NuGet config file: $ConfigFile" + Write-PipelineTelemetryError -Category 'Build' "Error: Eng/common/SetupNugetSources.sh returned a non-zero exit code. Couldn't find the NuGet config file: $ConfigFile" + ExitWithExitCode 1 +fi + +if [ -z "$CredToken" ]; then + Write-PipelineTelemetryError -category 'Build' "Error: Eng/common/SetupNugetSources.sh returned a non-zero exit code. Please supply a valid PAT" ExitWithExitCode 1 fi @@ -52,7 +57,7 @@ if [[ `uname -s` == "Darwin" ]]; then fi # Ensure there is a ... section. -grep -i "" $ConfigFile +grep -i "" $ConfigFile if [ "$?" != "0" ]; then echo "Adding ... section." ConfigNodeHeader="" @@ -62,7 +67,7 @@ if [ "$?" != "0" ]; then fi # Ensure there is a ... section. -grep -i "" $ConfigFile +grep -i "" $ConfigFile if [ "$?" != "0" ]; then echo "Adding ... section." @@ -72,37 +77,64 @@ if [ "$?" != "0" ]; then sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourcesNodeFooter${NL}$PackageSourceCredentialsTemplate|" NuGet.config fi -# Ensure dotnet3-internal and dotnet3-internal-transport is in the packageSources -grep -i "" $ConfigFile -if [ "$?" != "0" ]; then - echo "Adding dotnet3-internal to the packageSources." +PackageSources=() - PackageSourcesNodeFooter="" - PackageSourceTemplate="${TB}" +# Ensure dotnet3-internal and dotnet3-internal-transport are in the packageSources if the public dotnet3 feeds are present +grep -i "" $ConfigFile + if [ "$?" != "0" ]; then + echo "Adding dotnet3-internal to the packageSources." + PackageSourcesNodeFooter="" + PackageSourceTemplate="${TB}" + + sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile + fi + PackageSources+=('dotnet3-internal') + + grep -i "" - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" NuGet.config + sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile + fi + PackageSources+=('dotnet3-internal-transport') fi -# Ensure dotnet3-internal and dotnet3-internal-transport is in the packageSources -grep -i "" $ConfigFile -if [ "$?" != "0" ]; then - echo "Adding dotnet3-internal-transport to the packageSources." +# Ensure dotnet3.1-internal and dotnet3.1-internal-transport are in the packageSources if the public dotnet3.1 feeds are present +grep -i "" - PackageSourcesNodeFooter="" - PackageSourceTemplate="${TB}" + sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile + fi + PackageSources+=('dotnet3.1-internal') + + grep -i "" $ConfigFile + if [ "$?" != "0" ]; then + echo "Adding dotnet3.1-internal-transport to the packageSources." + PackageSourcesNodeFooter="" + PackageSourceTemplate="${TB}" - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" NuGet.config + sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile + fi + PackageSources+=('dotnet3.1-internal-transport') fi # I want things split line by line PrevIFS=$IFS IFS=$'\n' -PackageSources=$(grep -oh '"darc-int-[^"]*"' $ConfigFile | tr -d '"') +PackageSources+="$IFS" +PackageSources+=$(grep -oh '"darc-int-[^"]*"' $ConfigFile | tr -d '"') IFS=$PrevIFS -PackageSources+=('dotnet3-internal') -PackageSources+=('dotnet3-internal-transport') - for FeedName in ${PackageSources[@]} ; do # Check if there is no existing credential for this FeedName grep -i "<$FeedName>" $ConfigFile @@ -112,6 +144,6 @@ for FeedName in ${PackageSources[@]} ; do PackageSourceCredentialsNodeFooter="" NewCredential="${TB}${TB}<$FeedName>${NL}${NL}${NL}" - sed -i.bak "s|$PackageSourceCredentialsNodeFooter|$NewCredential${NL}$PackageSourceCredentialsNodeFooter|" NuGet.config + sed -i.bak "s|$PackageSourceCredentialsNodeFooter|$NewCredential${NL}$PackageSourceCredentialsNodeFooter|" $ConfigFile fi done diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 0fd4d9c7..88814514 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -62,6 +62,8 @@ function Print-Usage() { Write-Host "The above arguments can be shortened as much as to be unambiguous (e.g. -co for configuration, -t for test, etc.)." } +. $PSScriptRoot\tools.ps1 + function InitializeCustomToolset { if (-not $restore) { return @@ -113,8 +115,6 @@ function Build { } try { - . $PSScriptRoot\tools.ps1 - if ($clean) { if (Test-Path $ArtifactsDir) { Remove-Item -Recurse -Force $ArtifactsDir @@ -122,12 +122,7 @@ try { } exit 0 } - - if ((Test-Path variable:LastExitCode) -And ($LastExitCode -ne 0)) { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message 'Eng/common/tools.ps1 returned a non-zero exit code.' - ExitWithExitCode $LastExitCode - } - + if ($help -or (($null -ne $properties) -and ($properties.Contains('/help') -or $properties.Contains('/?')))) { Print-Usage exit 0 diff --git a/eng/common/init-tools-native.ps1 b/eng/common/init-tools-native.ps1 index f4409f08..db830c00 100644 --- a/eng/common/init-tools-native.ps1 +++ b/eng/common/init-tools-native.ps1 @@ -113,14 +113,16 @@ try { } $toolInstallationFailure = $true } else { - Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message $errMsg + # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 + Write-Host $errMsg exit 1 } } } if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) { - Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message 'Native tools bootstrap failed' + # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 + Write-Host 'Native tools bootstrap failed' exit 1 } } diff --git a/eng/common/native/install-tool.ps1 b/eng/common/native/install-tool.ps1 index ca180d03..f397e1c7 100644 --- a/eng/common/native/install-tool.ps1 +++ b/eng/common/native/install-tool.ps1 @@ -105,7 +105,7 @@ try { Write-Error "There are multiple copies of $ToolName in $($ToolInstallDirectory): `n$(@($ToolFilePath | out-string))" exit 1 } elseif (@($ToolFilePath).Length -Lt 1) { - Write-Error "$ToolName was not found in $ToolFilePath." + Write-Host "$ToolName was not found in $ToolFilePath." exit 1 } diff --git a/eng/common/pipeline-logging-functions.ps1 b/eng/common/pipeline-logging-functions.ps1 index e0656185..2688c389 100644 --- a/eng/common/pipeline-logging-functions.ps1 +++ b/eng/common/pipeline-logging-functions.ps1 @@ -85,7 +85,7 @@ function Write-PipelineTaskError { [switch]$AsOutput, [bool]$IsMultiJobVariable=$true) - if(-Not (Test-Path variable:ci) -Or !$ci) { + if((Test-Path variable:ci) -And $ci) { Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $Value -Properties @{ 'variable' = $Name 'isSecret' = $Secret diff --git a/global.json b/global.json index 8551add7..a83d4669 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19602.4" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19603.17" } } From 60024e13401e5b2f9a19a755be069647a1d0908f Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 5 Dec 2019 13:28:02 +0000 Subject: [PATCH 015/161] Update dependencies from https://github.com/dotnet/arcade build 20191204.4 (#526) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19604.4 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index c39e4f27..9ddaffad 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - b902fd6b6948e689a5128fa6d94dc7de13e6af84 + f2999360b70f05d7738cbfa2b61d1fdaff4ee77d diff --git a/global.json b/global.json index a83d4669..afe721b7 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19603.17" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19604.4" } } From 5f03e17cb71673d903adee17485aff281571dd90 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 6 Dec 2019 13:26:42 +0000 Subject: [PATCH 016/161] Update dependencies from https://github.com/dotnet/arcade build 20191205.6 (#527) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19605.6 --- eng/Version.Details.xml | 4 ++-- eng/common/pipeline-logging-functions.ps1 | 4 ++-- global.json | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 9ddaffad..ec9fd0ef 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - f2999360b70f05d7738cbfa2b61d1fdaff4ee77d + 45bfe0972a7354356daf799aae88533dcd0f071a diff --git a/eng/common/pipeline-logging-functions.ps1 b/eng/common/pipeline-logging-functions.ps1 index 2688c389..a3e1317a 100644 --- a/eng/common/pipeline-logging-functions.ps1 +++ b/eng/common/pipeline-logging-functions.ps1 @@ -101,7 +101,7 @@ function Write-PipelineTaskError { [string]$Path, [switch]$AsOutput) - if(-Not (Test-Path variable:ci) -Or !$ci) { + if((Test-Path variable:ci) -And $ci) { Write-LoggingCommand -Area 'task' -Event 'prependpath' -Data $Path -AsOutput:$AsOutput } } @@ -237,4 +237,4 @@ function Write-LogIssue { } Write-Host $command -ForegroundColor $foregroundColor -BackgroundColor $backgroundColor -} \ No newline at end of file +} diff --git a/global.json b/global.json index afe721b7..e2f13fe3 100644 --- a/global.json +++ b/global.json @@ -1,8 +1,8 @@ { "tools": { - "dotnet": "3.0.100" + "dotnet": "3.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19604.4" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19605.6" } } From 14da0f10e6681ac815b9474a577818ba472ebbb8 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Sat, 7 Dec 2019 13:26:40 +0000 Subject: [PATCH 017/161] Update dependencies from https://github.com/dotnet/arcade build 20191206.1 (#528) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19606.1 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index ec9fd0ef..ecda36d8 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 45bfe0972a7354356daf799aae88533dcd0f071a + 4c050b4705ee1743665e7ad721dfafdc9b82a4b7 diff --git a/global.json b/global.json index e2f13fe3..e84ccce4 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19605.6" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19606.1" } } From 780712fdd5bc4e4b762264976fb8e3dca0bd7d23 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Sun, 8 Dec 2019 13:26:16 +0000 Subject: [PATCH 018/161] Update dependencies from https://github.com/dotnet/arcade build 20191207.1 (#529) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19607.1 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index ecda36d8..dbacb1f3 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 4c050b4705ee1743665e7ad721dfafdc9b82a4b7 + 97cce7cb41de117e849d42a2eb7c99065dd1602f diff --git a/global.json b/global.json index e84ccce4..e5fa2339 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19606.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19607.1" } } From 2c59a9efc779fe5cb1db33d3b60397419adfddc2 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 9 Dec 2019 13:36:08 +0000 Subject: [PATCH 019/161] Update dependencies from https://github.com/dotnet/arcade build 20191208.1 (#531) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19608.1 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index dbacb1f3..b6709fa1 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 97cce7cb41de117e849d42a2eb7c99065dd1602f + 71ce4c736b882e6112b395a0e92313be5dcb4328 diff --git a/global.json b/global.json index e5fa2339..a8159c42 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19607.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19608.1" } } From d029a3961b05f1ce8954dd7787afa40a36f6dc79 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 9 Dec 2019 14:07:07 +0000 Subject: [PATCH 020/161] Update dependencies from https://github.com/dotnet/arcade build 20191208.1 (#532) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19608.1 From 4a59edde7b88f2ce9f378dd010439aa14f8b76a1 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2019 13:45:58 +0000 Subject: [PATCH 021/161] Update dependencies from https://github.com/dotnet/arcade build 20191211.6 (#533) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19611.6 --- eng/Version.Details.xml | 4 ++-- eng/common/tools.ps1 | 28 +++++++++++++++++++++++++--- eng/common/tools.sh | 2 ++ global.json | 2 +- 4 files changed, 30 insertions(+), 6 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index b6709fa1..0476544f 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 71ce4c736b882e6112b395a0e92313be5dcb4328 + 71a91bbb3c49bad2f1e7f2c7f28b32c5c0efc9e5 diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index d762c9f0..09c3a332 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -98,7 +98,7 @@ function Exec-Process([string]$command, [string]$commandArgs) { } } -function InitializeDotNetCli([bool]$install) { +function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { if (Test-Path variable:global:_DotNetInstallDir) { return $global:_DotNetInstallDir } @@ -146,6 +146,22 @@ function InitializeDotNetCli([bool]$install) { } $env:DOTNET_INSTALL_DIR = $dotnetRoot + + if ($createSdkLocationFile) { + # Create a temporary file under the toolset dir and rename it to sdk.txt to avoid races. + do { + $sdkCacheFileTemp = Join-Path $ToolsetDir $([System.IO.Path]::GetRandomFileName()) + } + until (!(Test-Path $sdkCacheFileTemp)) + Set-Content -Path $sdkCacheFileTemp -Value $dotnetRoot + + try { + Rename-Item -Force -Path $sdkCacheFileTemp 'sdk.txt' + } catch { + # Somebody beat us + Remove-Item -Path $sdkCacheFileTemp + } + } } # Add dotnet to PATH. This prevents any bare invocation of dotnet in custom @@ -216,7 +232,10 @@ function InstallDotNet([string] $dotnetRoot, } catch { Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet runtime '$runtime' from custom location '$runtimeSourceFeed'." + ExitWithExitCode 1 } + } else { + ExitWithExitCode 1 } } } @@ -274,8 +293,11 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = $vsMajorVersion = $vsMinVersion.Major $xcopyMSBuildVersion = "$vsMajorVersion.$($vsMinVersion.Minor).0-alpha" } - - $vsInstallDir = InitializeXCopyMSBuild $xcopyMSBuildVersion $install + + $vsInstallDir = $null + if ($xcopyMSBuildVersion.Trim() -ine "none") { + $vsInstallDir = InitializeXCopyMSBuild $xcopyMSBuildVersion $install + } if ($vsInstallDir -eq $null) { throw 'Unable to find Visual Studio that has required version and components installed' } diff --git a/eng/common/tools.sh b/eng/common/tools.sh index f58db5e2..e071af4e 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -211,6 +211,8 @@ function InstallDotNet { Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from custom location '$runtimeSourceFeed' (exit code '$exit_code')." ExitWithExitCode $exit_code } + else + ExitWithExitCode $exit_code fi fi } diff --git a/global.json b/global.json index a8159c42..6ca314f7 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19608.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19611.6" } } From 35dffc64ff1780bd1802861eb8ff4980ab2540f4 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2019 13:45:33 +0000 Subject: [PATCH 022/161] Update dependencies from https://github.com/dotnet/arcade build 20191212.1 (#534) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19612.1 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 0476544f..e59a5f8f 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 71a91bbb3c49bad2f1e7f2c7f28b32c5c0efc9e5 + b590bede6a5244ee732c63c99f725728cabbda48 diff --git a/global.json b/global.json index 6ca314f7..2b6c0204 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19611.6" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19612.1" } } From b105c57471fc69b7b1df7c3584d700981314c0c8 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Sat, 14 Dec 2019 13:24:35 +0000 Subject: [PATCH 023/161] Update dependencies from https://github.com/dotnet/arcade build 20191213.1 (#535) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19613.1 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index e59a5f8f..d91c9a63 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - b590bede6a5244ee732c63c99f725728cabbda48 + acd18a717ab18e484addef602e2af2b2bc4c4bd1 diff --git a/global.json b/global.json index 2b6c0204..5bf3fc03 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19612.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19613.1" } } From cb4668c4dd25e4afb455f80a287f2b7899116460 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Sun, 15 Dec 2019 13:25:26 +0000 Subject: [PATCH 024/161] Update dependencies from https://github.com/dotnet/arcade build 20191214.1 (#536) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19614.1 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index d91c9a63..6e4ca65d 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - acd18a717ab18e484addef602e2af2b2bc4c4bd1 + 64e95735b49d37b6cf908cfa5228ff65bee6dfff diff --git a/global.json b/global.json index 5bf3fc03..dd5c15d3 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19613.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19614.1" } } From 09457c998aac519ac231353c1caecbcbafa37d42 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2019 13:20:39 +0000 Subject: [PATCH 025/161] Update dependencies from https://github.com/dotnet/arcade build 20191215.1 (#538) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19615.1 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 6e4ca65d..7e92b0a4 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 64e95735b49d37b6cf908cfa5228ff65bee6dfff + 77698ebba1ab096ad9580a5d9ebefe0499ba00ce diff --git a/global.json b/global.json index dd5c15d3..bc3abbd6 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19614.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19615.1" } } From 29b2e53dcbba930e2ddc9a67d00b44aa55a9ef84 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2019 13:46:58 +0000 Subject: [PATCH 026/161] Update dependencies from https://github.com/dotnet/arcade build 20191215.1 (#539) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19615.1 From 6ca94ba66432ed1dedd09941c421934804e772db Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2019 13:46:34 +0000 Subject: [PATCH 027/161] Update dependencies from https://github.com/dotnet/arcade build 20191216.4 (#540) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19616.4 --- eng/Version.Details.xml | 4 +-- eng/common/templates/steps/publish-logs.yml | 2 +- eng/common/tools.ps1 | 33 ++++++++++++--------- global.json | 2 +- 4 files changed, 23 insertions(+), 18 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 7e92b0a4..5ad9b6c1 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 77698ebba1ab096ad9580a5d9ebefe0499ba00ce + 4736ddca60a4040447b450437cf3767c9045edc0 diff --git a/eng/common/templates/steps/publish-logs.yml b/eng/common/templates/steps/publish-logs.yml index 8903ba57..f91751fe 100644 --- a/eng/common/templates/steps/publish-logs.yml +++ b/eng/common/templates/steps/publish-logs.yml @@ -18,6 +18,6 @@ steps: inputs: PathtoPublish: '$(Build.SourcesDirectory)/PostBuildLogs' PublishLocation: Container - ArtifactName: PostBuilLogs + ArtifactName: PostBuildLogs continueOnError: true condition: always() diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 09c3a332..d3a43287 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -98,6 +98,9 @@ function Exec-Process([string]$command, [string]$commandArgs) { } } +# createSdkLocationFile parameter enables a file being generated under the toolset directory +# which writes the sdk's location into. This is only necessary for cmd --> powershell invocations +# as dot sourcing isn't possible. function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { if (Test-Path variable:global:_DotNetInstallDir) { return $global:_DotNetInstallDir @@ -146,21 +149,23 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { } $env:DOTNET_INSTALL_DIR = $dotnetRoot + } - if ($createSdkLocationFile) { - # Create a temporary file under the toolset dir and rename it to sdk.txt to avoid races. - do { - $sdkCacheFileTemp = Join-Path $ToolsetDir $([System.IO.Path]::GetRandomFileName()) - } - until (!(Test-Path $sdkCacheFileTemp)) - Set-Content -Path $sdkCacheFileTemp -Value $dotnetRoot - - try { - Rename-Item -Force -Path $sdkCacheFileTemp 'sdk.txt' - } catch { - # Somebody beat us - Remove-Item -Path $sdkCacheFileTemp - } + # Creates a temporary file under the toolset dir. + # The following code block is protecting against concurrent access so that this function can + # be called in parallel. + if ($createSdkLocationFile) { + do { + $sdkCacheFileTemp = Join-Path $ToolsetDir $([System.IO.Path]::GetRandomFileName()) + } + until (!(Test-Path $sdkCacheFileTemp)) + Set-Content -Path $sdkCacheFileTemp -Value $dotnetRoot + + try { + Rename-Item -Force -Path $sdkCacheFileTemp 'sdk.txt' + } catch { + # Somebody beat us + Remove-Item -Path $sdkCacheFileTemp } } diff --git a/global.json b/global.json index bc3abbd6..aed2268c 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19615.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19616.4" } } From 4674cae73d132959d169ce47b94bb8db3726143c Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Wed, 18 Dec 2019 13:30:26 +0000 Subject: [PATCH 028/161] Update dependencies from https://github.com/dotnet/arcade build 20191217.1 (#541) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19617.1 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 5ad9b6c1..5eb17948 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 4736ddca60a4040447b450437cf3767c9045edc0 + 66175ebd3756697a3ca515e16cd5ffddc30582cd diff --git a/global.json b/global.json index aed2268c..5f40d78d 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19616.4" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19617.1" } } From bfbb4c9887823599e597e4c0075896f7cdf05d24 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 20 Dec 2019 13:50:43 +0000 Subject: [PATCH 029/161] Update dependencies from https://github.com/dotnet/arcade build 20191219.10 (#542) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19619.10 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 5eb17948..7a8186f6 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 66175ebd3756697a3ca515e16cd5ffddc30582cd + 154d5981c6d2fe19f80e71a654ceddeba76ba7b5 diff --git a/global.json b/global.json index 5f40d78d..79a6f192 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19617.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19619.10" } } From 32bbcb5051da09e385af74fde80f6ddfdcf3da46 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Sat, 21 Dec 2019 13:29:10 +0000 Subject: [PATCH 030/161] Update dependencies from https://github.com/dotnet/arcade build 20191220.1 (#543) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19620.1 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 7a8186f6..16e21f81 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 154d5981c6d2fe19f80e71a654ceddeba76ba7b5 + d0fecb826dee1a24c0d2c9136476a79bed22ed41 diff --git a/global.json b/global.json index 79a6f192..dbdf7b81 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19619.10" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19620.1" } } From 6dced20c19ee1a470e27a0a2619d9a6f2cc25bef Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 23 Dec 2019 13:10:54 +0000 Subject: [PATCH 031/161] Update dependencies from https://github.com/dotnet/arcade build 20191222.1 (#545) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19622.1 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 16e21f81..eba36b03 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - d0fecb826dee1a24c0d2c9136476a79bed22ed41 + 163d551438aa5a1fdae12b8405651f894e841211 diff --git a/global.json b/global.json index dbdf7b81..3e187f59 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19620.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19622.1" } } From 9280567bfe01368f57df64353dd5aeeb6c1961f9 Mon Sep 17 00:00:00 2001 From: Tomas Matousek Date: Sat, 30 Nov 2019 10:49:49 -0800 Subject: [PATCH 032/161] Add tests validating bitbucket url with personal token --- .../GetSourceLinkUrlTests.cs | 21 +++++++ .../BitbucketGitTests.cs | 56 +++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/src/SourceLink.Bitbucket.Git.UnitTests/GetSourceLinkUrlTests.cs b/src/SourceLink.Bitbucket.Git.UnitTests/GetSourceLinkUrlTests.cs index 363a8db0..1d7f45e9 100644 --- a/src/SourceLink.Bitbucket.Git.UnitTests/GetSourceLinkUrlTests.cs +++ b/src/SourceLink.Bitbucket.Git.UnitTests/GetSourceLinkUrlTests.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using Microsoft.Build.Evaluation; using Microsoft.Build.Tasks.SourceControl; using TestUtilities; using Xunit; @@ -79,6 +80,26 @@ public void BuildSourceLinkUrl_BitbucketCloud(string s1, string s2) Assert.True(result); } + [Fact] + public void BuildSourceLinkUrl_BitbucketEnterprise_PersonalToken() + { + var engine = new MockEngine(); + var task = new GetSourceLinkUrl() + { + BuildEngine = engine, + SourceRoot = new MockItem("/src/", KVP("RepositoryUrl", ProjectCollection.Escape("https://user_name%40domain.com:Bitbucket_personaltoken@bitbucket.domain.tools/scm/abc/project1.git")), KVP("SourceControl", "git"), KVP("RevisionId", "0123456789abcdefABCDEF000000000000000000")), + Hosts = new[] + { + new MockItem("bitbucket.domain.tools", KVP("ContentUrl", "https://bitbucket.domain.tools")), + } + }; + + bool result = task.Execute(); + AssertEx.AssertEqualToleratingWhitespaceDifferences("", engine.Log); + AssertEx.AreEqual("https://bitbucket.domain.tools/projects/abc/repos/project1/raw/*?at=0123456789abcdefABCDEF000000000000000000", task.SourceLinkUrl); + Assert.True(result); + } + [Fact] public void BuildSourceLinkUrl_BitbucketEnterprise_InvalidUrl() { diff --git a/src/SourceLink.Git.IntegrationTests/BitbucketGitTests.cs b/src/SourceLink.Git.IntegrationTests/BitbucketGitTests.cs index e67410c3..236499ad 100644 --- a/src/SourceLink.Git.IntegrationTests/BitbucketGitTests.cs +++ b/src/SourceLink.Git.IntegrationTests/BitbucketGitTests.cs @@ -123,6 +123,62 @@ public void FullValidation_EnterpriseNewHttps() url: $"https://bitbucket.domain.com/scm/test-org/{repoName}"); } + [ConditionalFact(typeof(DotNetSdkAvailable))] + public void FullValidation_EnterpriseNewHttps_UrlWithPersonalToken() + { + var repoUrl = "https://user_name%40domain.com:Bitbucket_personaltoken@bitbucket.domain.com/scm/test-org/project1.git"; + var repoName = "project1"; + + var repo = GitUtilities.CreateGitRepositoryWithSingleCommit(ProjectDir.Path, new[] { ProjectFileName }, + repoUrl); + var commitSha = repo.Head.Tip.Sha; + + VerifyValues( + customProps: @" + + true + + + + +", + customTargets: "", + targets: new[] + { + "Build", "Pack" + }, + expressions: new[] + { + "@(SourceRoot)", + "@(SourceRoot->'%(SourceLinkUrl)')", + "$(SourceLink)", + "$(PrivateRepositoryUrl)", + "$(RepositoryUrl)" + }, + expectedResults: new[] + { + ProjectSourceRoot, + $"https://bitbucket.domain.com/projects/test-org/repos/{repoName}/raw/*?at={commitSha}", + s_relativeSourceLinkJsonPath, + $"https://bitbucket.domain.com/scm/test-org/{repoName}.git", + $"https://bitbucket.domain.com/scm/test-org/{repoName}.git" + }); + + AssertEx.AreEqual( + $@"{{""documents"":{{""{ProjectSourceRoot.Replace(@"\", @"\\")}*"":""https://bitbucket.domain.com/projects/test-org/repos/{repoName}/raw/*?at={commitSha}""}}}}", + File.ReadAllText(Path.Combine(ProjectDir.Path, s_relativeSourceLinkJsonPath))); + + TestUtilities.ValidateAssemblyInformationalVersion( + Path.Combine(ProjectDir.Path, s_relativeOutputFilePath), + "1.0.0+" + commitSha); + + TestUtilities.ValidateNuSpecRepository( + Path.Combine(ProjectDir.Path, s_relativePackagePath), + type: "git", + commit: commitSha, + url: $"https://bitbucket.domain.com/scm/test-org/{repoName}.git"); + } + [ConditionalFact(typeof(DotNetSdkAvailable))] public void FullValidation_EnterpriseNewHttpsWithDefaultFlags() { From 12461acdbf9c7206e05f032a37ec5c0456a46de9 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 26 Dec 2019 17:22:56 +0000 Subject: [PATCH 033/161] [master] Update dependencies from dotnet/arcade (#544) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update dependencies from https://github.com/dotnet/arcade build 20191221.1 - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19621.1 * Update dependencies from https://github.com/dotnet/arcade build 20191223.1 - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19623.1 * Update dependencies from https://github.com/dotnet/arcade build 20191224.1 - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19624.1 * Update dependencies from https://github.com/dotnet/arcade build 20191225.1 - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19625.1 Co-authored-by: Tomáš Matoušek --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index eba36b03..51cce8a6 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 163d551438aa5a1fdae12b8405651f894e841211 + 4808b9983dea4d99e40df6215233ea78d396df6f diff --git a/global.json b/global.json index 3e187f59..35ffe51d 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19622.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19625.1" } } From 89d7370b1d9245abf0b2142f1c73f706280f07d4 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 27 Dec 2019 13:35:23 +0000 Subject: [PATCH 034/161] Update dependencies from https://github.com/dotnet/arcade build 20191226.1 (#547) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19626.1 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 51cce8a6..fa6b39c5 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 4808b9983dea4d99e40df6215233ea78d396df6f + ade7c8cb46ba88f5e698c91da9e7605619e34b6c diff --git a/global.json b/global.json index 35ffe51d..69a6665c 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19625.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19626.1" } } From d338eb3300a955a100c17a1c84eafd35048bf9c8 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Sat, 28 Dec 2019 13:31:09 +0000 Subject: [PATCH 035/161] Update dependencies from https://github.com/dotnet/arcade build 20191227.1 (#548) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19627.1 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index fa6b39c5..ce181f02 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - ade7c8cb46ba88f5e698c91da9e7605619e34b6c + d2d025c1de37b1258f147851cb3e7373ad5ff09d diff --git a/global.json b/global.json index 69a6665c..5eaf9c28 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19626.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19627.1" } } From 7df76a396f7400004c03569cdab5b0ff78e107e9 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Sun, 29 Dec 2019 13:29:25 +0000 Subject: [PATCH 036/161] Update dependencies from https://github.com/dotnet/arcade build 20191228.1 (#549) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19628.1 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index ce181f02..818b0326 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - d2d025c1de37b1258f147851cb3e7373ad5ff09d + 44a53c66de431dbd54b4277d6338d2b103d6852d diff --git a/global.json b/global.json index 5eaf9c28..97e9ff5b 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19627.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19628.1" } } From 7c867a94fa48036d6349bc705fb84f7f4e89b1fd Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 30 Dec 2019 13:10:56 +0000 Subject: [PATCH 037/161] Update dependencies from https://github.com/dotnet/arcade build 20191229.1 (#550) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19629.1 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 818b0326..22c2b3ce 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 44a53c66de431dbd54b4277d6338d2b103d6852d + f40849c5bbdfd1f1b99801f5a40343f8d4a192c9 diff --git a/global.json b/global.json index 97e9ff5b..7509420c 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19628.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19629.1" } } From 2e989592d05ca1336e19340e2c998fda3073cb2c Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Tue, 31 Dec 2019 13:26:58 +0000 Subject: [PATCH 038/161] Update dependencies from https://github.com/dotnet/arcade build 20191230.1 (#551) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19630.1 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 22c2b3ce..074a61a9 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - f40849c5bbdfd1f1b99801f5a40343f8d4a192c9 + 01d0b636bfb96edf62a77912de25aa5cef143946 diff --git a/global.json b/global.json index 7509420c..5aa88ae5 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19629.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19630.1" } } From b947f5ffb7386bc18083ad6b66b1a49bea4ec958 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Wed, 1 Jan 2020 13:26:45 +0000 Subject: [PATCH 039/161] Update dependencies from https://github.com/dotnet/arcade build 20191231.1 (#552) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.19631.1 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 074a61a9..b38d39fa 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 01d0b636bfb96edf62a77912de25aa5cef143946 + cc596c55ac68c952d0e052e6ed50334ed170b53a diff --git a/global.json b/global.json index 5aa88ae5..341cce90 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19630.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19631.1" } } From 00447de25a5e3eeda38db9b5bd0c95026b15cbba Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 2 Jan 2020 13:30:01 +0000 Subject: [PATCH 040/161] Update dependencies from https://github.com/dotnet/arcade build 20200101.1 (#553) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.20051.1 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index b38d39fa..8691a8da 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - cc596c55ac68c952d0e052e6ed50334ed170b53a + bbd4a95d4bcb6e06f88a8590e18e499a6169b66e diff --git a/global.json b/global.json index 341cce90..b43cc3f6 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.19631.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20051.1" } } From 00cc94e51c71d33763dd02b86951d8acbd7f42d9 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 3 Jan 2020 13:29:41 +0000 Subject: [PATCH 041/161] Update dependencies from https://github.com/dotnet/arcade build 20200102.1 (#554) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.20052.1 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 8691a8da..d87b36dd 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - bbd4a95d4bcb6e06f88a8590e18e499a6169b66e + 98cc270f5e08096729fc7d54ffafa9e3c6576316 diff --git a/global.json b/global.json index b43cc3f6..e34e2f18 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20051.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20052.1" } } From 006f20365226e8ea1c91ca4a2d21a8bab1835964 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Sat, 4 Jan 2020 13:37:06 +0000 Subject: [PATCH 042/161] Update dependencies from https://github.com/dotnet/arcade build 20200103.2 (#555) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.20053.2 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index d87b36dd..b8e9e828 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 98cc270f5e08096729fc7d54ffafa9e3c6576316 + b49ab5e13eb7c5678d6720ef041dc675cf8e9307 diff --git a/global.json b/global.json index e34e2f18..9da5afac 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20052.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20053.2" } } From d4d307413a34a55faae960f33e2fe3cebca6cce4 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Sun, 5 Jan 2020 13:24:19 +0000 Subject: [PATCH 043/161] Update dependencies from https://github.com/dotnet/arcade build 20200104.1 (#556) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.20054.1 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index b8e9e828..56e60061 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - b49ab5e13eb7c5678d6720ef041dc675cf8e9307 + 96ab522b32ff4d0f7bfc18074fd5ae3673c59d0a diff --git a/global.json b/global.json index 9da5afac..6f425671 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20053.2" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20054.1" } } From 539306134f9d4f7259e282945bf123ed48f08745 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 6 Jan 2020 13:35:07 +0000 Subject: [PATCH 044/161] Update dependencies from https://github.com/dotnet/arcade build 20200105.1 (#558) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.20055.1 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 56e60061..9fd9fffc 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 96ab522b32ff4d0f7bfc18074fd5ae3673c59d0a + 111d78c79ac38f596688af152779d7a39e4bf973 diff --git a/global.json b/global.json index 6f425671..82f48ac2 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20054.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20055.1" } } From 671d6895094a8ea5118c9120f48079f368c639f0 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 6 Jan 2020 13:35:44 +0000 Subject: [PATCH 045/161] Update dependencies from https://github.com/dotnet/arcade build 20200105.1 (#557) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.20055.1 From 8c41cd0788a7a61303f1b7f9e1aef5fd3a0ee937 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Tue, 7 Jan 2020 13:38:41 +0000 Subject: [PATCH 046/161] Update dependencies from https://github.com/dotnet/arcade build 20200106.6 (#559) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.20056.6 --- eng/Version.Details.xml | 4 +- .../channels/generic-internal-channel.yml | 38 +++++++----------- .../channels/generic-public-channel.yml | 39 +++++++------------ global.json | 2 +- 4 files changed, 33 insertions(+), 50 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 9fd9fffc..cfdceb16 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 111d78c79ac38f596688af152779d7a39e4bf973 + 5cbca575e9002cc2d9814c3471d38882df2fcc8e diff --git a/eng/common/templates/post-build/channels/generic-internal-channel.yml b/eng/common/templates/post-build/channels/generic-internal-channel.yml index 3a8755fb..70021104 100644 --- a/eng/common/templates/post-build/channels/generic-internal-channel.yml +++ b/eng/common/templates/post-build/channels/generic-internal-channel.yml @@ -30,16 +30,15 @@ stages: vmImage: 'windows-2019' steps: - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - artifactName: 'BlobArtifacts' + displayName: Download Build Assets continueOnError: true - - - task: DownloadBuildArtifacts@0 - displayName: Download PDB Artifacts inputs: - artifactName: 'PDBArtifacts' - continueOnError: true + buildType: 'current' + downloadType: 'specific' + itemPattern: | + PDBArtifacts/** + BlobArtifacts/** + downloadPath: '$(Build.ArtifactStagingDirectory)' # This is necessary whenever we want to publish/restore to an AzDO private feed # Since sdk-task.ps1 tries to restore packages we need to do this authentication here @@ -85,23 +84,16 @@ stages: vmImage: 'windows-2019' steps: - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: current - artifactName: PackageArtifacts - continueOnError: true - - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: current - artifactName: BlobArtifacts + displayName: Download Build Assets continueOnError: true - - task: DownloadBuildArtifacts@0 - displayName: Download Asset Manifests inputs: - buildType: current - artifactName: AssetManifests + buildType: 'current' + downloadType: 'specific' + itemPattern: | + PackageArtifacts/** + BlobArtifacts/** + AssetManifests/** + downloadPath: '$(Build.ArtifactStagingDirectory)' - task: NuGetToolInstaller@1 displayName: 'Install NuGet.exe' diff --git a/eng/common/templates/post-build/channels/generic-public-channel.yml b/eng/common/templates/post-build/channels/generic-public-channel.yml index 3f572f8b..fbb5a19b 100644 --- a/eng/common/templates/post-build/channels/generic-public-channel.yml +++ b/eng/common/templates/post-build/channels/generic-public-channel.yml @@ -30,16 +30,15 @@ stages: vmImage: 'windows-2019' steps: - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - artifactName: 'BlobArtifacts' + displayName: Download Build Assets continueOnError: true - - - task: DownloadBuildArtifacts@0 - displayName: Download PDB Artifacts inputs: - artifactName: 'PDBArtifacts' - continueOnError: true + buildType: 'current' + downloadType: 'specific' + itemPattern: | + PDBArtifacts/** + BlobArtifacts/** + downloadPath: '$(Build.ArtifactStagingDirectory)' # This is necessary whenever we want to publish/restore to an AzDO private feed # Since sdk-task.ps1 tries to restore packages we need to do this authentication here @@ -84,24 +83,16 @@ stages: vmImage: 'windows-2019' steps: - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: current - artifactName: PackageArtifacts + displayName: Download Build Assets continueOnError: true - - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: current - artifactName: BlobArtifacts - continueOnError: true - - - task: DownloadBuildArtifacts@0 - displayName: Download Asset Manifests inputs: - buildType: current - artifactName: AssetManifests + buildType: 'current' + downloadType: 'specific' + itemPattern: | + PackageArtifacts/** + BlobArtifacts/** + AssetManifests/** + downloadPath: '$(Build.ArtifactStagingDirectory)' - task: NuGetToolInstaller@1 displayName: 'Install NuGet.exe' diff --git a/global.json b/global.json index 82f48ac2..42219747 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20055.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20056.6" } } From 225b098449703e9362d8ece018c3e5bafbfaddd9 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Wed, 8 Jan 2020 13:28:07 +0000 Subject: [PATCH 047/161] Update dependencies from https://github.com/dotnet/arcade build 20200107.5 (#560) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.20057.5 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index cfdceb16..c98fbfbe 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 5cbca575e9002cc2d9814c3471d38882df2fcc8e + d0833c8e5e58cfc507ce3c8da364e55931190263 diff --git a/global.json b/global.json index 42219747..8c12a1ea 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20056.6" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20057.5" } } From 51e3bee8f079599971b03b9430288a690213b85c Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 9 Jan 2020 13:44:48 +0000 Subject: [PATCH 048/161] Update dependencies from https://github.com/dotnet/arcade build 20200108.1 (#562) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.20058.1 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index c98fbfbe..a5b85ce1 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - d0833c8e5e58cfc507ce3c8da364e55931190263 + 48bae09d7d8ec0e6879fe2b8d08872cd87f084ca diff --git a/global.json b/global.json index 8c12a1ea..012813dd 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20057.5" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20058.1" } } From 80eb691a644709e186534a8f9774ffd1c2bb315a Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 10 Jan 2020 13:25:51 +0000 Subject: [PATCH 049/161] Update dependencies from https://github.com/dotnet/arcade build 20200109.3 (#564) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.20059.3 --- eng/Version.Details.xml | 4 ++-- eng/common/performance/performance-setup.ps1 | 3 ++- eng/common/performance/performance-setup.sh | 2 +- global.json | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index a5b85ce1..7e1dd07a 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 48bae09d7d8ec0e6879fe2b8d08872cd87f084ca + 96a4497b2c38c6b7e19c39e4cb8fe5adac299fb4 diff --git a/eng/common/performance/performance-setup.ps1 b/eng/common/performance/performance-setup.ps1 index ec41965f..f6e87829 100644 --- a/eng/common/performance/performance-setup.ps1 +++ b/eng/common/performance/performance-setup.ps1 @@ -49,7 +49,8 @@ if ($Internal) { $HelixSourcePrefix = "official" } -$CommonSetupArguments="--frameworks $Framework --queue $Queue --build-number $BuildNumber --build-configs $Configurations" +# FIX ME: This is a workaround until we get this from the actual pipeline +$CommonSetupArguments="--channel master --queue $Queue --build-number $BuildNumber --build-configs $Configurations" $SetupArguments = "--repository https://github.com/$Repository --branch $Branch --get-perf-hash --commit-sha $CommitSha $CommonSetupArguments" if ($RunFromPerformanceRepo) { diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index 2f209216..7ef15514 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -164,7 +164,7 @@ if [[ "$internal" == true ]]; then fi fi -common_setup_arguments="--frameworks $framework --queue $queue --build-number $build_number --build-configs $configurations" +common_setup_arguments="--channel master --queue $queue --build-number $build_number --build-configs $configurations" setup_arguments="--repository https://github.com/$repository --branch $branch --get-perf-hash --commit-sha $commit_sha $common_setup_arguments" if [[ "$run_from_perf_repo" = true ]]; then diff --git a/global.json b/global.json index 012813dd..6da4ee20 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20058.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20059.3" } } From d8741a2551b8dcee996e2bcf74a0eec3d9e79740 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 13 Jan 2020 14:31:36 +0000 Subject: [PATCH 050/161] Update dependencies from https://github.com/dotnet/arcade build 20200112.1 (#565) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.20062.1 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 7e1dd07a..0172a222 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 96a4497b2c38c6b7e19c39e4cb8fe5adac299fb4 + 250782df013aa8586e21bb225f8efd715372dcf1 diff --git a/global.json b/global.json index 6da4ee20..f73caf35 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20059.3" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20062.1" } } From 058eaf8c12116a4dc64c32dea3271b76689a150b Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 20 Jan 2020 13:11:26 +0000 Subject: [PATCH 051/161] Update dependencies from https://github.com/dotnet/arcade build 20200118.1 (#568) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.20068.1 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 0172a222..36084ef8 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 250782df013aa8586e21bb225f8efd715372dcf1 + ebc71292545adda514c1e6a5657cd9ab753240bc diff --git a/global.json b/global.json index f73caf35..51eb1928 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20062.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20068.1" } } From f61f34798be0641e4bbfa3a50174c8ee1f4f7ee5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Matou=C5=A1ek?= Date: Tue, 21 Jan 2020 10:37:04 -0700 Subject: [PATCH 052/161] Nullable annotations (#546) --- SourceLink.sln | 1 + global.json | 2 +- src/Common/AzureDevOpsUrlParser.cs | 21 +- src/Common/GetSourceLinkUrlGitTask.cs | 39 ++-- src/Common/NullableAttributes.cs | 110 ++++++++++ src/Common/TranslateRepositoryUrlGitTask.cs | 20 +- src/Common/UriUtilities.cs | 3 +- src/Directory.Build.props | 1 + .../Microsoft.Build.StandardCI.csproj | 4 + .../GitConfigTests.cs | 4 +- .../GitDataTests.cs | 6 +- .../GitOperationsTests.cs | 188 +++++++++++------- .../GitRepositoryTests.cs | 4 +- .../GlobTests.cs | 2 +- .../TestUtilities.cs | 5 +- .../AssemblyResolver.cs | 3 +- .../GetUntrackedFiles.cs | 18 +- .../GitDataReader/GitConfig.Reader.cs | 9 +- .../GitDataReader/GitConfig.cs | 15 +- .../GitDataReader/GitEnvironment.cs | 32 +-- .../GitDataReader/GitIgnore.Matcher.cs | 14 +- .../GitDataReader/GitIgnore.cs | 21 +- .../GitDataReader/GitReferenceResolver.cs | 12 +- .../GitDataReader/GitRepository.cs | 76 +++---- .../GitDataReader/GitRepositoryLocation.cs | 12 +- .../GitDataReader/GitSubmodule.cs | 15 +- .../GitDataReader/GitVariableName.cs | 6 +- .../GitOperations.cs | 141 ++++++------- .../LocateRepository.cs | 24 ++- .../Microsoft.Build.Tasks.Git.csproj | 1 + .../RepositoryTask.cs | 19 +- .../GetRepositoryUrl.cs | 19 +- .../GetSourceRevisionId.cs | 11 +- .../GetSourceRoots.cs | 55 +++-- .../GetUntrackedSourceFiles.cs | 35 ++-- .../LocateRepository.cs | 7 +- .../Microsoft.Build.Tasks.Tfvc.csproj | 1 + .../RepositoryTask.cs | 5 +- .../GetSourceLinkUrl.cs | 4 +- ...ft.SourceLink.AzureDevOpsServer.Git.csproj | 1 + .../TranslateRepositoryUrls.cs | 2 +- .../GetSourceLinkUrl.cs | 6 +- ...Microsoft.SourceLink.AzureRepos.Git.csproj | 1 + .../TranslateRepositoryUrls.cs | 2 +- .../GetSourceLinkUrl.cs | 7 +- ...icrosoft.SourceLink.AzureRepos.Tfvc.csproj | 3 + .../GetSourceLinkUrl.cs | 28 ++- .../Microsoft.SourceLink.Bitbucket.Git.csproj | 1 + .../Mocks/MockGetSourceLinkUrlGitTask.cs | 2 +- .../UriUtilitiesTests.cs | 4 +- .../GenerateSourceLinkFile.cs | 17 +- .../Microsoft.SourceLink.Common.csproj | 1 + .../SourceLinkHasSingleProvider.cs | 2 +- src/SourceLink.GitHub/GetSourceLinkUrl.cs | 2 +- .../Microsoft.SourceLink.GitHub.csproj | 1 + src/SourceLink.GitLab/GetSourceLinkUrl.cs | 2 +- .../Microsoft.SourceLink.GitLab.csproj | 1 + src/TestUtilities/AssertEx.cs | 83 ++++---- src/TestUtilities/DiffUtil.cs | 4 +- .../DotNetSdk/DotNetSdkTestBase.cs | 28 +-- src/TestUtilities/ProcessUtilities.cs | 12 +- src/TestUtilities/TempFiles/DisposableFile.cs | 2 +- src/TestUtilities/TempFiles/TempDirectory.cs | 6 +- src/TestUtilities/TempFiles/TempFile.cs | 2 +- src/TestUtilities/TempFiles/TempRoot.cs | 2 +- src/TestUtilities/TestUtilities.csproj | 4 + 66 files changed, 673 insertions(+), 518 deletions(-) create mode 100644 src/Common/NullableAttributes.cs diff --git a/SourceLink.sln b/SourceLink.sln index 61cf926b..49098d14 100644 --- a/SourceLink.sln +++ b/SourceLink.sln @@ -38,6 +38,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Common", "Common", "{762115 src\Common\CommonResources.resx = src\Common\CommonResources.resx src\Common\GetSourceLinkUrlGitTask.cs = src\Common\GetSourceLinkUrlGitTask.cs src\Common\Names.cs = src\Common\Names.cs + src\Common\NullableAttributes.cs = src\Common\NullableAttributes.cs src\Common\PathUtilities.cs = src\Common\PathUtilities.cs src\Common\SequenceComparer.cs = src\Common\SequenceComparer.cs src\Common\TeamFoundationUrlParser.cs = src\Common\TeamFoundationUrlParser.cs diff --git a/global.json b/global.json index 51eb1928..0d065575 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "tools": { - "dotnet": "3.0.101" + "dotnet": "3.1.100" }, "msbuild-sdks": { "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20068.1" diff --git a/src/Common/AzureDevOpsUrlParser.cs b/src/Common/AzureDevOpsUrlParser.cs index a287fe8f..4f212f5e 100644 --- a/src/Common/AzureDevOpsUrlParser.cs +++ b/src/Common/AzureDevOpsUrlParser.cs @@ -1,8 +1,9 @@ // Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; -using System.Linq; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Linq; using Microsoft.Build.Tasks.SourceControl; namespace Microsoft.SourceLink @@ -17,7 +18,7 @@ public static bool IsVisualStudioHostedServer(string host) => host.EndsWith(".visualstudio.com", StringComparison.OrdinalIgnoreCase) || host.EndsWith(".vsts.me", StringComparison.OrdinalIgnoreCase); - public static bool TryParseHostedHttp(string host, string relativeUrl, out string projectPath, out string repositoryName) + public static bool TryParseHostedHttp(string host, string relativeUrl, [NotNullWhen(true)]out string? projectPath, [NotNullWhen(true)]out string? repositoryName) { projectPath = repositoryName = null; @@ -27,7 +28,7 @@ public static bool TryParseHostedHttp(string host, string relativeUrl, out strin } int index = 0; - string account; + string? account; bool isVisualStudioHost = IsVisualStudioHostedServer(host); if (isVisualStudioHost) @@ -78,7 +79,7 @@ public static bool TryParseHostedHttp(string host, string relativeUrl, out strin return true; } - public static bool TryParseOnPremHttp(string relativeUrl, string virtualDirectory, out string projectPath, out string repositoryName) + public static bool TryParseOnPremHttp(string relativeUrl, string virtualDirectory, [NotNullWhen(true)]out string? projectPath, [NotNullWhen(true)]out string? repositoryName) { projectPath = repositoryName = null; @@ -107,9 +108,9 @@ public static bool TryParseOnPremHttp(string relativeUrl, string virtualDirector return true; } - public static bool TryParseHostedSsh(Uri uri, out string account, out string repositoryPath, out string repositoryName) + public static bool TryParseHostedSsh(Uri uri, [NotNullWhen(true)]out string? account, [NotNullWhen(true)]out string? repositoryPath, [NotNullWhen(true)]out string? repositoryName) { - Debug.Assert(uri != null); + NullableDebug.Assert(uri != null); account = repositoryPath = repositoryName = null; @@ -156,7 +157,7 @@ public static bool TryParseHostedSsh(Uri uri, out string account, out string rep return true; } - public static bool TryParseOnPremSsh(Uri uri, out string repositoryPath, out string repositoryName) + public static bool TryParseOnPremSsh(Uri uri, [NotNullWhen(true)]out string? repositoryPath, [NotNullWhen(true)]out string? repositoryName) { repositoryPath = repositoryName = null; @@ -174,7 +175,7 @@ public static bool TryParseOnPremSsh(Uri uri, out string repositoryPath, out str return true; } - private static bool TryParsePath(string[] parts, int startIndex, string type, out string repositoryPath, out string repositoryName) + private static bool TryParsePath(string[] parts, int startIndex, string? type, [NotNullWhen(true)]out string? repositoryPath, [NotNullWhen(true)]out string? repositoryName) { if (TryParsePath(parts, startIndex, type, out var projectName, out var teamName, out repositoryName)) { @@ -186,7 +187,7 @@ private static bool TryParsePath(string[] parts, int startIndex, string type, ou return false; } - private static bool TryParsePath(string[] parts, int projectPartIndex, string type, out string projectName, out string teamName, out string repositoryName) + private static bool TryParsePath(string[] parts, int projectPartIndex, string? type, out string? projectName, out string? teamName, [NotNullWhen(true)]out string? repositoryName) { // {projectName?}/{teamName?}/{type}/{"_full"|"_optimized"|""}/{repositoryName} @@ -216,7 +217,7 @@ private static bool TryParsePath(string[] parts, int projectPartIndex, string ty } } - private static bool TryParseRepositoryName(string[] parts, out int teamNameIndex, string type, out string repositoryName) + private static bool TryParseRepositoryName(string[] parts, out int teamNameIndex, string? type, [NotNullWhen(true)]out string? repositoryName) { Debug.Assert(type == "_git" || type == "_ssh" || type == null); diff --git a/src/Common/GetSourceLinkUrlGitTask.cs b/src/Common/GetSourceLinkUrlGitTask.cs index 72a9b5f4..b49cac57 100644 --- a/src/Common/GetSourceLinkUrlGitTask.cs +++ b/src/Common/GetSourceLinkUrlGitTask.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.Linq; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; @@ -15,22 +16,22 @@ public abstract class GetSourceLinkUrlGitTask : Task protected const string NotApplicableValue = "N/A"; private const string ContentUrlMetadataName = "ContentUrl"; - [Required] - public ITaskItem SourceRoot { get; set; } + [Required, NotNull] + public ITaskItem? SourceRoot { get; set; } /// /// List of additional repository hosts for which the task produces SourceLink URLs. /// Each item maps a domain of a repository host (stored in the item identity) to a URL of the server that provides source file content (stored in ContentUrl metadata). /// ContentUrl is optional. /// - public ITaskItem[] Hosts { get; set; } + public ITaskItem[]? Hosts { get; set; } - public string RepositoryUrl { get; set; } + public string? RepositoryUrl { get; set; } public bool IsSingleProvider { get; set; } [Output] - public string SourceLinkUrl { get; set; } + public string? SourceLinkUrl { get; set; } internal GetSourceLinkUrlGitTask() { } @@ -52,7 +53,7 @@ protected virtual Uri GetDefaultContentUriFromHostUri(string authority, Uri gitU protected virtual Uri GetDefaultContentUriFromRepositoryUri(Uri repositoryUri) => GetDefaultContentUriFromHostUri(repositoryUri.GetAuthority(), repositoryUri); - protected abstract string BuildSourceLinkUrl(Uri contentUrl, Uri gitUri, string relativeUrl, string revisionId, ITaskItem hostItem); + protected abstract string? BuildSourceLinkUrl(Uri contentUrl, Uri gitUri, string relativeUrl, string revisionId, ITaskItem? hostItem); public override bool Execute() { @@ -96,18 +97,17 @@ private void ExecuteImpl() return; } - var contentUri = GetMatchingContentUri(mappings, gitUri, out var hostItem); - if (contentUri == null) + if (!TryGetMatchingContentUri(mappings, gitUri, out var contentUri, out var hostItem)) { SourceLinkUrl = NotApplicableValue; return; } - static bool IsHexDigit(char c) + static bool isHexDigit(char c) => c >= '0' && c <= '9' || c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F'; string revisionId = SourceRoot.GetMetadata(Names.SourceRoot.RevisionId); - if (revisionId == null || revisionId.Length != 40 || !revisionId.All(IsHexDigit)) + if (revisionId == null || revisionId.Length != 40 || !revisionId.All(isHexDigit)) { Log.LogError(CommonResources.ValueOfWithIdentityIsNotValidCommitHash, Names.SourceRoot.RevisionIdFullName, SourceRoot.ItemSpec, revisionId); return; @@ -128,16 +128,16 @@ static bool IsHexDigit(char c) private readonly struct UrlMapping { public readonly string Host; - public readonly ITaskItem HostItem; + public readonly ITaskItem? HostItem; public readonly int Port; public readonly Uri ContentUri; public readonly bool HasDefaultContentUri; - public UrlMapping(string host, ITaskItem hostItem, int port, Uri contentUri, bool hasDefaultContentUri) + public UrlMapping(string host, ITaskItem? hostItem, int port, Uri contentUri, bool hasDefaultContentUri) { - Debug.Assert(port >= -1); - Debug.Assert(!string.IsNullOrEmpty(host)); - Debug.Assert(contentUri != null); + NullableDebug.Assert(port >= -1); + NullableDebug.Assert(!string.IsNullOrEmpty(host)); + NullableDebug.Assert(contentUri != null); Host = host; Port = port; @@ -204,7 +204,7 @@ static bool isValidContentUri(Uri uri) } } - private static Uri GetMatchingContentUri(UrlMapping[] mappings, Uri repoUri, out ITaskItem hostItem) + private static bool TryGetMatchingContentUri(UrlMapping[] mappings, Uri repoUri, [NotNullWhen(true)]out Uri? contentUri, out ITaskItem? hostItem) { UrlMapping? findMatch(bool exactHost) { @@ -235,12 +235,13 @@ private static Uri GetMatchingContentUri(UrlMapping[] mappings, Uri repoUri, out var result = findMatch(exactHost: true) ?? findMatch(exactHost: false); if (result == null) { + contentUri = null; hostItem = null; - return null; + return false; } var value = result.Value; - var contentUri = value.ContentUri; + contentUri = value.ContentUri; hostItem = value.HostItem; // If the mapping did not specify ContentUrl and did not specify port, @@ -250,7 +251,7 @@ private static Uri GetMatchingContentUri(UrlMapping[] mappings, Uri repoUri, out contentUri = new Uri($"{contentUri.Scheme}://{contentUri.Host}:{repoUri.Port}{contentUri.PathAndQuery}"); } - return contentUri; + return true; } } } diff --git a/src/Common/NullableAttributes.cs b/src/Common/NullableAttributes.cs new file mode 100644 index 00000000..8db4ab49 --- /dev/null +++ b/src/Common/NullableAttributes.cs @@ -0,0 +1,110 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; + +namespace System.Diagnostics.CodeAnalysis +{ + /// Specifies that null is allowed as an input even if the corresponding type disallows it. + [AttributeUsage(AttributeTargets.Field | AttributeTargets.Parameter | AttributeTargets.Property, Inherited = false)] + internal sealed class AllowNullAttribute : Attribute { } + + /// Specifies that null is disallowed as an input even if the corresponding type allows it. + [AttributeUsage(AttributeTargets.Field | AttributeTargets.Parameter | AttributeTargets.Property, Inherited = false)] + internal sealed class DisallowNullAttribute : Attribute { } + + /// Specifies that an output may be null even if the corresponding type disallows it. + [AttributeUsage(AttributeTargets.Field | AttributeTargets.Parameter | AttributeTargets.Property | AttributeTargets.ReturnValue, Inherited = false)] + internal sealed class MaybeNullAttribute : Attribute { } + + /// Specifies that an output will not be null even if the corresponding type allows it. + [AttributeUsage(AttributeTargets.Field | AttributeTargets.Parameter | AttributeTargets.Property | AttributeTargets.ReturnValue, Inherited = false)] + internal sealed class NotNullAttribute : Attribute { } + + /// Specifies that when a method returns , the parameter may be null even if the corresponding type disallows it. + [AttributeUsage(AttributeTargets.Parameter, Inherited = false)] + internal sealed class MaybeNullWhenAttribute : Attribute + { + /// Initializes the attribute with the specified return value condition. + /// + /// The return value condition. If the method returns this value, the associated parameter may be null. + /// + public MaybeNullWhenAttribute(bool returnValue) => ReturnValue = returnValue; + + /// Gets the return value condition. + public bool ReturnValue { get; } + } + + /// Specifies that when a method returns , the parameter will not be null even if the corresponding type allows it. + [AttributeUsage(AttributeTargets.Parameter, Inherited = false)] + internal sealed class NotNullWhenAttribute : Attribute + { + /// Initializes the attribute with the specified return value condition. + /// + /// The return value condition. If the method returns this value, the associated parameter will not be null. + /// + public NotNullWhenAttribute(bool returnValue) => ReturnValue = returnValue; + + /// Gets the return value condition. + public bool ReturnValue { get; } + } + + /// Specifies that the output will be non-null if the named parameter is non-null. + [AttributeUsage(AttributeTargets.Parameter | AttributeTargets.Property | AttributeTargets.ReturnValue, AllowMultiple = true, Inherited = false)] + internal sealed class NotNullIfNotNullAttribute : Attribute + { + /// Initializes the attribute with the associated parameter name. + /// + /// The associated parameter name. The output will be non-null if the argument to the parameter specified is non-null. + /// + public NotNullIfNotNullAttribute(string parameterName) => ParameterName = parameterName; + + /// Gets the associated parameter name. + public string ParameterName { get; } + } + + /// Applied to a method that will never return under any circumstance. + [AttributeUsage(AttributeTargets.Method, Inherited = false)] + internal sealed class DoesNotReturnAttribute : Attribute { } + + /// Specifies that the method will not return if the associated Boolean parameter is passed the specified value. + [AttributeUsage(AttributeTargets.Parameter, Inherited = false)] + internal sealed class DoesNotReturnIfAttribute : Attribute + { + /// Initializes the attribute with the specified parameter value. + /// + /// The condition parameter value. Code after the method will be considered unreachable by diagnostics if the argument to + /// the associated parameter matches this value. + /// + public DoesNotReturnIfAttribute(bool parameterValue) => ParameterValue = parameterValue; + + /// Gets the condition parameter value. + public bool ParameterValue { get; } + } +} + +namespace System +{ + internal static class NullableString + { + public static bool IsNullOrEmpty([NotNullWhen(false)]string? str) + => string.IsNullOrEmpty(str); + + public static bool IsNullOrWhiteSpace([NotNullWhen(false)]string? str) + => string.IsNullOrWhiteSpace(str); + } + + internal static class NullableDebug + { + /// + [Conditional("DEBUG")] + public static void Assert([DoesNotReturnIf(false)]bool b) + => Debug.Assert(b); + + /// + [Conditional("DEBUG")] + public static void Assert([DoesNotReturnIf(false)]bool b, string message) + => Debug.Assert(b, message); + + } +} \ No newline at end of file diff --git a/src/Common/TranslateRepositoryUrlGitTask.cs b/src/Common/TranslateRepositoryUrlGitTask.cs index 53ed0f69..b87b2d70 100644 --- a/src/Common/TranslateRepositoryUrlGitTask.cs +++ b/src/Common/TranslateRepositoryUrlGitTask.cs @@ -12,25 +12,25 @@ public class TranslateRepositoryUrlsGitTask : Task { private const string SourceControlName = "git"; - public string RepositoryUrl { get; set; } - public ITaskItem[] SourceRoots { get; set; } + public string? RepositoryUrl { get; set; } + public ITaskItem[]? SourceRoots { get; set; } - public ITaskItem[] Hosts { get; set; } + public ITaskItem[]? Hosts { get; set; } public bool IsSingleProvider { get; set; } [Output] - public string TranslatedRepositoryUrl { get; set; } + public string? TranslatedRepositoryUrl { get; set; } [Output] - public ITaskItem[] TranslatedSourceRoots { get; set; } + public ITaskItem[]? TranslatedSourceRoots { get; set; } - protected virtual string TranslateSshUrl(Uri uri) + protected virtual string? TranslateSshUrl(Uri uri) => "https://" + uri.GetHost() + uri.GetPathAndQuery(); - protected virtual string TranslateGitUrl(Uri uri) + protected virtual string? TranslateGitUrl(Uri uri) => "https://" + uri.GetHost() + uri.GetPathAndQuery(); - protected virtual string TranslateHttpUrl(Uri uri) + protected virtual string? TranslateHttpUrl(Uri uri) => uri.GetScheme() + "://" + uri.GetAuthority() + uri.GetPathAndQuery(); public override bool Execute() @@ -47,12 +47,12 @@ private void ExecuteImpl() return; } - bool isMatchingHostUri(Uri hostUri, Uri uri) + static bool isMatchingHostUri(Uri hostUri, Uri uri) => uri.GetHost().Equals(hostUri.GetHost(), StringComparison.OrdinalIgnoreCase) || uri.GetHost().EndsWith("." + hostUri.GetHost(), StringComparison.OrdinalIgnoreCase); // only need to translate valid ssh URLs that match one of our hosts: - string translate(string url) + string? translate(string? url) { if (Uri.TryCreate(url, UriKind.Absolute, out var uri) && hostUris.Any(h => isMatchingHostUri(h, uri))) diff --git a/src/Common/UriUtilities.cs b/src/Common/UriUtilities.cs index 9f5a72a9..6c6d6be0 100644 --- a/src/Common/UriUtilities.cs +++ b/src/Common/UriUtilities.cs @@ -2,6 +2,7 @@ using System; using System.Linq; +using System.Diagnostics.CodeAnalysis; namespace Microsoft.Build.Tasks.SourceControl { @@ -35,7 +36,7 @@ public static bool UrlStartsWith(string url, string prefix) return url.StartsWith(prefix, StringComparison.Ordinal); } - public static bool TrySplitRelativeUrl(string relativeUrl, out string[] parts) + public static bool TrySplitRelativeUrl(string relativeUrl, [NotNullWhen(true)]out string[]? parts) { // required leading slash: if (relativeUrl.Length == 0 || relativeUrl == "/") diff --git a/src/Directory.Build.props b/src/Directory.Build.props index a5c664a2..01bbff1b 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -4,6 +4,7 @@ Preview + enable Apache-2.0 true $(RepositoryEngineeringDir)runtimeconfig.template.json diff --git a/src/Microsoft.Build.StandardCI/Microsoft.Build.StandardCI.csproj b/src/Microsoft.Build.StandardCI/Microsoft.Build.StandardCI.csproj index a7842e4a..e04370c0 100644 --- a/src/Microsoft.Build.StandardCI/Microsoft.Build.StandardCI.csproj +++ b/src/Microsoft.Build.StandardCI/Microsoft.Build.StandardCI.csproj @@ -17,6 +17,10 @@ false + + + + diff --git a/src/Microsoft.Build.Tasks.Git.UnitTests/GitConfigTests.cs b/src/Microsoft.Build.Tasks.Git.UnitTests/GitConfigTests.cs index 7e3da0aa..6959d605 100644 --- a/src/Microsoft.Build.Tasks.Git.UnitTests/GitConfigTests.cs +++ b/src/Microsoft.Build.Tasks.Git.UnitTests/GitConfigTests.cs @@ -225,14 +225,14 @@ public void HierarchicalLoad(bool enableProgramData, bool enableXdg, string expe homeDir.CreateFile(".gitconfig").WriteAllText("[cfg]dir=home1"); homeDir.CreateDirectory(".config").CreateDirectory("git").CreateFile("config").WriteAllText("[cfg]dir=home2"); - TempDirectory xdgDir = null; + TempDirectory? xdgDir = null; if (enableXdg) { xdgDir = root.CreateDirectory("xdg"); xdgDir.CreateDirectory("git").CreateFile("config").WriteAllText("[cfg]dir=xdg"); } - TempDirectory programDataDir = null; + TempDirectory? programDataDir = null; if (enableProgramData) { programDataDir = root.CreateDirectory("programdata"); diff --git a/src/Microsoft.Build.Tasks.Git.UnitTests/GitDataTests.cs b/src/Microsoft.Build.Tasks.Git.UnitTests/GitDataTests.cs index e73c9199..d41ebdd0 100644 --- a/src/Microsoft.Build.Tasks.Git.UnitTests/GitDataTests.cs +++ b/src/Microsoft.Build.Tasks.Git.UnitTests/GitDataTests.cs @@ -41,12 +41,12 @@ public void MinimalGitData() gitDirSub.CreateDirectory("objects"); gitDirSub.CreateDirectory("refs"); - var repository = GitRepository.OpenRepository(repoDir.Path, GitEnvironment.Empty); + var repository = GitRepository.OpenRepository(repoDir.Path, GitEnvironment.Empty)!; Assert.Equal("http://github.com/test-org/test-repo", GitOperations.GetRepositoryUrl(repository, remoteName: null)); Assert.Equal("1111111111111111111111111111111111111111", repository.GetHeadCommitSha()); - var warnings = new List<(string, object[])>(); + var warnings = new List<(string, object?[])>(); var sourceRoots = GitOperations.GetSourceRoots(repository, remoteName: null, (message, args) => warnings.Add((message, args))); AssertEx.Equal(new[] { @@ -64,7 +64,7 @@ public void MinimalGitData() new MockItem(@"sub\ignore_in_submodule_d"), }; - var untrackedFiles = GitOperations.GetUntrackedFiles(repository, files, repoDir.Path, path => GitRepository.OpenRepository(path, GitEnvironment.Empty)); + var untrackedFiles = GitOperations.GetUntrackedFiles(repository, files, repoDir.Path); AssertEx.Equal(new[] { diff --git a/src/Microsoft.Build.Tasks.Git.UnitTests/GitOperationsTests.cs b/src/Microsoft.Build.Tasks.Git.UnitTests/GitOperationsTests.cs index cbba9cb0..eaef98e8 100644 --- a/src/Microsoft.Build.Tasks.Git.UnitTests/GitOperationsTests.cs +++ b/src/Microsoft.Build.Tasks.Git.UnitTests/GitOperationsTests.cs @@ -4,7 +4,6 @@ using System.Collections.Immutable; using System.IO; using System.Linq; -using Microsoft.Build.Framework; using TestUtilities; using Xunit; @@ -16,14 +15,14 @@ public class GitOperationsTests private static readonly char s = Path.DirectorySeparatorChar; private static readonly string s_root = (s == '/') ? "/usr/src" : @"C:\src"; - private string _workingDir = s_root; + private readonly string _workingDir = s_root; private GitRepository CreateRepository( - string workingDir = null, - GitConfig config = null, - string commitSha = null, + string? workingDir = null, + GitConfig? config = null, + string? commitSha = null, ImmutableArray submodules = default, - GitIgnore ignore = null) + GitIgnore? ignore = null) { workingDir ??= _workingDir; var gitDir = Path.Combine(workingDir, ".git"); @@ -39,7 +38,7 @@ private GitRepository CreateRepository( commitSha); } - private GitSubmodule CreateSubmodule(string name, string relativePath, string url, string headCommitSha, string containingRepositoryWorkingDir = null) + private GitSubmodule CreateSubmodule(string name, string relativePath, string url, string? headCommitSha, string? containingRepositoryWorkingDir = null) => new GitSubmodule(name, relativePath, Path.GetFullPath(Path.Combine(containingRepositoryWorkingDir ?? _workingDir, relativePath)), url, headCommitSha); internal static GitIgnore CreateIgnore(string workingDirectory, string[] filePathsRelativeToWorkingDirectory) @@ -75,7 +74,7 @@ private static GitConfig CreateConfig(params (string Name, string[] Values)[] va public void GetRepositoryUrl_NoRemotes() { var repo = CreateRepository(); - var warnings = new List<(string, object[])>(); + var warnings = new List<(string, object?[])>(); Assert.Null(GitOperations.GetRepositoryUrl(repo, remoteName: null, logWarning: (message, args) => warnings.Add((message, args)))); AssertEx.Equal(new[] { string.Format(Resources.RepositoryHasNoRemote, repo.WorkingDirectory) }, warnings.Select(TestUtilities.InspectDiagnostic)); } @@ -87,7 +86,7 @@ public void GetRepositoryUrl_Origin() ("remote.abc.url", "http://github.com/abc"), ("remote.origin.url", "http://github.com/origin"))); - var warnings = new List<(string, object[])>(); + var warnings = new List<(string, object?[])>(); Assert.Equal("http://github.com/origin", GitOperations.GetRepositoryUrl(repo, remoteName: null, logWarning: (message, args) => warnings.Add((message, args)))); @@ -101,7 +100,7 @@ public void GetRepositoryUrl_NoOrigin() ("remote.abc.url", "http://github.com/abc"), ("remote.def.url", "http://github.com/def"))); - var warnings = new List<(string, object[])>(); + var warnings = new List<(string, object?[])>(); Assert.Equal("http://github.com/abc", GitOperations.GetRepositoryUrl(repo, remoteName: null, logWarning: (message, args) => warnings.Add((message, args)))); @@ -115,7 +114,7 @@ public void GetRepositoryUrl_Specified() ("remote.abc.url", "http://github.com/abc"), ("remote.origin.url", "http://github.com/origin"))); - var warnings = new List<(string, object[])>(); + var warnings = new List<(string, object?[])>(); Assert.Equal("http://github.com/abc", GitOperations.GetRepositoryUrl(repo, remoteName: "abc", @@ -131,7 +130,7 @@ public void GetRepositoryUrl_SpecifiedNotFound_OriginFallback() ("remote.abc.url", "http://github.com/abc"), ("remote.origin.url", "http://github.com/origin"))); - var warnings = new List<(string, object[])>(); + var warnings = new List<(string, object?[])>(); Assert.Equal("http://github.com/origin", GitOperations.GetRepositoryUrl(repo, remoteName: "myremote", @@ -150,7 +149,7 @@ public void GetRepositoryUrl_SpecifiedNotFound_FirstFallback() ("remote.abc.url", "http://github.com/abc"), ("remote.def.url", "http://github.com/def"))); - var warnings = new List<(string, object[])>(); + var warnings = new List<(string, object?[])>(); Assert.Equal("http://github.com/abc", GitOperations.GetRepositoryUrl(repo, remoteName: "myremote", @@ -167,7 +166,7 @@ public void GetRepositoryUrl_BadUrl() { var repo = CreateRepository(config: CreateConfig(("remote.origin.url", "http://?"))); - var warnings = new List<(string, object[])>(); + var warnings = new List<(string, object?[])>(); Assert.Null(GitOperations.GetRepositoryUrl(repo, remoteName: null, logWarning: (message, args) => warnings.Add((message, args)))); AssertEx.Equal(new[] { @@ -184,7 +183,7 @@ public void GetRepositoryUrl_InsteadOf() new KeyValuePair>(new GitVariableName("url", "git@github.com:org/repo", "insteadOf"), ImmutableArray.Create("http://?")) }))); - var warnings = new List<(string, object[])>(); + var warnings = new List<(string, object?[])>(); Assert.Equal("ssh://git@github.com/org/repo", GitOperations.GetRepositoryUrl(repo, remoteName: null, logWarning: (message, args) => warnings.Add((message, args)))); Assert.Empty(warnings); } @@ -211,7 +210,7 @@ public void GetRepositoryUrl_Local(bool useFileUrl) ImmutableArray.Create(useFileUrl ? new Uri(mainWorkingDir.Path).AbsolutePath : mainWorkingDir.Path)), }))); - var warnings = new List<(string, object[])>(); + var warnings = new List<(string, object?[])>(); Assert.Equal("http://github.com/repo", GitOperations.GetRepositoryUrl(repo, remoteName: null, logWarning: (message, args) => warnings.Add((message, args)))); Assert.Empty(warnings); } @@ -235,16 +234,16 @@ public void GetRepositoryUrl_Local_CustomRemoteName() new KeyValuePair>(new GitVariableName("remote", "origin", "url"), ImmutableArray.Create(mainWorkingDir.Path)), }))); - var warnings = new List<(string, object[])>(); + var warnings = new List<(string, object?[])>(); Assert.Equal("http://github.com/repo", GitOperations.GetRepositoryUrl(repo, remoteName: "myremote", logWarning: (message, args) => warnings.Add((message, args)))); AssertEx.Equal(new[] { string.Format(Resources.RepositoryDoesNotHaveSpecifiedRemote, repo.WorkingDirectory, "myremote", "origin") }, warnings.Select(TestUtilities.InspectDiagnostic)); } /// - /// Test scenario where a local repository is cloned from another local repository that was cloned from a remote repository. + /// Test scenario where a local repository is cloned from another local repository that does not have remote URL specified. /// [Fact] - public void GetRepositoryUrl_Local_NoRemote() + public void GetRepositoryUrl_Local_NoRemoteOriginUrl() { using var temp = new TempRoot(); @@ -257,11 +256,39 @@ public void GetRepositoryUrl_Local_NoRemote() new KeyValuePair>(new GitVariableName("remote", "origin", "url"), ImmutableArray.Create(mainWorkingDir.Path)), }))); - var warnings = new List<(string, object[])>(); + var warnings = new List<(string, object?[])>(); Assert.Equal(new Uri(mainWorkingDir.Path).AbsoluteUri, GitOperations.GetRepositoryUrl(repo, remoteName: null, logWarning: (message, args) => warnings.Add((message, args)))); AssertEx.Equal(new[] { string.Format(Resources.RepositoryHasNoRemote, mainWorkingDir.Path) }, warnings.Select(TestUtilities.InspectDiagnostic)); } + /// + /// Test scenario where a local repository is cloned from another local repository that does not have a working directory. + /// + [Fact] + public void GetRepositoryUrl_Local_NoWorkingDirectory() + { + using var temp = new TempRoot(); + + var dir = temp.CreateDirectory(); + + var gitDir1 = dir.CreateDirectory("1"); + gitDir1.CreateFile("HEAD"); + gitDir1.CreateFile("config").WriteAllText(@"[remote ""origin""] url = http://github.com/repo"); + + var gitDir2 = dir.CreateDirectory("2"); + gitDir2.CreateFile("HEAD"); + gitDir2.CreateFile("config").WriteAllText(@"[remote ""origin""] url = ../1"); + + var repo = CreateRepository(config: new GitConfig(ImmutableDictionary.CreateRange(new[] + { + new KeyValuePair>(new GitVariableName("remote", "origin", "url"), ImmutableArray.Create(gitDir2.Path)), + }))); + + var warnings = new List<(string, object?[])>(); + Assert.Null(GitOperations.GetRepositoryUrl(repo, remoteName: null, logWarning: (message, args) => warnings.Add((message, args)))); + AssertEx.Equal(new[] { string.Format(Resources.UnableToLocateRepository, gitDir2.Path) }, warnings.Select(TestUtilities.InspectDiagnostic)); + } + /// /// Test scenario where a local repository is cloned from another local repository that was cloned from a remote repository. /// @@ -278,7 +305,7 @@ public void GetRepositoryUrl_Local_BadRepo() new KeyValuePair>(new GitVariableName("remote", "origin", "url"), ImmutableArray.Create(mainWorkingDir.Path)), }))); - var warnings = new List<(string, object[])>(); + var warnings = new List<(string, object?[])>(); Assert.Equal(new Uri(mainWorkingDir.Path).AbsoluteUri, GitOperations.GetRepositoryUrl(repo, remoteName: null, logWarning: (message, args) => warnings.Add((message, args)))); AssertEx.Equal(new[] { string.Format(Resources.RepositoryHasNoRemote, mainWorkingDir.Path) }, warnings.Select(TestUtilities.InspectDiagnostic)); } @@ -299,7 +326,7 @@ public void GetRepositoryUrl_LocalRecursion() ImmutableArray.Create(mainWorkingDir.Path)), }))); - var warnings = new List<(string, object[])>(); + var warnings = new List<(string, object?[])>(); Assert.Equal(new Uri(mainWorkingDir.Path).AbsoluteUri, GitOperations.GetRepositoryUrl(repo, remoteName: null, logWarning: (message, args) => warnings.Add((message, args)))); AssertEx.Equal(new[] { string.Format(Resources.RepositoryUrlEvaluationExceededMaximumAllowedDepth, "10") }, warnings.Select(TestUtilities.InspectDiagnostic)); } @@ -422,7 +449,7 @@ public void GetSourceRoots_RepoWithoutCommits() { var repo = CreateRepository(); - var warnings = new List<(string, object[])>(); + var warnings = new List<(string, object?[])>(); var items = GitOperations.GetSourceRoots(repo, remoteName: null, (message, args) => warnings.Add((message, args))); Assert.Empty(items); @@ -439,7 +466,7 @@ public void GetSourceRoots_RepoWithoutCommitsWithSubmodules() CreateSubmodule("1", "sub/1", "http://1.com", "1111111111111111111111111111111111111111"), CreateSubmodule("1", "sub/2", "http://2.com", "2222222222222222222222222222222222222222"))); ; - var warnings = new List<(string, object[])>(); + var warnings = new List<(string, object?[])>(); var items = GitOperations.GetSourceRoots(repo, remoteName: null, (message, args) => warnings.Add((message, args))); AssertEx.Equal(new[] @@ -460,7 +487,7 @@ public void GetSourceRoots_RepoWithCommitsWithSubmodules() CreateSubmodule("1", "sub/1", "http://1.com", headCommitSha: null), CreateSubmodule("1", "sub/2", "http://2.com", "2222222222222222222222222222222222222222"))); - var warnings = new List<(string, object[])>(); + var warnings = new List<(string, object?[])>(); var items = GitOperations.GetSourceRoots(repo, remoteName: null, (message, args) => warnings.Add((message, args))); AssertEx.Equal(new[] @@ -494,7 +521,7 @@ public void GetSourceRoots_RelativeSubmodulePaths() submodules: ImmutableArray.Create( CreateSubmodule("1", "sub/1", "../1", "1111111111111111111111111111111111111111", containingRepositoryWorkingDir: repoDir.Path))); - var warnings = new List<(string, object[])>(); + var warnings = new List<(string, object?[])>(); var items = GitOperations.GetSourceRoots(repo, remoteName: null, (message, args) => warnings.Add((message, args))); AssertEx.Equal(new[] @@ -515,7 +542,7 @@ public void GetSourceRoots_InvalidSubmoduleUrl() CreateSubmodule("1", "sub/1", "http:///", "1111111111111111111111111111111111111111"), CreateSubmodule("3", "sub/3", "http://3.com", "3333333333333333333333333333333333333333"))); - var warnings = new List<(string, object[])>(); + var warnings = new List<(string, object?[])>(); var items = GitOperations.GetSourceRoots(repo, remoteName: null, (message, args) => warnings.Add((message, args))); AssertEx.Equal(new[] @@ -530,84 +557,91 @@ public void GetSourceRoots_InvalidSubmoduleUrl() }, warnings.Select(TestUtilities.InspectDiagnostic)); } + private static GitOperations.DirectoryNode CreateNode(string name, string? submoduleWorkingDirectory, List? children = null) + => new GitOperations.DirectoryNode(name, children ?? new List()) + { + Matcher = (submoduleWorkingDirectory != null) ? new Lazy(() => + new GitIgnore.Matcher(new GitIgnore(null, submoduleWorkingDirectory, ignoreCase: false))) : null + }; + [ConditionalTheory(typeof(WindowsOnly))] [InlineData(@"C:\", null)] [InlineData(@"C:\x", null)] [InlineData(@"C:\x\y\z", null)] [InlineData(@"C:\src", null)] [InlineData(@"C:\src\", null)] - [InlineData(@"C:\src\a\x.cs", @"C:\src\a")] - [InlineData(@"C:\src\b\x.cs", @"C:\src")] - [InlineData(@"C:\src\ab\x.cs", @"C:\src")] - [InlineData(@"C:\src\a\b\x.cs", @"C:\src\a")] - [InlineData(@"C:\src\c\x.cs", @"C:\src\c")] - [InlineData(@"C:\src\c", @"C:\src")] - [InlineData(@"C:\src\c\", @"C:\src")] - [InlineData(@"C:\src\c.cs", @"C:\src")] - [InlineData(@"C:\src\c\x\x.cs", @"C:\src\c\x")] - [InlineData(@"C:\src\d\x.cs", @"C:\src")] - [InlineData(@"C:\src\e\x.cs", @"C:\src\e")] + [InlineData(@"C:\src\a\x.cs", "C:/src/a/")] + [InlineData(@"C:\src\b\x.cs", "C:/src/")] + [InlineData(@"C:\src\ab\x.cs", "C:/src/")] + [InlineData(@"C:\src\a\b\x.cs", "C:/src/a/")] + [InlineData(@"C:\src\c\x.cs", "C:/src/c/")] + [InlineData(@"C:\src\c", "C:/src/")] + [InlineData(@"C:\src\c\", "C:/src/")] + [InlineData(@"C:\src\c.cs", "C:/src/")] + [InlineData(@"C:\src\c\x\x.cs", "C:/src/c/x/")] + [InlineData(@"C:\src\d\x.cs", "C:/src/")] + [InlineData(@"C:\src\e\x.cs", "C:/src/e/")] public void GetContainingRepository_Windows(string path, string expectedDirectory) { - var actual = GitOperations.GetContainingRepository(path, - new GitOperations.DirectoryNode("", null, + var actual = GitOperations.GetContainingRepositoryMatcher(path, + CreateNode("", null, new List { - new GitOperations.DirectoryNode("C:", null, new List + CreateNode("C:", null, new List { - new GitOperations.DirectoryNode("src", @"C:\src", new List + CreateNode("src", @"C:\src", new List { - new GitOperations.DirectoryNode("a", @"C:\src\a"), - new GitOperations.DirectoryNode("c", @"C:\src\c", new List + CreateNode("a", @"C:\src\a"), + CreateNode("c", @"C:\src\c", new List { - new GitOperations.DirectoryNode("x", @"C:\src\c\x") + CreateNode("x", @"C:\src\c\x") }), - new GitOperations.DirectoryNode("e", @"C:\src\e") + CreateNode("e", @"C:\src\e") }), }) })); - Assert.Equal(expectedDirectory, actual?.WorkingDirectoryFullPath); + Assert.Equal(expectedDirectory, actual?.Ignore.WorkingDirectory); } [ConditionalTheory(typeof(UnixOnly))] - [InlineData(@"/", null)] - [InlineData(@"/x", null)] - [InlineData(@"/x/y/z", null)] - [InlineData(@"/src", null)] - [InlineData(@"/src/", null)] - [InlineData(@"/src/a/x.cs", @"/src/a")] - [InlineData(@"/src/b/x.cs", @"/src")] - [InlineData(@"/src/ab/x.cs", @"/src")] - [InlineData(@"/src/a/b/x.cs", @"/src/a")] - [InlineData(@"/src/c/x.cs", @"/src/c")] - [InlineData(@"/src/c", @"/src")] - [InlineData(@"/src/c/", @"/src")] - [InlineData(@"/src/c.cs", @"/src")] - [InlineData(@"/src/c/x/x.cs", @"/src/c/x")] - [InlineData(@"/src/d/x.cs", @"/src")] - [InlineData(@"/src/e/x.cs", @"/src/e")] + [InlineData("/", null)] + [InlineData("/x", null)] + [InlineData("/x/y/z", null)] + [InlineData("/src", null)] + [InlineData("/src/", null)] + [InlineData("/src/a/x.cs", "/src/a/")] + [InlineData("/src/b/x.cs", "/src/")] + [InlineData("/src/ab/x.cs", "/src/")] + [InlineData("/src/a/b/x.cs", "/src/a/")] + [InlineData("/src/c/x.cs", "/src/c/")] + [InlineData("/src/c", "/src/")] + [InlineData("/src/c/", "/src/")] + [InlineData("/src/c.cs", "/src/")] + [InlineData("/src/c/x/x.cs", "/src/c/x/")] + [InlineData("/src/d/x.cs", "/src/")] + [InlineData("/src/e/x.cs", "/src/e/")] public void GetContainingRepository_Unix(string path, string expectedDirectory) { - var actual = GitOperations.GetContainingRepository(path, - new GitOperations.DirectoryNode("", null, + var actual = GitOperations.GetContainingRepositoryMatcher(path, + CreateNode("", null, new List { - new GitOperations.DirectoryNode("/", null, new List + CreateNode("/", null, new List { - new GitOperations.DirectoryNode("src", "/src", new List + CreateNode("src", "/src", new List { - new GitOperations.DirectoryNode("a", "/src/a"), - new GitOperations.DirectoryNode("c", "/src/c", new List + CreateNode("a", "/src/a"), + CreateNode("c", "/src/c", new List { - new GitOperations.DirectoryNode("x", "/src/c/x"), + CreateNode("x", "/src/c/x"), }), - new GitOperations.DirectoryNode("e", "/src/e"), + CreateNode("e", "/src/e"), }), }) })); - Assert.Equal(expectedDirectory, actual?.WorkingDirectoryFullPath); + Assert.Equal(expectedDirectory, actual?.Ignore.WorkingDirectory); } [Fact] @@ -623,10 +657,10 @@ public void BuildDirectoryTree() CreateSubmodule("5", "c", "http://github.com/5", null), CreateSubmodule("6", "a/z", "http://github.com/6", null))); - var root = GitOperations.BuildDirectoryTree(repo); + var root = GitOperations.BuildDirectoryTree(repo, (e, d) => null); - string inspect(GitOperations.DirectoryNode node) - => node.Name + (node.WorkingDirectoryFullPath != null ? $"!" : "") + "{" + string.Join(",", node.OrderedChildren.Select(inspect)) + "}"; + static string inspect(GitOperations.DirectoryNode node) + => node.Name + (node.Matcher != null ? $"!" : "") + "{" + string.Join(",", node.OrderedChildren.Select(inspect)) + "}"; var expected = IsUnix ? "{/{usr{src!{a!{a{a{a!{}}},z!{}},c!{x!{}},e!{}}}}}" : @@ -665,7 +699,7 @@ public void GetUntrackedFiles_ProjectInMainRepoIncludesFilesInSubmodules() new MockItem(IsUnix ? "/d/w.cs" : @"D:\w.cs"), // outside of repo }, projectDirectory: Path.Combine(_workingDir, "p"), - root => subRepos[root]); + (_, root) => subRepos[root]); AssertEx.Equal(new[] { @@ -706,7 +740,7 @@ public void GetUntrackedFiles_ProjectInSubmodule() new MockItem(@"..\..\c.cs"), // ignored in main repo }, projectDirectory: subRoot1, - root => subRepos[root]); + (_, root) => subRepos[root]); AssertEx.Equal(new[] { diff --git a/src/Microsoft.Build.Tasks.Git.UnitTests/GitRepositoryTests.cs b/src/Microsoft.Build.Tasks.Git.UnitTests/GitRepositoryTests.cs index aa8db1b2..8d611bb4 100644 --- a/src/Microsoft.Build.Tasks.Git.UnitTests/GitRepositoryTests.cs +++ b/src/Microsoft.Build.Tasks.Git.UnitTests/GitRepositoryTests.cs @@ -159,7 +159,7 @@ public void OpenRepository() var src = workingDir.CreateDirectory("src"); - var repository = GitRepository.OpenRepository(src.Path, new GitEnvironment(homeDir.Path)); + var repository = GitRepository.OpenRepository(src.Path, new GitEnvironment(homeDir.Path))!; Assert.Equal(gitDir.Path, repository.CommonDirectory); Assert.Equal(gitDir.Path, repository.GitDirectory); @@ -218,7 +218,7 @@ public void Submodules() "); var repository = new GitRepository(GitEnvironment.Empty, GitConfig.Empty, gitDir.Path, gitDir.Path, workingDir.Path); - var submodules = GitRepository.EnumerateSubmoduleConfig(repository.ReadSubmoduleConfig()); + var submodules = GitRepository.EnumerateSubmoduleConfig(repository.ReadSubmoduleConfig()!); AssertEx.Equal(new[] { "S1: 'subs/s1' 'http://github.com/test1'", diff --git a/src/Microsoft.Build.Tasks.Git.UnitTests/GlobTests.cs b/src/Microsoft.Build.Tasks.Git.UnitTests/GlobTests.cs index f519c24c..13ff1f65 100644 --- a/src/Microsoft.Build.Tasks.Git.UnitTests/GlobTests.cs +++ b/src/Microsoft.Build.Tasks.Git.UnitTests/GlobTests.cs @@ -103,7 +103,7 @@ public void NonMatching(string pattern, string path) [InlineData("[^]", new char[0])] [InlineData("[]", new char[0])] [InlineData("[a-]]", new char[0])] - public void MatchingRange(string pattern, char[] matchingChars, char[] matchingCharsWildCardMatchesSeparator = null) + public void MatchingRange(string pattern, char[] matchingChars, char[]? matchingCharsWildCardMatchesSeparator = null) { for (int i = 0; i < 255; i++) { diff --git a/src/Microsoft.Build.Tasks.Git.UnitTests/TestUtilities.cs b/src/Microsoft.Build.Tasks.Git.UnitTests/TestUtilities.cs index 03254d58..6497485f 100644 --- a/src/Microsoft.Build.Tasks.Git.UnitTests/TestUtilities.cs +++ b/src/Microsoft.Build.Tasks.Git.UnitTests/TestUtilities.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; -using System.IO; using Microsoft.Build.Framework; namespace Microsoft.Build.Tasks.Git.UnitTests @@ -26,10 +25,10 @@ public static string InspectSourceRoot(ITaskItem sourceRoot) (string.IsNullOrEmpty(sourceLinkUrl) ? "" : $" SourceLinkUrl='{sourceLinkUrl}'"); } - public static string InspectDiagnostic((string Message, object[] Args) warning) + public static string InspectDiagnostic((string Message, object?[] Args) warning) => string.Format(warning.Message, warning.Args); - public static string GetExceptionMessage(Action action) + public static string? GetExceptionMessage(Action action) { try { diff --git a/src/Microsoft.Build.Tasks.Git/AssemblyResolver.cs b/src/Microsoft.Build.Tasks.Git/AssemblyResolver.cs index 1430a7f1..9889bb54 100644 --- a/src/Microsoft.Build.Tasks.Git/AssemblyResolver.cs +++ b/src/Microsoft.Build.Tasks.Git/AssemblyResolver.cs @@ -8,7 +8,6 @@ using System.Collections.Generic; using System.IO; using System.Reflection; -using Microsoft.Build.Framework; namespace Microsoft.Build.Tasks.Git { @@ -38,7 +37,7 @@ internal static string[] GetLog() } } - private static Assembly AssemblyResolve(object sender, ResolveEventArgs args) + private static Assembly? AssemblyResolve(object sender, ResolveEventArgs args) { var name = new AssemblyName(args.Name); diff --git a/src/Microsoft.Build.Tasks.Git/GetUntrackedFiles.cs b/src/Microsoft.Build.Tasks.Git/GetUntrackedFiles.cs index ad213419..7465313b 100644 --- a/src/Microsoft.Build.Tasks.Git/GetUntrackedFiles.cs +++ b/src/Microsoft.Build.Tasks.Git/GetUntrackedFiles.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System.Diagnostics.CodeAnalysis; using Microsoft.Build.Framework; namespace Microsoft.Build.Tasks.Git @@ -9,24 +10,23 @@ namespace Microsoft.Build.Tasks.Git /// public sealed class GetUntrackedFiles : RepositoryTask { - public string RepositoryId { get; set; } + public string? RepositoryId { get; set; } - [Required] - public ITaskItem[] Files { get; set; } + [Required, NotNull] + public ITaskItem[]? Files { get; set; } - [Required] - public string ProjectDirectory { get; set; } + [Required, NotNull] + public string? ProjectDirectory { get; set; } [Output] - public ITaskItem[] UntrackedFiles { get; private set; } + public ITaskItem[]? UntrackedFiles { get; private set; } - protected override string GetRepositoryId() => RepositoryId; + protected override string? GetRepositoryId() => RepositoryId; protected override string GetInitialPath() => ProjectDirectory; private protected override void Execute(GitRepository repository) { - UntrackedFiles = GitOperations.GetUntrackedFiles( - repository, Files, ProjectDirectory, dir => GitRepository.OpenRepository(dir, repository.Environment)); + UntrackedFiles = GitOperations.GetUntrackedFiles(repository, Files, ProjectDirectory); } } } diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitConfig.Reader.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitConfig.Reader.cs index 61507b04..34c28916 100644 --- a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitConfig.Reader.cs +++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitConfig.Reader.cs @@ -6,7 +6,6 @@ using System.Diagnostics; using System.IO; using System.Text; -using Microsoft.Build.Tasks.SourceControl; namespace Microsoft.Build.Tasks.Git { @@ -26,9 +25,9 @@ internal class Reader private readonly Func _fileOpener; private readonly GitEnvironment _environment; - public Reader(string gitDirectory, string commonDirectory, GitEnvironment environment, Func fileOpener = null) + public Reader(string gitDirectory, string commonDirectory, GitEnvironment environment, Func? fileOpener = null) { - Debug.Assert(environment != null); + NullableDebug.Assert(environment != null); _environment = environment; _gitDirectoryPosix = PathUtils.ToPosixDirectoryPath(gitDirectory); @@ -61,7 +60,7 @@ internal GitConfig LoadFrom(string path) return new GitConfig(variables.ToImmutableDictionary(kvp => kvp.Key, kvp => kvp.Value.ToImmutableArray())); } - private string GetXdgDirectory() + private string? GetXdgDirectory() { var xdgConfigHome = _environment.XdgConfigHomeDirectory; if (xdgConfigHome != null) @@ -132,7 +131,7 @@ internal IEnumerable EnumerateExistingConfigurationFiles() // worktree config } - private string GetSystemConfigurationDirectory() + private string? GetSystemConfigurationDirectory() { if (_environment.SystemDirectory == null) { diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitConfig.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitConfig.cs index 9f930f52..d55cd02d 100644 --- a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitConfig.cs +++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitConfig.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; using System.Text; @@ -18,7 +19,7 @@ internal sealed partial class GitConfig public GitConfig(ImmutableDictionary> variables) { - Debug.Assert(variables != null); + NullableDebug.Assert(variables != null); Variables = variables; } @@ -32,19 +33,19 @@ public ImmutableArray GetVariableValues(string section, string name) public ImmutableArray GetVariableValues(string section, string subsection, string name) => Variables.TryGetValue(new GitVariableName(section, subsection, name), out var multiValue) ? multiValue : default; - public string GetVariableValue(string section, string name) + public string? GetVariableValue(string section, string name) => GetVariableValue(section, "", name); - public string GetVariableValue(string section, string subsection, string name) + public string? GetVariableValue(string section, string subsection, string name) { var values = GetVariableValues(section, subsection, name); return values.IsDefault ? null : values[values.Length - 1]; } - public static bool ParseBooleanValue(string str, bool defaultValue = false) + public static bool ParseBooleanValue(string? str, bool defaultValue = false) => TryParseBooleanValue(str, out var value) ? value : defaultValue; - public static bool TryParseBooleanValue(string str, out bool value) + public static bool TryParseBooleanValue(string? str, out bool value) { // https://git-scm.com/docs/git-config#Documentation/git-config.txt-boolean @@ -75,9 +76,9 @@ public static bool TryParseBooleanValue(string str, out bool value) internal static long ParseInt64Value(string str, long defaultValue = 0) => TryParseInt64Value(str, out var value) ? value : defaultValue; - internal static bool TryParseInt64Value(string str, out long value) + internal static bool TryParseInt64Value(string? str, out long value) { - if (string.IsNullOrEmpty(str)) + if (NullableString.IsNullOrEmpty(str)) { value = 0; return false; diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitEnvironment.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitEnvironment.cs index 75bc39a1..6d7be601 100644 --- a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitEnvironment.cs +++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitEnvironment.cs @@ -2,9 +2,9 @@ using System; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; -using Microsoft.Build.Tasks.SourceControl; namespace Microsoft.Build.Tasks.Git { @@ -15,10 +15,10 @@ internal sealed class GitEnvironment public static readonly GitEnvironment Empty = new GitEnvironment(); - public string HomeDirectory { get; } - public string XdgConfigHomeDirectory { get; } - public string ProgramDataDirectory { get; } - public string SystemDirectory { get; } + public string? HomeDirectory { get; } + public string? XdgConfigHomeDirectory { get; } + public string? ProgramDataDirectory { get; } + public string? SystemDirectory { get; } // TODO: https://github.com/dotnet/sourcelink/issues/301 // consider adding environment variables: GIT_DIR, GIT_DISCOVERY_ACROSS_FILESYSTEM, GIT_CEILING_DIRECTORIES @@ -33,10 +33,10 @@ internal sealed class GitEnvironment // https://git-scm.com/docs/git#Documentation/git.txt-codeGITWORKTREEcode public GitEnvironment( - string homeDirectory = null, - string xdgConfigHomeDirectory = null, - string programDataDirectory = null, - string systemDirectory = null) + string? homeDirectory = null, + string? xdgConfigHomeDirectory = null, + string? programDataDirectory = null, + string? systemDirectory = null) { if (!string.IsNullOrWhiteSpace(homeDirectory)) { @@ -59,9 +59,9 @@ public GitEnvironment( } } - public static GitEnvironment Create(string configurationScope) + public static GitEnvironment Create(string? configurationScope) { - if (string.IsNullOrEmpty(configurationScope)) + if (NullableString.IsNullOrEmpty(configurationScope)) { return CreateFromProcessEnvironment(); } @@ -76,7 +76,7 @@ public static GitEnvironment Create(string configurationScope) public static GitEnvironment CreateFromProcessEnvironment() { - static string getVariable(string name) + static string? getVariable(string name) { try { @@ -88,7 +88,7 @@ static string getVariable(string name) } } - string homeDirectory; + string? homeDirectory; try { homeDirectory = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile, Environment.SpecialFolderOption.DoNotVerify); @@ -106,7 +106,7 @@ static string getVariable(string name) } // internal for testing - internal static string FindSystemDirectory(string pathList, string unixEtcDir) + internal static string? FindSystemDirectory(string? pathList, string? unixEtcDir) { if (PathUtils.IsUnixLikePlatform) { @@ -122,11 +122,11 @@ internal static string FindSystemDirectory(string pathList, string unixEtcDir) return null; } - private static string FindWindowsGitInstallation(string pathList) + private static string? FindWindowsGitInstallation(string? pathList) { Debug.Assert(!PathUtils.IsUnixLikePlatform); - if (string.IsNullOrEmpty(pathList)) + if (NullableString.IsNullOrEmpty(pathList)) { return null; } diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitIgnore.Matcher.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitIgnore.Matcher.cs index d36ce869..5196ac9b 100644 --- a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitIgnore.Matcher.cs +++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitIgnore.Matcher.cs @@ -16,7 +16,7 @@ internal sealed class Matcher /// /// Maps full posix slash-terminated directory name to a pattern group. /// - private readonly Dictionary _patternGroups; + private readonly Dictionary _patternGroups; /// /// The result of "is ignored" for directories. @@ -28,7 +28,7 @@ internal sealed class Matcher internal Matcher(GitIgnore ignore) { Ignore = ignore; - _patternGroups = new Dictionary(StringComparer.Ordinal); + _patternGroups = new Dictionary(StringComparer.Ordinal); _directoryIgnoreStateCache = new Dictionary(Ignore.PathComparer); _reusableGroupList = new List(); } @@ -37,7 +37,7 @@ internal Matcher(GitIgnore ignore) internal IReadOnlyDictionary DirectoryIgnoreStateCache => _directoryIgnoreStateCache; - private PatternGroup GetPatternGroup(string directory) + private PatternGroup? GetPatternGroup(string directory) { Debug.Assert(PathUtils.HasTrailingSlash(directory)); @@ -46,7 +46,7 @@ private PatternGroup GetPatternGroup(string directory) return group; } - PatternGroup parent; + PatternGroup? parent; if (directory.Equals(Ignore.WorkingDirectory, Ignore.PathComparison)) { parent = Ignore.Root; @@ -159,7 +159,7 @@ private bool IsIgnoredRecursive(string normalizedPosixPath, bool isDirectoryPath return isIgnored; } - private static void SplitPath(string fullPath, out string directoryWithSlash, out string fileName) + private static void SplitPath(string fullPath, out string? directoryWithSlash, out string fileName) { Debug.Assert(!PathUtils.HasTrailingSlash(fullPath)); int i = fullPath.LastIndexOf('/'); @@ -189,7 +189,7 @@ private bool IsIgnored(string normalizedPosixPath, string directory, string file // Patterns specified closer to the target file override those specified above. _reusableGroupList.Clear(); var groups = _reusableGroupList; - for (var patternGroup = GetPatternGroup(directory); patternGroup != null; patternGroup = patternGroup.Parent) + for (PatternGroup? patternGroup = GetPatternGroup(directory); patternGroup != null; patternGroup = patternGroup.Parent) { groups.Add(patternGroup); } @@ -203,7 +203,7 @@ private bool IsIgnored(string normalizedPosixPath, string directory, string file continue; } - string lazyRelativePath = null; + string? lazyRelativePath = null; foreach (var pattern in patternGroup.Patterns) { diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitIgnore.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitIgnore.cs index 99f19436..19ab8688 100644 --- a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitIgnore.cs +++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitIgnore.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.Text; @@ -21,12 +22,12 @@ internal sealed class PatternGroup public readonly ImmutableArray Patterns; - public readonly PatternGroup Parent; + public readonly PatternGroup? Parent; - public PatternGroup(PatternGroup parent, string containingDirectory, ImmutableArray patterns) + public PatternGroup(PatternGroup? parent, string containingDirectory, ImmutableArray patterns) { - Debug.Assert(PathUtils.IsPosixPath(containingDirectory)); - Debug.Assert(PathUtils.HasTrailingSlash(containingDirectory)); + NullableDebug.Assert(PathUtils.IsPosixPath(containingDirectory)); + NullableDebug.Assert(PathUtils.HasTrailingSlash(containingDirectory)); Parent = parent; ContainingDirectory = containingDirectory; @@ -41,8 +42,6 @@ internal readonly struct Pattern public Pattern(string glob, PatternFlags flags) { - Debug.Assert(glob != null); - Glob = glob; Flags = flags; } @@ -74,11 +73,11 @@ internal enum PatternFlags public bool IgnoreCase { get; } - public PatternGroup Root { get; } + public PatternGroup? Root { get; } - internal GitIgnore(PatternGroup root, string workingDirectory, bool ignoreCase) + internal GitIgnore(PatternGroup? root, string workingDirectory, bool ignoreCase) { - Debug.Assert(PathUtils.IsAbsolute(workingDirectory)); + NullableDebug.Assert(PathUtils.IsAbsolute(workingDirectory)); IgnoreCase = ignoreCase; WorkingDirectory = PathUtils.ToPosixDirectoryPath(workingDirectory); @@ -97,7 +96,7 @@ public Matcher CreateMatcher() /// /// is invalid - internal static PatternGroup LoadFromFile(string path, PatternGroup parent) + internal static PatternGroup? LoadFromFile(string? path, PatternGroup? parent) { // See https://git-scm.com/docs/gitignore#_pattern_format @@ -146,7 +145,7 @@ internal static PatternGroup LoadFromFile(string path, PatternGroup parent) return new PatternGroup(parent, directory, patterns.ToImmutable()); } - internal static bool TryParsePattern(string line, StringBuilder reusableBuffer, out string glob, out PatternFlags flags) + internal static bool TryParsePattern(string line, StringBuilder reusableBuffer, [NotNullWhen(true)]out string? glob, out PatternFlags flags) { glob = null; flags = PatternFlags.None; diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitReferenceResolver.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitReferenceResolver.cs index 80518f9f..e62bd4ed 100644 --- a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitReferenceResolver.cs +++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitReferenceResolver.cs @@ -72,7 +72,7 @@ internal static ImmutableDictionary ReadPackedReferences(TextRea throw new InvalidDataException($"Expected header not found at the beginning of file '{path}'."); } - string previousObjectId = null; + string? previousObjectId = null; while (true) { var line = reader.ReadLine(); @@ -147,18 +147,18 @@ internal static ImmutableDictionary ReadPackedReferences(TextRea /// /// - public string ResolveHeadReference() + public string? ResolveHeadReference() => ResolveReference(ReadReferenceFromFile(Path.Combine(_gitDirectory, GitRepository.GitHeadFileName))); - public string ResolveReference(string reference) + public string? ResolveReference(string reference) { - HashSet lazyVisitedReferences = null; + HashSet? lazyVisitedReferences = null; return ResolveReference(reference, ref lazyVisitedReferences); } /// /// - private string ResolveReference(string reference, ref HashSet lazyVisitedReferences) + private string? ResolveReference(string reference, ref HashSet? lazyVisitedReferences) { // See https://git-scm.com/docs/gitrepository-layout#Documentation/gitrepository-layout.txt-HEAD @@ -231,7 +231,7 @@ internal static string ReadReferenceFromFile(string path) } } - private string ResolvePackedReference(string reference) + private string? ResolvePackedReference(string reference) => _lazyPackedReferences.Value.TryGetValue(reference, out var objectId) ? objectId : null; private static bool IsObjectId(string reference) diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs index 196c57dc..9bab50f5 100644 --- a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs +++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; @@ -40,22 +41,22 @@ internal sealed class GitRepository /// /// Normalized full path. OS specific directory separators. Optional. /// - public string WorkingDirectory { get; } + public string? WorkingDirectory { get; } public GitEnvironment Environment { get; } private readonly Lazy<(ImmutableArray Submodules, ImmutableArray Diagnostics)> _lazySubmodules; private readonly Lazy _lazyIgnore; - private readonly Lazy _lazyHeadCommitSha; + private readonly Lazy _lazyHeadCommitSha; private readonly GitReferenceResolver _referenceResolver; - internal GitRepository(GitEnvironment environment, GitConfig config, string gitDirectory, string commonDirectory, string workingDirectory) + internal GitRepository(GitEnvironment environment, GitConfig config, string gitDirectory, string commonDirectory, string? workingDirectory) { - Debug.Assert(environment != null); - Debug.Assert(config != null); - Debug.Assert(PathUtils.IsNormalized(gitDirectory)); - Debug.Assert(PathUtils.IsNormalized(commonDirectory)); - Debug.Assert(workingDirectory == null || PathUtils.IsNormalized(workingDirectory)); + NullableDebug.Assert(environment != null); + NullableDebug.Assert(config != null); + NullableDebug.Assert(PathUtils.IsNormalized(gitDirectory)); + NullableDebug.Assert(PathUtils.IsNormalized(commonDirectory)); + NullableDebug.Assert(workingDirectory == null || PathUtils.IsNormalized(workingDirectory)); Config = config; GitDirectory = gitDirectory; @@ -66,7 +67,7 @@ internal GitRepository(GitEnvironment environment, GitConfig config, string gitD _referenceResolver = new GitReferenceResolver(gitDirectory, commonDirectory); _lazySubmodules = new Lazy<(ImmutableArray, ImmutableArray)>(ReadSubmodules); _lazyIgnore = new Lazy(LoadIgnore); - _lazyHeadCommitSha = new Lazy(ReadHeadCommitSha); + _lazyHeadCommitSha = new Lazy(ReadHeadCommitSha); } // test only @@ -75,16 +76,16 @@ internal GitRepository( GitConfig config, string gitDirectory, string commonDirectory, - string workingDirectory, + string? workingDirectory, ImmutableArray submodules, ImmutableArray submoduleDiagnostics, GitIgnore ignore, - string headCommitSha) + string? headCommitSha) : this(environment, config, gitDirectory, commonDirectory, workingDirectory) { _lazySubmodules = new Lazy<(ImmutableArray, ImmutableArray)>(() => (submodules, submoduleDiagnostics)); _lazyIgnore = new Lazy(() => ignore); - _lazyHeadCommitSha = new Lazy(() => headCommitSha); + _lazyHeadCommitSha = new Lazy(() => headCommitSha); } /// @@ -94,7 +95,7 @@ internal GitRepository( /// /// The repository found requires higher version of git repository format that is currently supported. /// null if no git repository can be found that contains the specified path. - internal static GitRepository OpenRepository(string path, GitEnvironment environment) + internal static GitRepository? OpenRepository(string path, GitEnvironment environment) => TryFindRepository(path, out var location) ? OpenRepository(location, environment) : null; /// @@ -105,19 +106,19 @@ internal static GitRepository OpenRepository(string path, GitEnvironment environ /// The repository found requires higher version of git repository format that is currently supported. public static GitRepository OpenRepository(GitRepositoryLocation location, GitEnvironment environment) { - Debug.Assert(environment != null); - Debug.Assert(location.GitDirectory != null); - Debug.Assert(location.CommonDirectory != null); + NullableDebug.Assert(environment != null); + NullableDebug.Assert(location.GitDirectory != null); + NullableDebug.Assert(location.CommonDirectory != null); // See https://git-scm.com/docs/gitrepository-layout var reader = new GitConfig.Reader(location.GitDirectory, location.CommonDirectory, environment); var config = reader.Load(); - var workingDirectory = GetWorkingDirectory(config, location.GitDirectory, location.CommonDirectory) ?? location.WorkingDirectory; + var workingDirectory = GetWorkingDirectory(config, location); // See https://github.com/git/git/blob/master/Documentation/technical/repository-version.txt - string versionStr = config.GetVariableValue("core", "repositoryformatversion"); + string? versionStr = config.GetVariableValue("core", "repositoryformatversion"); if (GitConfig.TryParseInt64Value(versionStr, out var version) && version > SupportedGitRepoFormatVersion) { throw new NotSupportedException(string.Format(Resources.UnsupportedRepositoryVersion, versionStr, SupportedGitRepoFormatVersion)); @@ -126,15 +127,14 @@ public static GitRepository OpenRepository(GitRepositoryLocation location, GitEn return new GitRepository(environment, config, location.GitDirectory, location.CommonDirectory, workingDirectory); } - // internal for testing - internal static string GetWorkingDirectory(GitConfig config, string gitDirectory, string commonDirectory) + private static string? GetWorkingDirectory(GitConfig config, GitRepositoryLocation location) { // Working trees cannot have the same common directory and git directory. // 'gitdir' file in a git directory indicates a working tree. - var gitdirFilePath = Path.Combine(gitDirectory, GitDirFileName); + var gitdirFilePath = Path.Combine(location.GitDirectory, GitDirFileName); - var isLinkedWorkingTree = PathUtils.ToPosixDirectoryPath(commonDirectory) != PathUtils.ToPosixDirectoryPath(gitDirectory) && + var isLinkedWorkingTree = PathUtils.ToPosixDirectoryPath(location.CommonDirectory) != PathUtils.ToPosixDirectoryPath(location.GitDirectory) && File.Exists(gitdirFilePath); if (isLinkedWorkingTree) @@ -170,13 +170,13 @@ internal static string GetWorkingDirectory(GitConfig config, string gitDirectory } // See https://git-scm.com/docs/git-config#Documentation/git-config.txt-coreworktree - string value = config.GetVariableValue("core", "worktree"); + string? value = config.GetVariableValue("core", "worktree"); if (value != null) { // git does not expand home dir relative path ("~/") try { - return Path.GetFullPath(Path.Combine(gitDirectory, value)); + return Path.GetFullPath(Path.Combine(location.GitDirectory, value)); } catch { @@ -184,7 +184,7 @@ internal static string GetWorkingDirectory(GitConfig config, string gitDirectory } } - return null; + return location.WorkingDirectory; } /// @@ -193,12 +193,12 @@ internal static string GetWorkingDirectory(GitConfig config, string gitDirectory /// /// /// Null if the HEAD tip reference can't be resolved. - public string GetHeadCommitSha() + public string? GetHeadCommitSha() => _lazyHeadCommitSha.Value; /// /// - private string ReadHeadCommitSha() + private string? ReadHeadCommitSha() => _referenceResolver.ResolveHeadReference(); /// @@ -207,7 +207,7 @@ private string ReadHeadCommitSha() /// /// /// Null if the HEAD tip reference can't be resolved. - internal string ReadSubmoduleHeadCommitSha(string submoduleWorkingDirectoryFullPath) + internal string? ReadSubmoduleHeadCommitSha(string submoduleWorkingDirectoryFullPath) { var gitDirectory = ReadDotGitFile(Path.Combine(submoduleWorkingDirectoryFullPath, GitDirName)); if (!IsGitDirectory(gitDirectory, out var commonDirectory)) @@ -246,7 +246,7 @@ public ImmutableArray GetSubmoduleDiagnostics() return (ImmutableArray.Empty, ImmutableArray.Empty); } - ImmutableArray.Builder lazyDiagnostics = null; + ImmutableArray.Builder? lazyDiagnostics = null; void reportDiagnostic(string diagnostic) => (lazyDiagnostics ??= ImmutableArray.CreateBuilder()).Add(diagnostic); @@ -255,13 +255,13 @@ void reportDiagnostic(string diagnostic) foreach (var (name, path, url) in EnumerateSubmoduleConfig(submoduleConfig)) { - if (string.IsNullOrWhiteSpace(path)) + if (NullableString.IsNullOrWhiteSpace(path)) { reportDiagnostic(string.Format(Resources.InvalidSubmodulePath, name, path)); continue; } - if (string.IsNullOrWhiteSpace(url)) + if (NullableString.IsNullOrWhiteSpace(url)) { reportDiagnostic(string.Format(Resources.InvalidSubmoduleUrl, name, url)); continue; @@ -278,7 +278,7 @@ void reportDiagnostic(string diagnostic) continue; } - string headCommitSha; + string? headCommitSha; try { headCommitSha = ReadSubmoduleHeadCommitSha(fullPath); @@ -296,7 +296,7 @@ void reportDiagnostic(string diagnostic) } // internal for testing - internal GitConfig ReadSubmoduleConfig() + internal GitConfig? ReadSubmoduleConfig() { var workingDirectory = GetWorkingDirectory(); var submodulesConfigFile = Path.Combine(workingDirectory, GitModulesFileName); @@ -310,7 +310,7 @@ internal GitConfig ReadSubmoduleConfig() } // internal for testing - internal static IEnumerable<(string Name, string Path, string Url)> EnumerateSubmoduleConfig(GitConfig submoduleConfig) + internal static IEnumerable<(string Name, string? Path, string? Url)> EnumerateSubmoduleConfig(GitConfig submoduleConfig) { foreach (var group in submoduleConfig.Variables. Where(kvp => kvp.Key.SectionNameEquals("submodule")). @@ -318,8 +318,8 @@ internal GitConfig ReadSubmoduleConfig() OrderBy(group => group.Key)) { string name = group.Key; - string url = null; - string path = null; + string? url = null; + string? path = null; foreach (var variable in group) { @@ -408,7 +408,7 @@ public static bool TryFindRepository(string directory, out GitRepositoryLocation private static bool TryGetRepositoryLocationImpl(string directory, out GitRepositoryLocation location) { - string commonDirectory; + string? commonDirectory; var dotGitPath = Path.Combine(directory, GitDirName); if (Directory.Exists(dotGitPath)) @@ -475,7 +475,7 @@ private static string ReadDotGitFile(string path) } } - private static bool IsGitDirectory(string directory, out string commonDirectory) + private static bool IsGitDirectory(string directory, [NotNullWhen(true)]out string? commonDirectory) { // HEAD file is required if (!File.Exists(Path.Combine(directory, GitHeadFileName))) diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepositoryLocation.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepositoryLocation.cs index 62d1eff3..6d1d96f3 100644 --- a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepositoryLocation.cs +++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepositoryLocation.cs @@ -1,6 +1,6 @@ // Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using System.Diagnostics; +using System; namespace Microsoft.Build.Tasks.Git { @@ -19,13 +19,13 @@ internal readonly struct GitRepositoryLocation /// /// Normalized full path. OS specific directory separators. Optional. /// - public readonly string WorkingDirectory { get; } + public readonly string? WorkingDirectory { get; } - internal GitRepositoryLocation(string gitDirectory, string commonDirectory, string workingDirectory) + internal GitRepositoryLocation(string gitDirectory, string commonDirectory, string? workingDirectory) { - Debug.Assert(PathUtils.IsNormalized(gitDirectory)); - Debug.Assert(PathUtils.IsNormalized(commonDirectory)); - Debug.Assert(workingDirectory == null || PathUtils.IsNormalized(workingDirectory)); + NullableDebug.Assert(PathUtils.IsNormalized(gitDirectory)); + NullableDebug.Assert(PathUtils.IsNormalized(commonDirectory)); + NullableDebug.Assert(workingDirectory == null || PathUtils.IsNormalized(workingDirectory)); GitDirectory = gitDirectory; CommonDirectory = commonDirectory; diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitSubmodule.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitSubmodule.cs index 4797564f..cba708d7 100644 --- a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitSubmodule.cs +++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitSubmodule.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using System.Diagnostics; +using System; +using System.Diagnostics.CodeAnalysis; namespace Microsoft.Build.Tasks.Git { @@ -27,14 +28,14 @@ internal readonly struct GitSubmodule /// /// Head tip commit SHA. Null, if there is no commit. /// - public string HeadCommitSha { get; } + public string? HeadCommitSha { get; } - internal GitSubmodule(string name, string workingDirectoryRelativePath, string workingDirectoryFullPath, string url, string headCommitSha) + internal GitSubmodule(string name, string workingDirectoryRelativePath, string workingDirectoryFullPath, string url, string? headCommitSha) { - Debug.Assert(name != null); - Debug.Assert(workingDirectoryRelativePath != null); - Debug.Assert(workingDirectoryFullPath != null); - Debug.Assert(url != null); + NullableDebug.Assert(name != null); + NullableDebug.Assert(workingDirectoryRelativePath != null); + NullableDebug.Assert(workingDirectoryFullPath != null); + NullableDebug.Assert(url != null); Name = name; WorkingDirectoryRelativePath = workingDirectoryRelativePath; diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitVariableName.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitVariableName.cs index e0d8b66e..f4df8a7a 100644 --- a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitVariableName.cs +++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitVariableName.cs @@ -17,9 +17,9 @@ namespace Microsoft.Build.Tasks.Git public GitVariableName(string sectionName, string subsectionName, string variableName) { - Debug.Assert(sectionName != null); - Debug.Assert(subsectionName != null); - Debug.Assert(variableName != null); + NullableDebug.Assert(sectionName != null); + NullableDebug.Assert(subsectionName != null); + NullableDebug.Assert(variableName != null); SectionName = sectionName; SubsectionName = subsectionName; diff --git a/src/Microsoft.Build.Tasks.Git/GitOperations.cs b/src/Microsoft.Build.Tasks.Git/GitOperations.cs index be7f3139..e21948a4 100644 --- a/src/Microsoft.Build.Tasks.Git/GitOperations.cs +++ b/src/Microsoft.Build.Tasks.Git/GitOperations.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; using Microsoft.Build.Framework; @@ -21,18 +22,20 @@ internal static class GitOperations private const string UrlSectionName = "url"; private const string UrlVariableName = "url"; - public static string GetRepositoryUrl(GitRepository repository, string remoteName, Action logWarning = null) + public static string? GetRepositoryUrl(GitRepository repository, string? remoteName, Action? logWarning = null) => GetRepositoryUrl(repository, remoteName, recursionDepth: 0, logWarning); - private static string GetRepositoryUrl(GitRepository repository, string remoteName, int recursionDepth, Action logWarning = null) + private static string? GetRepositoryUrl(GitRepository repository, string? remoteName, int recursionDepth, Action? logWarning = null) { + NullableDebug.Assert(repository.WorkingDirectory != null); + var remoteUrl = GetRemoteUrl(repository, ref remoteName, logWarning); if (remoteUrl == null) { return null; } - var uri = NormalizeUrl(repository.Config, remoteUrl, repository.WorkingDirectory); + var uri = NormalizeUrl(repository, remoteUrl); if (uri == null) { logWarning?.Invoke(Resources.InvalidRepositoryRemoteUrl, new[] { remoteName, remoteUrl }); @@ -42,11 +45,11 @@ private static string GetRepositoryUrl(GitRepository repository, string remoteNa return ResolveUrl(uri, repository.Environment, remoteName, recursionDepth, logWarning); } - private static string GetRemoteUrl(GitRepository repository, ref string remoteName, Action logWarning) + private static string? GetRemoteUrl(GitRepository repository, ref string? remoteName, Action? logWarning) { - string unknownRemoteName = null; - string remoteUrl = null; - if (!string.IsNullOrEmpty(remoteName)) + string? unknownRemoteName = null; + string? remoteUrl = null; + if (!NullableString.IsNullOrEmpty(remoteName)) { remoteUrl = repository.Config.GetVariableValue(RemoteSectionName, remoteName, UrlVariableName); if (remoteUrl == null) @@ -69,7 +72,7 @@ private static string GetRemoteUrl(GitRepository repository, ref string remoteNa return remoteUrl; } - private static string ResolveUrl(Uri uri, GitEnvironment environment, string remoteName, int recursionDepth, Action logWarning) + private static string? ResolveUrl(Uri uri, GitEnvironment environment, string? remoteName, int recursionDepth, Action? logWarning) { if (!uri.IsFile) { @@ -90,10 +93,16 @@ private static string ResolveUrl(Uri uri, GitEnvironment environment, string rem } var remoteRepository = GitRepository.OpenRepository(remoteRepositoryLocation, environment); + if (remoteRepository.WorkingDirectory == null) + { + logWarning?.Invoke(Resources.UnableToLocateRepository, new[] { repositoryPath }); + return null; + } + return GetRepositoryUrl(remoteRepository, remoteName, recursionDepth + 1, logWarning) ?? uri.AbsoluteUri; } - private static bool TryGetRemote(GitConfig config, out string remoteName, out string remoteUrl) + private static bool TryGetRemote(GitConfig config, [NotNullWhen(true)]out string? remoteName, [NotNullWhen(true)]out string? remoteUrl) { remoteName = RemoteOriginName; remoteUrl = config.GetVariableValue(RemoteSectionName, remoteName, UrlVariableName); @@ -125,7 +134,7 @@ internal static string ApplyInsteadOfUrlMapping(GitConfig config, string url) // - if the replacement is empty the URL is prefixed with the replacement string int longestPrefixLength = -1; - string replacement = null; + string? replacement = null; foreach (var variable in config.Variables) { @@ -146,10 +155,18 @@ internal static string ApplyInsteadOfUrlMapping(GitConfig config, string url) return (longestPrefixLength >= 0) ? replacement + url.Substring(longestPrefixLength) : url; } - internal static Uri NormalizeUrl(GitConfig config, string url, string root) - => NormalizeUrl(ApplyInsteadOfUrlMapping(config, url), root); + internal static Uri? NormalizeUrl(GitRepository repository, string url) + { + // Git (v2.23.0) treats local relative URLs as relative to the working directory. + // This doesn't work when a relative URL is used in a config file locatede in a main .git directory + // but is resolved from a worktree that has a different working directory. + // Currently we implement the same behavior as git. + + NullableDebug.Assert(repository.WorkingDirectory != null); + return NormalizeUrl(ApplyInsteadOfUrlMapping(repository.Config, url), root: repository.WorkingDirectory); + } - internal static Uri NormalizeUrl(string url, string root) + internal static Uri? NormalizeUrl(string url, string root) { // Since git supports scp-like syntax for SSH URLs we convert it here, // so that RepositoryUrl is actually a valid URL in that case. @@ -192,7 +209,7 @@ private static bool IsWindowsAbsoluteOrDriveRelativePath(string value) value[1] == ':' && (value[0] >= 'A' && value[0] <= 'Z' || value[0] >= 'a' && value[0] <= 'z'); - private static bool TryParseScp(string value, out Uri uri) + private static bool TryParseScp(string value, [NotNullWhen(true)]out Uri? uri) { uri = null; @@ -219,16 +236,19 @@ private static bool TryParseScp(string value, out Uri uri) return Uri.TryCreate(url, UriKind.Absolute, out uri); } - public static ITaskItem[] GetSourceRoots(GitRepository repository, string remoteName, Action logWarning) + public static ITaskItem[] GetSourceRoots(GitRepository repository, string? remoteName, Action logWarning) { + // Not supported for repositories without a working directory. + NullableDebug.Assert(repository.WorkingDirectory != null); + var result = new List(); - var repoRoot = GetRepositoryRoot(repository); + var repoRoot = repository.WorkingDirectory.EndWithSeparator(); var revisionId = repository.GetHeadCommitSha(); if (revisionId != null) { // Don't report a warning since it has already been reported by GetRepositoryUrl task. - string repositoryUrl = GetRepositoryUrl(repository, remoteName, logWarning: null); + string? repositoryUrl = GetRepositoryUrl(repository, remoteName, logWarning: null); // Item metadata are stored msbuild-escaped. GetMetadata unescapes, SetMetadata stores the value as specified. // Escape msbuild special characters so that URL escapes in the URL are preserved when the URL is read by GetMetadata. @@ -256,7 +276,7 @@ public static ITaskItem[] GetSourceRoots(GitRepository repository, string remote } // https://git-scm.com/docs/git-submodule - var submoduleUri = NormalizeUrl(repository.Config, submodule.Url, repoRoot); + var submoduleUri = NormalizeUrl(repository, submodule.Url); if (submoduleUri == null) { logWarning(Resources.SourceCodeWontBeAvailableViaSourceLink, @@ -295,31 +315,27 @@ public static ITaskItem[] GetSourceRoots(GitRepository repository, string remote return result.ToArray(); } - private static string GetRepositoryRoot(GitRepository repository) - => repository.WorkingDirectory.EndWithSeparator(); + public static ITaskItem[] GetUntrackedFiles(GitRepository repository, ITaskItem[] files, string projectDirectory) + => GetUntrackedFiles(repository, files, projectDirectory, CreateSubmoduleRepository); + + private static GitRepository? CreateSubmoduleRepository(GitEnvironment environment, string directoryFullPath) + => GitRepository.TryGetRepositoryLocation(directoryFullPath, out var location) ? + GitRepository.OpenRepository(location, environment) : null; - public static ITaskItem[] GetUntrackedFiles( - GitRepository repository, - ITaskItem[] files, - string projectDirectory, - Func repositoryFactory) + // internal for testing + internal static ITaskItem[] GetUntrackedFiles(GitRepository repository, ITaskItem[] files, string projectDirectory, Func repositoryFactory) { - var directoryTree = BuildDirectoryTree(repository); + var directoryTree = BuildDirectoryTree(repository, repositoryFactory); return files.Where(file => { // file.ItemSpec are relative to projectDirectory. var fullPath = Path.GetFullPath(Path.Combine(projectDirectory, file.ItemSpec)); - var containingDirectory = GetContainingRepository(fullPath, directoryTree); + var containingDirectoryMatcher = GetContainingRepositoryMatcher(fullPath, directoryTree); // Files that are outside of the repository are considered untracked. - if (containingDirectory == null) - { - return true; - } - - return containingDirectory.GetMatcher(repositoryFactory).IsNormalizedFilePathIgnored(fullPath) ?? true; + return containingDirectoryMatcher?.IsNormalizedFilePathIgnored(fullPath) ?? true; }).ToArray(); } @@ -329,55 +345,37 @@ internal sealed class DirectoryNode public readonly List OrderedChildren; // set on nodes that represent submodule working directory: - public string WorkingDirectoryFullPath; - private GitIgnore.Matcher _lazyMatcher; + public Lazy? Matcher; - public DirectoryNode(string name) - : this(name, null, new List()) - { - } - - public DirectoryNode(string name, string fullPath) - : this(name, fullPath, new List()) - { - } - - public DirectoryNode(string name, string workingDirectoryFullPath, List orderedChildren) + public DirectoryNode(string name, List orderedChildren) { Name = name; - WorkingDirectoryFullPath = workingDirectoryFullPath; OrderedChildren = orderedChildren; } - public void SetMatcher(string workingDirectory, GitIgnore.Matcher matcher) - { - WorkingDirectoryFullPath = workingDirectory; - _lazyMatcher = matcher; - } - public int FindChildIndex(string name) => BinarySearch(OrderedChildren, name, (n, v) => n.Name.CompareTo(v)); - - public GitIgnore.Matcher GetMatcher(Func repositoryFactory) - => _lazyMatcher ?? (_lazyMatcher = repositoryFactory(WorkingDirectoryFullPath).Ignore.CreateMatcher()); } - internal static DirectoryNode BuildDirectoryTree(GitRepository repository) + internal static DirectoryNode BuildDirectoryTree(GitRepository repository, Func repositoryFactory) { - var workingDirectory = repository.WorkingDirectory; + NullableDebug.Assert(repository.WorkingDirectory != null); - var treeRoot = new DirectoryNode(""); - AddTreeNode(treeRoot, workingDirectory, repository.Ignore.CreateMatcher()); + var treeRoot = new DirectoryNode(name: "", new List()); + AddTreeNode(treeRoot, repository.WorkingDirectory, new Lazy(() => repository.Ignore.CreateMatcher())); foreach (var submodule in repository.GetSubmodules()) { - AddTreeNode(treeRoot, submodule.WorkingDirectoryFullPath, matcherOpt: null); + var submoduleWorkingDirectory = submodule.WorkingDirectoryFullPath; + + AddTreeNode(treeRoot, submoduleWorkingDirectory, + new Lazy(() => repositoryFactory(repository.Environment, submoduleWorkingDirectory)?.Ignore.CreateMatcher())); } return treeRoot; } - private static void AddTreeNode(DirectoryNode root, string workingDirectory, GitIgnore.Matcher matcherOpt) + private static void AddTreeNode(DirectoryNode root, string workingDirectory, Lazy matcher) { var segments = PathUtilities.Split(workingDirectory); @@ -392,26 +390,26 @@ private static void AddTreeNode(DirectoryNode root, string workingDirectory, Git } else { - var newNode = new DirectoryNode(segments[i]); + var newNode = new DirectoryNode(segments[i], new List()); node.OrderedChildren.Insert(~index, newNode); node = newNode; } if (i == segments.Length - 1) { - node.SetMatcher(workingDirectory, matcherOpt); + node.Matcher = matcher; } } } // internal for testing - internal static DirectoryNode GetContainingRepository(string fullPath, DirectoryNode root) + internal static GitIgnore.Matcher? GetContainingRepositoryMatcher(string fullPath, DirectoryNode root) { var segments = PathUtilities.Split(fullPath); Debug.Assert(segments.Length > 0); - Debug.Assert(root.WorkingDirectoryFullPath == null); - DirectoryNode containingRepositoryNode = null; + Debug.Assert(root.Matcher == null); + GitIgnore.Matcher? containingRepositoryMatcher = null; var node = root; for (int i = 0; i < segments.Length - 1; i++) @@ -423,13 +421,16 @@ internal static DirectoryNode GetContainingRepository(string fullPath, Directory } node = node.OrderedChildren[index]; - if (node.WorkingDirectoryFullPath != null) + + var matcher = node.Matcher?.Value; + if (matcher != null) { - containingRepositoryNode = node; + // inner-most repository determines the ignore state of the file: + containingRepositoryMatcher = matcher; } } - return containingRepositoryNode; + return containingRepositoryMatcher; } internal static int BinarySearch(IReadOnlyList list, TValue value, Func compare) diff --git a/src/Microsoft.Build.Tasks.Git/LocateRepository.cs b/src/Microsoft.Build.Tasks.Git/LocateRepository.cs index ef40f708..ac6bfb79 100644 --- a/src/Microsoft.Build.Tasks.Git/LocateRepository.cs +++ b/src/Microsoft.Build.Tasks.Git/LocateRepository.cs @@ -1,24 +1,26 @@ // Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; +using System.Diagnostics.CodeAnalysis; using Microsoft.Build.Framework; namespace Microsoft.Build.Tasks.Git { public sealed class LocateRepository : RepositoryTask { - public string RemoteName { get; set; } + public string? RemoteName { get; set; } - [Required] - public string Path { get; set; } + [Required, NotNull] + public string? Path { get; set; } [Output] - public string RepositoryId { get; private set; } + public string? RepositoryId { get; private set; } [Output] - public string WorkingDirectory { get; private set; } + public string? WorkingDirectory { get; private set; } [Output] - public string Url { get; private set; } + public string? Url { get; private set; } /// /// Returns items describing repository source roots: @@ -32,19 +34,21 @@ public sealed class LocateRepository : RepositoryTask /// NestedRoot: For a submodule root, a path of the submodule root relative to the repository root. Ends with a slash. /// [Output] - public ITaskItem[] Roots { get; private set; } + public ITaskItem[]? Roots { get; private set; } /// /// Head tip commit SHA. /// [Output] - public string RevisionId { get; private set; } + public string? RevisionId { get; private set; } - protected override string GetRepositoryId() => null; - protected override string GetInitialPath() => Path; + protected override string? GetRepositoryId() => null; + protected override string GetInitialPath() => Path!; private protected override void Execute(GitRepository repository) { + NullableDebug.Assert(repository.WorkingDirectory != null); + RepositoryId = repository.GitDirectory; WorkingDirectory = repository.WorkingDirectory; Url = GitOperations.GetRepositoryUrl(repository, RemoteName, Log.LogWarning); diff --git a/src/Microsoft.Build.Tasks.Git/Microsoft.Build.Tasks.Git.csproj b/src/Microsoft.Build.Tasks.Git/Microsoft.Build.Tasks.Git.csproj index bb6c3b3f..e79a425e 100644 --- a/src/Microsoft.Build.Tasks.Git/Microsoft.Build.Tasks.Git.csproj +++ b/src/Microsoft.Build.Tasks.Git/Microsoft.Build.Tasks.Git.csproj @@ -12,6 +12,7 @@ $(TargetsForTfmSpecificContentInPackage);_GetTfmSpecificFilesToPackage + diff --git a/src/Microsoft.Build.Tasks.Git/RepositoryTask.cs b/src/Microsoft.Build.Tasks.Git/RepositoryTask.cs index 98a09fd4..e4b1a000 100644 --- a/src/Microsoft.Build.Tasks.Git/RepositoryTask.cs +++ b/src/Microsoft.Build.Tasks.Git/RepositoryTask.cs @@ -1,10 +1,10 @@ // Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.Runtime.CompilerServices; using Microsoft.Build.Framework; -using Microsoft.Build.Tasks.SourceControl; using Microsoft.Build.Utilities; namespace Microsoft.Build.Tasks.Git @@ -21,7 +21,7 @@ public abstract class RepositoryTask : Task /// Supported values: /// If "local" is specified the configuration is only read from the configuration files local to the repository (or work tree). /// - public string ConfigurationScope { get; set; } + public string? ConfigurationScope { get; set; } #if NET461 static RepositoryTask() => AssemblyResolver.Initialize(); @@ -55,10 +55,9 @@ bool logAssemblyLoadingErrors() private protected abstract void Execute(GitRepository repository); - protected abstract string GetRepositoryId(); + protected abstract string? GetRepositoryId(); protected abstract string GetInitialPath(); - [MethodImpl(MethodImplOptions.NoInlining)] private void ExecuteImpl() { try @@ -78,9 +77,9 @@ private void ExecuteImpl() } } - private GitRepository GetOrCreateRepositoryInstance() + private GitRepository? GetOrCreateRepositoryInstance() { - GitRepository repository; + GitRepository? repository; var repositoryId = GetRepositoryId(); if (repositoryId != null) @@ -131,9 +130,9 @@ private GitRepository GetOrCreateRepositoryInstance() private string GetCacheKey(string repositoryId) => s_cacheKeyPrefix + (string.IsNullOrEmpty(ConfigurationScope) ? "*" : ConfigurationScope) + ":" + repositoryId; - private bool TryGetCachedRepositoryInstance(string cacheKey, bool requireCached, out GitRepository repository) + private bool TryGetCachedRepositoryInstance(string cacheKey, bool requireCached, [NotNullWhen(true)]out GitRepository? repository) { - var entry = (StrongBox)BuildEngine4.GetRegisteredTaskObject(cacheKey, RegisteredTaskObjectLifetime.Build); + var entry = (StrongBox)BuildEngine4.GetRegisteredTaskObject(cacheKey, RegisteredTaskObjectLifetime.Build); if (entry != null) { @@ -151,11 +150,11 @@ private bool TryGetCachedRepositoryInstance(string cacheKey, bool requireCached, return false; } - private void CacheRepositoryInstance(string cacheKey, GitRepository repository) + private void CacheRepositoryInstance(string cacheKey, GitRepository? repository) { BuildEngine4.RegisterTaskObject( cacheKey, - new StrongBox(repository), + new StrongBox(repository), RegisteredTaskObjectLifetime.Build, allowEarlyCollection: true); } diff --git a/src/Microsoft.Build.Tasks.Tfvc/GetRepositoryUrl.cs b/src/Microsoft.Build.Tasks.Tfvc/GetRepositoryUrl.cs index 622904d1..3f476dca 100644 --- a/src/Microsoft.Build.Tasks.Tfvc/GetRepositoryUrl.cs +++ b/src/Microsoft.Build.Tasks.Tfvc/GetRepositoryUrl.cs @@ -13,22 +13,21 @@ namespace Microsoft.Build.Tasks.Tfvc public sealed class GetRepositoryUrl : RepositoryTask { [Output] - public string Url { get; private set; } + public string? Url { get; private set; } protected override bool Execute(WorkspaceInfo workspaceInfo) { - using (var collection = new TfsTeamProjectCollection(workspaceInfo.ServerUri)) - { - var workspace = workspaceInfo.GetWorkspace(collection); + using var collection = new TfsTeamProjectCollection(workspaceInfo.ServerUri); - // Use the first project: - var project = workspace.GetTeamProjectForLocalPath(workspaceInfo.MappedPaths.First()); + var workspace = workspaceInfo.GetWorkspace(collection); - // Extract GUID from ArtifactUri "vstfs:///Classification/TeamProject/{Guid}": - var projectId = Path.GetFileName(project.ArtifactUri.GetPath()); + // Use the first project: + var project = workspace.GetTeamProjectForLocalPath(workspaceInfo.MappedPaths.First()); - Url = collection.Uri.ToString() + "/" + projectId; - } + // Extract GUID from ArtifactUri "vstfs:///Classification/TeamProject/{Guid}": + var projectId = Path.GetFileName(project.ArtifactUri.GetPath()); + + Url = collection.Uri.ToString() + "/" + projectId; return true; } diff --git a/src/Microsoft.Build.Tasks.Tfvc/GetSourceRevisionId.cs b/src/Microsoft.Build.Tasks.Tfvc/GetSourceRevisionId.cs index b83726d0..253ae8e9 100644 --- a/src/Microsoft.Build.Tasks.Tfvc/GetSourceRevisionId.cs +++ b/src/Microsoft.Build.Tasks.Tfvc/GetSourceRevisionId.cs @@ -9,15 +9,14 @@ namespace Microsoft.Build.Tasks.Tfvc public sealed class GetSourceRevisionId : RepositoryTask { [Output] - public string RevisionId { get; private set; } + public string? RevisionId { get; private set; } protected override bool Execute(WorkspaceInfo workspaceInfo) { - using (var collection = new TfsTeamProjectCollection(workspaceInfo.ServerUri)) - { - var vcServer = collection.GetService(); - RevisionId = vcServer.GetLatestChangesetId().ToString(); - } + using var collection = new TfsTeamProjectCollection(workspaceInfo.ServerUri); + + var vcServer = collection.GetService(); + RevisionId = vcServer.GetLatestChangesetId().ToString(); return true; } diff --git a/src/Microsoft.Build.Tasks.Tfvc/GetSourceRoots.cs b/src/Microsoft.Build.Tasks.Tfvc/GetSourceRoots.cs index 04ffeea0..b35f74c6 100644 --- a/src/Microsoft.Build.Tasks.Tfvc/GetSourceRoots.cs +++ b/src/Microsoft.Build.Tasks.Tfvc/GetSourceRoots.cs @@ -25,47 +25,46 @@ public sealed class GetSourceRoots : RepositoryTask /// RevisionId: Revision (changeset number). /// [Output] - public ITaskItem[] Mapping { get; private set; } + public ITaskItem[]? Mapping { get; private set; } protected override bool Execute(WorkspaceInfo workspaceInfo) { var result = new List(); - using (var collection = new TfsTeamProjectCollection(workspaceInfo.ServerUri)) - { - var vcServer = collection.GetService(); - var changesetId = vcServer.GetLatestChangesetId().ToString(); + using var collection = new TfsTeamProjectCollection(workspaceInfo.ServerUri); + + var vcServer = collection.GetService(); + var changesetId = vcServer.GetLatestChangesetId().ToString(); - var workspace = workspaceInfo.GetWorkspace(collection); - var collectionUrl = collection.Uri.ToString(); + var workspace = workspaceInfo.GetWorkspace(collection); + var collectionUrl = collection.Uri.ToString(); - // TODO: eliminate redundant mappings - we can use RepositoryRoot calculation here - // E.g. A\B -> $/X/A/B, A\C -> $/X/A/C can be reduced to A -> $/X/A + // TODO: eliminate redundant mappings - we can use RepositoryRoot calculation here + // E.g. A\B -> $/X/A/B, A\C -> $/X/A/C can be reduced to A -> $/X/A - foreach (var folder in workspace.Folders) + foreach (var folder in workspace.Folders) + { + if (!folder.IsCloaked) { - if (!folder.IsCloaked) - { - var project = workspace.GetTeamProjectForLocalPath(folder.LocalItem); + var project = workspace.GetTeamProjectForLocalPath(folder.LocalItem); - // Extract GUID from ArtifactUri "vstfs:///Classification/TeamProject/{Guid}": - var projectId = Path.GetFileName(project.ArtifactUri.GetPath()); + // Extract GUID from ArtifactUri "vstfs:///Classification/TeamProject/{Guid}": + var projectId = Path.GetFileName(project.ArtifactUri.GetPath()); - // SourceLink.AzureRepos will map each source root to: - // {RepositoryUrl}/_versionControl?path={ServerPath}&version={RevisionId} - var item = new TaskItem(folder.LocalItem); - item.SetMetadata("SourceControl", "tfvc"); - item.SetMetadata("CollectionUrl", collectionUrl); - item.SetMetadata("ProjectId", projectId); - item.SetMetadata("ServerPath", folder.ServerItem); - item.SetMetadata("RevisionId", changesetId); - result.Add(item); - } + // SourceLink.AzureRepos will map each source root to: + // {RepositoryUrl}/_versionControl?path={ServerPath}&version={RevisionId} + var item = new TaskItem(folder.LocalItem); + item.SetMetadata("SourceControl", "tfvc"); + item.SetMetadata("CollectionUrl", collectionUrl); + item.SetMetadata("ProjectId", projectId); + item.SetMetadata("ServerPath", folder.ServerItem); + item.SetMetadata("RevisionId", changesetId); + result.Add(item); } - - Mapping = result.ToArray(); - return true; } + + Mapping = result.ToArray(); + return true; } } } diff --git a/src/Microsoft.Build.Tasks.Tfvc/GetUntrackedSourceFiles.cs b/src/Microsoft.Build.Tasks.Tfvc/GetUntrackedSourceFiles.cs index e1c1ed06..21469d67 100644 --- a/src/Microsoft.Build.Tasks.Tfvc/GetUntrackedSourceFiles.cs +++ b/src/Microsoft.Build.Tasks.Tfvc/GetUntrackedSourceFiles.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using Microsoft.Build.Framework; using Microsoft.TeamFoundation.Client; using Microsoft.TeamFoundation.Common; @@ -10,33 +12,32 @@ namespace Microsoft.Build.Tasks.Tfvc { public sealed class GetUntrackedSourceFiles : RepositoryTask { - [Required] - public ITaskItem[] SourceFiles { get; set; } + [Required, NotNull] + public ITaskItem[]? SourceFiles { get; set; } [Output] - public ITaskItem[] UntrackedFiles { get; set; } + public ITaskItem[]? UntrackedFiles { get; set; } protected override bool Execute(WorkspaceInfo workspaceInfo) { - using (var collection = new TfsTeamProjectCollection(workspaceInfo.ServerUri)) - { - var vcServer = collection.GetService(); - var workspace = vcServer.GetWorkspace(workspaceInfo); - var evaluator = new LocalItemExclusionEvaluator(workspace, startLocalItem: ""); // TODO? + using var collection = new TfsTeamProjectCollection(workspaceInfo.ServerUri); + + var vcServer = collection.GetService(); + var workspace = vcServer.GetWorkspace(workspaceInfo); + var evaluator = new LocalItemExclusionEvaluator(workspace, startLocalItem: ""); // TODO? - var result = new List(); - foreach (var item in SourceFiles) + var result = new List(); + foreach (var item in SourceFiles) + { + if (FileSpec.IsSubItem(item.ItemSpec, evaluator.StartLocalItem) && + evaluator.IsExcluded(item.ItemSpec, isFolder: false)) { - if (FileSpec.IsSubItem(item.ItemSpec, evaluator.StartLocalItem) && - evaluator.IsExcluded(item.ItemSpec, isFolder: false)) - { - result.Add(item); - } + result.Add(item); } - - UntrackedFiles = result.ToArray(); } + UntrackedFiles = result.ToArray(); + return true; } } diff --git a/src/Microsoft.Build.Tasks.Tfvc/LocateRepository.cs b/src/Microsoft.Build.Tasks.Tfvc/LocateRepository.cs index f509b5ef..1c44ee09 100644 --- a/src/Microsoft.Build.Tasks.Tfvc/LocateRepository.cs +++ b/src/Microsoft.Build.Tasks.Tfvc/LocateRepository.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System.Diagnostics.CodeAnalysis; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; using Microsoft.TeamFoundation.VersionControl.Client; @@ -8,11 +9,11 @@ namespace Microsoft.Build.Tasks.Tfvc { public class LocateRepository : Task { - [Required] - public string Directory { get; set; } + [Required, NotNull] + public string? Directory { get; set; } [Output] - public string Id { get; set; } + public string? Id { get; set; } #if UNUSED [Output] diff --git a/src/Microsoft.Build.Tasks.Tfvc/Microsoft.Build.Tasks.Tfvc.csproj b/src/Microsoft.Build.Tasks.Tfvc/Microsoft.Build.Tasks.Tfvc.csproj index 2bc9ee25..567d9f56 100644 --- a/src/Microsoft.Build.Tasks.Tfvc/Microsoft.Build.Tasks.Tfvc.csproj +++ b/src/Microsoft.Build.Tasks.Tfvc/Microsoft.Build.Tasks.Tfvc.csproj @@ -14,6 +14,7 @@ $(TargetsForTfmSpecificContentInPackage);_GetTfmSpecificFilesToPackage + diff --git a/src/Microsoft.Build.Tasks.Tfvc/RepositoryTask.cs b/src/Microsoft.Build.Tasks.Tfvc/RepositoryTask.cs index d855a03b..385cf5cd 100644 --- a/src/Microsoft.Build.Tasks.Tfvc/RepositoryTask.cs +++ b/src/Microsoft.Build.Tasks.Tfvc/RepositoryTask.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System.Diagnostics.CodeAnalysis; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; using Microsoft.TeamFoundation.VersionControl.Client; @@ -8,8 +9,8 @@ namespace Microsoft.Build.Tasks.Tfvc { public abstract class RepositoryTask : Task { - [Required] - public string WorkspaceDirectory { get; set; } + [Required, NotNull] + public string? WorkspaceDirectory { get; set; } protected abstract bool Execute(WorkspaceInfo workspaceInfo); diff --git a/src/SourceLink.AzureDevOpsServer.Git/GetSourceLinkUrl.cs b/src/SourceLink.AzureDevOpsServer.Git/GetSourceLinkUrl.cs index 50f3f06f..5a6d516e 100644 --- a/src/SourceLink.AzureDevOpsServer.Git/GetSourceLinkUrl.cs +++ b/src/SourceLink.AzureDevOpsServer.Git/GetSourceLinkUrl.cs @@ -19,9 +19,9 @@ public sealed class GetSourceLinkUrl : GetSourceLinkUrlGitTask /// protected override bool SupportsImplicitHost => false; - protected override string BuildSourceLinkUrl(Uri contentUri, Uri gitUri, string relativeUrl, string revisionId, ITaskItem hostItem) + protected override string? BuildSourceLinkUrl(Uri contentUri, Uri gitUri, string relativeUrl, string revisionId, ITaskItem? hostItem) { - var virtualDirectory = hostItem.GetMetadata(VirtualDirectoryMetadataName); + var virtualDirectory = hostItem?.GetMetadata(VirtualDirectoryMetadataName) ?? ""; if (!AzureDevOpsUrlParser.TryParseOnPremHttp(relativeUrl, virtualDirectory, out var projectPath, out var repositoryName)) { Log.LogError(CommonResources.ValueOfWithIdentityIsInvalid, Names.SourceRoot.RepositoryUrlFullName, SourceRoot.ItemSpec, gitUri); diff --git a/src/SourceLink.AzureDevOpsServer.Git/Microsoft.SourceLink.AzureDevOpsServer.Git.csproj b/src/SourceLink.AzureDevOpsServer.Git/Microsoft.SourceLink.AzureDevOpsServer.Git.csproj index e294f3f2..40abab53 100644 --- a/src/SourceLink.AzureDevOpsServer.Git/Microsoft.SourceLink.AzureDevOpsServer.Git.csproj +++ b/src/SourceLink.AzureDevOpsServer.Git/Microsoft.SourceLink.AzureDevOpsServer.Git.csproj @@ -13,6 +13,7 @@ true + diff --git a/src/SourceLink.AzureDevOpsServer.Git/TranslateRepositoryUrls.cs b/src/SourceLink.AzureDevOpsServer.Git/TranslateRepositoryUrls.cs index 58fd1c00..9b12b949 100644 --- a/src/SourceLink.AzureDevOpsServer.Git/TranslateRepositoryUrls.cs +++ b/src/SourceLink.AzureDevOpsServer.Git/TranslateRepositoryUrls.cs @@ -11,7 +11,7 @@ public sealed class TranslateRepositoryUrls : TranslateRepositoryUrlsGitTask // ssh://{account}@{domain}:{port}/{repositoryPath}/_ssh/{"_full"|"_optimized"}/{repositoryName} // to // https://{domain}/{repositoryPath}/_git/{repositoryName} - protected override string TranslateSshUrl(Uri uri) + protected override string? TranslateSshUrl(Uri uri) { if (!AzureDevOpsUrlParser.TryParseOnPremSsh(uri, out var repositoryPath, out var repositoryName)) { diff --git a/src/SourceLink.AzureRepos.Git/GetSourceLinkUrl.cs b/src/SourceLink.AzureRepos.Git/GetSourceLinkUrl.cs index 7f6f4035..5a74a384 100644 --- a/src/SourceLink.AzureRepos.Git/GetSourceLinkUrl.cs +++ b/src/SourceLink.AzureRepos.Git/GetSourceLinkUrl.cs @@ -28,7 +28,7 @@ protected override Uri GetDefaultContentUriFromHostUri(string authority, Uri git protected override Uri GetDefaultContentUriFromRepositoryUri(Uri repositoryUri) => new Uri($"{repositoryUri.Scheme}://{repositoryUri.GetAuthority()}", UriKind.Absolute); - protected override string BuildSourceLinkUrl(Uri contentUri, Uri gitUri, string relativeUrl, string revisionId, ITaskItem hostItem) + protected override string? BuildSourceLinkUrl(Uri contentUri, Uri gitUri, string relativeUrl, string revisionId, ITaskItem? hostItem) { if (!AzureDevOpsUrlParser.TryParseHostedHttp(gitUri.GetHost(), relativeUrl, out var projectPath, out var repositoryName)) { @@ -43,10 +43,10 @@ protected override string BuildSourceLinkUrl(Uri contentUri, Uri gitUri, string } // TODO: confirm design and test https://github.com/dotnet/sourcelink/issues/2 - private Dictionary TryGetEnvironmentUriMap() + private Dictionary? TryGetEnvironmentUriMap() { var urlSeparators = new[] { Path.PathSeparator }; - Dictionary map = null; + Dictionary? map = null; bool parse(string urls) { diff --git a/src/SourceLink.AzureRepos.Git/Microsoft.SourceLink.AzureRepos.Git.csproj b/src/SourceLink.AzureRepos.Git/Microsoft.SourceLink.AzureRepos.Git.csproj index bc5c1521..5ea7d293 100644 --- a/src/SourceLink.AzureRepos.Git/Microsoft.SourceLink.AzureRepos.Git.csproj +++ b/src/SourceLink.AzureRepos.Git/Microsoft.SourceLink.AzureRepos.Git.csproj @@ -13,6 +13,7 @@ true + diff --git a/src/SourceLink.AzureRepos.Git/TranslateRepositoryUrls.cs b/src/SourceLink.AzureRepos.Git/TranslateRepositoryUrls.cs index 7085a94d..73452eec 100644 --- a/src/SourceLink.AzureRepos.Git/TranslateRepositoryUrls.cs +++ b/src/SourceLink.AzureRepos.Git/TranslateRepositoryUrls.cs @@ -15,7 +15,7 @@ public sealed class TranslateRepositoryUrls : TranslateRepositoryUrlsGitTask // Dommain mapping: // ssh://vs-ssh.*.com -> https://{account}.*.com // ssh://ssh.*.com -> https://*.com/{account} - protected override string TranslateSshUrl(Uri uri) + protected override string? TranslateSshUrl(Uri uri) { var host = uri.GetHost(); var isVisualStudioHost = AzureDevOpsUrlParser.IsVisualStudioHostedServer(host); diff --git a/src/SourceLink.AzureRepos.Tfvc/GetSourceLinkUrl.cs b/src/SourceLink.AzureRepos.Tfvc/GetSourceLinkUrl.cs index fdbb4b93..59794c42 100644 --- a/src/SourceLink.AzureRepos.Tfvc/GetSourceLinkUrl.cs +++ b/src/SourceLink.AzureRepos.Tfvc/GetSourceLinkUrl.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; using Microsoft.Build.Framework; @@ -10,11 +11,11 @@ namespace Microsoft.SourceLink.AzureRepos.Tfvc { public sealed class GetSourceLinkUrl : Task { - [Required] - public ITaskItem SourceRoot { get; set; } + [Required, NotNull] + public ITaskItem? SourceRoot { get; set; } [Output] - public string SourceLinkUrl { get; set; } + public string? SourceLinkUrl { get; set; } public override bool Execute() { diff --git a/src/SourceLink.AzureRepos.Tfvc/Microsoft.SourceLink.AzureRepos.Tfvc.csproj b/src/SourceLink.AzureRepos.Tfvc/Microsoft.SourceLink.AzureRepos.Tfvc.csproj index 4098db84..4c2d6cc8 100644 --- a/src/SourceLink.AzureRepos.Tfvc/Microsoft.SourceLink.AzureRepos.Tfvc.csproj +++ b/src/SourceLink.AzureRepos.Tfvc/Microsoft.SourceLink.AzureRepos.Tfvc.csproj @@ -14,6 +14,9 @@ MSBuild Tasks Azure DevOps Repos VSTS TFVC source link true + + + diff --git a/src/SourceLink.Bitbucket.Git/GetSourceLinkUrl.cs b/src/SourceLink.Bitbucket.Git/GetSourceLinkUrl.cs index 3a5673ff..66c66349 100644 --- a/src/SourceLink.Bitbucket.Git/GetSourceLinkUrl.cs +++ b/src/SourceLink.Bitbucket.Git/GetSourceLinkUrl.cs @@ -2,6 +2,7 @@ using System; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using Microsoft.Build.Framework; using Microsoft.Build.Tasks.SourceControl; @@ -21,7 +22,7 @@ public sealed class GetSourceLinkUrl : GetSourceLinkUrlGitTask private const string VersionMetadataName = "Version"; private static readonly Version s_versionWithNewUrlFormat = new Version(4, 7); - protected override string BuildSourceLinkUrl(Uri contentUri, Uri gitUri, string relativeUrl, string revisionId, ITaskItem hostItem) + protected override string? BuildSourceLinkUrl(Uri contentUri, Uri gitUri, string relativeUrl, string revisionId, ITaskItem? hostItem) { // The SourceLinkBitbucketGitHost item for bitbucket.org specifies EnterpriseEdition="false". // Other items that may be specified by the project default to EnterpriseEdition="true" without specifying it. @@ -57,7 +58,7 @@ internal static string BuildSourceLinkUrlForEnterpriseEdition( return UriUtilities.Combine(contentUri.ToString(), UriUtilities.Combine(relativeBaseUrl, relativeUrl)); } - internal static bool TryParseEnterpriseUrl(string relativeUrl, out string relativeBaseUrl, out string projectName, out string repositoryName) + internal static bool TryParseEnterpriseUrl(string relativeUrl, [NotNullWhen(true)]out string? relativeBaseUrl, [NotNullWhen(true)]out string? projectName, [NotNullWhen(true)]out string? repositoryName) { // HTTP: {baseUrl}/scm/{projectName}/{repositoryName} // SSH: {baseUrl}/{projectName}/{repositoryName} @@ -83,26 +84,21 @@ internal static bool TryParseEnterpriseUrl(string relativeUrl, out string relati return true; } - private Version GetBitbucketEnterpriseVersion(ITaskItem hostItem) + private Version GetBitbucketEnterpriseVersion(ITaskItem? hostItem) { var bitbucketEnterpriseVersionAsString = hostItem?.GetMetadata(VersionMetadataName); - Version bitbucketEnterpriseVersion; - if (!string.IsNullOrEmpty(bitbucketEnterpriseVersionAsString)) + if (!NullableString.IsNullOrEmpty(bitbucketEnterpriseVersionAsString)) { - if (!Version.TryParse(bitbucketEnterpriseVersionAsString, out bitbucketEnterpriseVersion)) + if (Version.TryParse(bitbucketEnterpriseVersionAsString, out var version)) { - Log.LogError(CommonResources.ItemOfItemGroupMustSpecifyMetadata, hostItem.ItemSpec, - HostsItemGroupName, VersionMetadataName); - - return null; + return version; } - } - else - { - bitbucketEnterpriseVersion = s_versionWithNewUrlFormat; - } - return bitbucketEnterpriseVersion; + Log.LogError(CommonResources.ItemOfItemGroupMustSpecifyMetadata, hostItem!.ItemSpec, + HostsItemGroupName, VersionMetadataName); + } + + return s_versionWithNewUrlFormat; } private static string BuildSourceLinkUrlForCloudEdition(Uri contentUri, string relativeUrl, string revisionId) diff --git a/src/SourceLink.Bitbucket.Git/Microsoft.SourceLink.Bitbucket.Git.csproj b/src/SourceLink.Bitbucket.Git/Microsoft.SourceLink.Bitbucket.Git.csproj index 7abd791f..621598ef 100644 --- a/src/SourceLink.Bitbucket.Git/Microsoft.SourceLink.Bitbucket.Git.csproj +++ b/src/SourceLink.Bitbucket.Git/Microsoft.SourceLink.Bitbucket.Git.csproj @@ -13,6 +13,7 @@ true + diff --git a/src/SourceLink.Common.UnitTests/Mocks/MockGetSourceLinkUrlGitTask.cs b/src/SourceLink.Common.UnitTests/Mocks/MockGetSourceLinkUrlGitTask.cs index 2ad072c1..34e1e869 100644 --- a/src/SourceLink.Common.UnitTests/Mocks/MockGetSourceLinkUrlGitTask.cs +++ b/src/SourceLink.Common.UnitTests/Mocks/MockGetSourceLinkUrlGitTask.cs @@ -16,7 +16,7 @@ protected override string ProviderDisplayName protected override string HostsItemGroupName => "SourceLinkMockHost"; - protected override string BuildSourceLinkUrl(Uri contentUrl, Uri gitUri, string relativeUrl, string revisionId, ITaskItem hostItem) + protected override string? BuildSourceLinkUrl(Uri contentUrl, Uri gitUri, string relativeUrl, string revisionId, ITaskItem? hostItem) => $"ContentUrl='{contentUrl}' GitUrl='{gitUri}' RelativeUrl='{relativeUrl}' RevisionId='{revisionId}'"; protected override Uri GetDefaultContentUriFromHostUri(string authority, Uri gitUri) diff --git a/src/SourceLink.Common.UnitTests/UriUtilitiesTests.cs b/src/SourceLink.Common.UnitTests/UriUtilitiesTests.cs index f40ef6cf..9ab1dbb2 100644 --- a/src/SourceLink.Common.UnitTests/UriUtilitiesTests.cs +++ b/src/SourceLink.Common.UnitTests/UriUtilitiesTests.cs @@ -38,9 +38,9 @@ public void UrlStartsWith(string url, string prefix, bool expected) [InlineData("/a/", new[] { "a" })] [InlineData("/a//b/", null)] [InlineData("/a/b//", null)] - public void TrySplitRelativeUrl(string url, string[] parts) + public void TrySplitRelativeUrl(string url, string[]? parts) { - if (!UriUtilities.TrySplitRelativeUrl(url, out var actualParts)) + if (!UriUtilities.TrySplitRelativeUrl(url, out string[]? actualParts)) { actualParts = null; } diff --git a/src/SourceLink.Common/GenerateSourceLinkFile.cs b/src/SourceLink.Common/GenerateSourceLinkFile.cs index f269f800..b3d763b2 100644 --- a/src/SourceLink.Common/GenerateSourceLinkFile.cs +++ b/src/SourceLink.Common/GenerateSourceLinkFile.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; using System.Text; @@ -12,11 +13,11 @@ namespace Microsoft.SourceLink.Common { public sealed class GenerateSourceLinkFile : Task { - [Required] - public ITaskItem[] SourceRoots { get; set; } + [Required, NotNull] + public ITaskItem[]? SourceRoots { get; set; } - [Required] - public string OutputFile { get; set; } + [Required, NotNull] + public string? OutputFile { get; set; } public override bool Execute() { @@ -29,9 +30,9 @@ public override bool Execute() return !Log.HasLoggedErrors; } - internal string GenerateSourceLinkContent() + internal string? GenerateSourceLinkContent() { - string JsonEscape(string str) + static string jsonEscape(string str) => str.Replace(@"\", @"\\").Replace("\"", "\\\""); var result = new StringBuilder(); @@ -84,12 +85,12 @@ string JsonEscape(string str) } result.Append('"'); - result.Append(JsonEscape(localPath)); + result.Append(jsonEscape(localPath)); result.Append('*'); result.Append('"'); result.Append(':'); result.Append('"'); - result.Append(JsonEscape(url)); + result.Append(jsonEscape(url)); result.Append('"'); } diff --git a/src/SourceLink.Common/Microsoft.SourceLink.Common.csproj b/src/SourceLink.Common/Microsoft.SourceLink.Common.csproj index 11730b81..efc214ab 100644 --- a/src/SourceLink.Common/Microsoft.SourceLink.Common.csproj +++ b/src/SourceLink.Common/Microsoft.SourceLink.Common.csproj @@ -10,6 +10,7 @@ true + diff --git a/src/SourceLink.Common/SourceLinkHasSingleProvider.cs b/src/SourceLink.Common/SourceLinkHasSingleProvider.cs index 9305b71c..c08290e0 100644 --- a/src/SourceLink.Common/SourceLinkHasSingleProvider.cs +++ b/src/SourceLink.Common/SourceLinkHasSingleProvider.cs @@ -8,7 +8,7 @@ namespace Microsoft.SourceLink.Common { public sealed class SourceLinkHasSingleProvider : Task { - public string ProviderTargets { get; set; } + public string? ProviderTargets { get; set; } [Output] public bool HasSingleProvider { get; set; } diff --git a/src/SourceLink.GitHub/GetSourceLinkUrl.cs b/src/SourceLink.GitHub/GetSourceLinkUrl.cs index 9df77801..db632e7a 100644 --- a/src/SourceLink.GitHub/GetSourceLinkUrl.cs +++ b/src/SourceLink.GitHub/GetSourceLinkUrl.cs @@ -19,7 +19,7 @@ public sealed class GetSourceLinkUrl : GetSourceLinkUrlGitTask protected override Uri GetDefaultContentUriFromHostUri(string authority, Uri gitUri) => new Uri($"{gitUri.Scheme}://{authority}/raw", UriKind.Absolute); - protected override string BuildSourceLinkUrl(Uri contentUri, Uri gitUri, string relativeUrl, string revisionId, ITaskItem hostItem) + protected override string? BuildSourceLinkUrl(Uri contentUri, Uri gitUri, string relativeUrl, string revisionId, ITaskItem? hostItem) => UriUtilities.Combine(UriUtilities.Combine(contentUri.ToString(), relativeUrl), revisionId + "/*"); } } diff --git a/src/SourceLink.GitHub/Microsoft.SourceLink.GitHub.csproj b/src/SourceLink.GitHub/Microsoft.SourceLink.GitHub.csproj index 651620d3..59bdd335 100644 --- a/src/SourceLink.GitHub/Microsoft.SourceLink.GitHub.csproj +++ b/src/SourceLink.GitHub/Microsoft.SourceLink.GitHub.csproj @@ -13,6 +13,7 @@ true + diff --git a/src/SourceLink.GitLab/GetSourceLinkUrl.cs b/src/SourceLink.GitLab/GetSourceLinkUrl.cs index 3abab888..2edd3917 100644 --- a/src/SourceLink.GitLab/GetSourceLinkUrl.cs +++ b/src/SourceLink.GitLab/GetSourceLinkUrl.cs @@ -16,7 +16,7 @@ public sealed class GetSourceLinkUrl : GetSourceLinkUrlGitTask protected override string HostsItemGroupName => "SourceLinkGitLabHost"; protected override string ProviderDisplayName => "GitLab"; - protected override string BuildSourceLinkUrl(Uri contentUri, Uri gitUri, string relativeUrl, string revisionId, ITaskItem hostItem) + protected override string? BuildSourceLinkUrl(Uri contentUri, Uri gitUri, string relativeUrl, string revisionId, ITaskItem? hostItem) => UriUtilities.Combine(UriUtilities.Combine(contentUri.ToString(), relativeUrl), "raw/" + revisionId + "/*"); } } diff --git a/src/SourceLink.GitLab/Microsoft.SourceLink.GitLab.csproj b/src/SourceLink.GitLab/Microsoft.SourceLink.GitLab.csproj index 557305a4..dd54bbb7 100644 --- a/src/SourceLink.GitLab/Microsoft.SourceLink.GitLab.csproj +++ b/src/SourceLink.GitLab/Microsoft.SourceLink.GitLab.csproj @@ -13,6 +13,7 @@ true + diff --git a/src/TestUtilities/AssertEx.cs b/src/TestUtilities/AssertEx.cs index 705a62ac..77c780a8 100644 --- a/src/TestUtilities/AssertEx.cs +++ b/src/TestUtilities/AssertEx.cs @@ -37,7 +37,7 @@ public static bool IsNull(T @object) return false; } - return object.Equals(@object, default(T)); + return object.Equals(@object, default(T)!); } public static bool Equals(T left, T right) @@ -49,44 +49,38 @@ bool IEqualityComparer.Equals(T x, T y) { if (CanBeNull()) { - if (object.Equals(x, default(T))) + if (object.Equals(x, default(T)!)) { - return object.Equals(y, default(T)); + return object.Equals(y, default(T)!); } - if (object.Equals(y, default(T))) + if (object.Equals(y, default(T)!)) { return false; } } - if (x.GetType() != y.GetType()) + if (x!.GetType() != y!.GetType()) { return false; } - var equatable = x as IEquatable; - if (equatable != null) + if (x is IEquatable equatable) { return equatable.Equals(y); } - var comparableT = x as IComparable; - if (comparableT != null) + if (x is IComparable comparableT) { return comparableT.CompareTo(y) == 0; } - var comparable = x as IComparable; - if (comparable != null) + if (x is IComparable comparable) { return comparable.CompareTo(y) == 0; } - var enumerableX = x as IEnumerable; - var enumerableY = y as IEnumerable; - - if (enumerableX != null && enumerableY != null) + if (x is IEnumerable enumerableX && y is IEnumerable enumerableY) { var enumeratorX = enumerableX.GetEnumerator(); var enumeratorY = enumerableY.GetEnumerator(); @@ -119,7 +113,7 @@ int IEqualityComparer.GetHashCode(T obj) #endregion - public static void AreEqual(T expected, T actual, string message = null, IEqualityComparer comparer = null) + public static void AreEqual(T expected, T actual, string? message = null, IEqualityComparer? comparer = null) { if (ReferenceEquals(expected, actual)) { @@ -148,7 +142,7 @@ public static void AreEqual(T expected, T actual, string message = null, IEqu } } - public static void Equal(ImmutableArray expected, IEnumerable actual, Func comparer = null, string message = null) + public static void Equal(ImmutableArray expected, IEnumerable actual, Func? comparer = null, string? message = null) { if (actual == null || expected.IsDefault) { @@ -160,7 +154,7 @@ public static void Equal(ImmutableArray expected, IEnumerable actual, F } } - public static void Equal(IEnumerable expected, ImmutableArray actual, Func comparer = null, string message = null, string itemSeparator = null) + public static void Equal(IEnumerable expected, ImmutableArray actual, Func? comparer = null, string? message = null, string? itemSeparator = null) { if (expected == null || actual.IsDefault) { @@ -172,13 +166,13 @@ public static void Equal(IEnumerable expected, ImmutableArray actual, F } } - public static void Equal(ImmutableArray expected, ImmutableArray actual, Func comparer = null, string message = null, string itemSeparator = null) + public static void Equal(ImmutableArray expected, ImmutableArray actual, Func? comparer = null, string? message = null, string? itemSeparator = null) { Equal(expected, (IEnumerable)actual, comparer, message, itemSeparator); } - public static void Equal(IEnumerable expected, IEnumerable actual, Func comparer = null, string message = null, - string itemSeparator = null, Func itemInspector = null) + public static void Equal(IEnumerable? expected, IEnumerable? actual, Func? comparer = null, string? message = null, + string? itemSeparator = null, Func? itemInspector = null) { if (ReferenceEquals(expected, actual)) { @@ -206,7 +200,7 @@ public static void Equal(IEnumerable expected, IEnumerable actual, Func } } - private static bool SequenceEqual(IEnumerable expected, IEnumerable actual, Func comparer = null) + private static bool SequenceEqual(IEnumerable expected, IEnumerable actual, Func? comparer = null) { var enumerator1 = expected.GetEnumerator(); var enumerator2 = actual.GetEnumerator(); @@ -238,7 +232,7 @@ private static bool SequenceEqual(IEnumerable expected, IEnumerable act return true; } - public static void SetEqual(IEnumerable expected, IEnumerable actual, IEqualityComparer comparer = null, string message = null, string itemSeparator = "\r\n") + public static void SetEqual(IEnumerable expected, IEnumerable actual, IEqualityComparer? comparer = null, string? message = null) { var expectedSet = new HashSet(expected, comparer); var result = expected.Count() == actual.Count() && expectedSet.SetEquals(actual); @@ -288,12 +282,12 @@ public static void All(IEnumerable actual, Func predicate) } } - public static string ToString(object o) + public static string ToString(object? o) { return Convert.ToString(o); } - public static string ToString(IEnumerable list, string separator = ", ", Func itemInspector = null) + public static string ToString(IEnumerable list, string separator = ", ", Func? itemInspector = null) { if (itemInspector == null) { @@ -313,30 +307,25 @@ public static void Fail(string format, params object[] args) Assert.False(true, string.Format(format, args)); } - public static void Null(T @object, string message = null) + public static void Null(T @object, string? message = null) { Assert.True(AssertEqualityComparer.IsNull(@object), message); } - public static void NotNull(T @object, string message = null) + public static void NotNull(T @object, string? message = null) { Assert.False(AssertEqualityComparer.IsNull(@object), message); } // compares against a baseline - public static void AssertEqualToleratingWhitespaceDifferences( - string expected, - string actual, - bool escapeQuotes = true, - [CallerFilePath]string expectedValueSourcePath = null, - [CallerLineNumber]int expectedValueSourceLine = 0) + public static void AssertEqualToleratingWhitespaceDifferences(string expected, string actual, bool escapeQuotes = true) { var normalizedExpected = NormalizeWhitespace(expected); var normalizedActual = NormalizeWhitespace(actual); if (normalizedExpected != normalizedActual) { - Assert.True(false, GetAssertMessage(expected, actual, escapeQuotes, expectedValueSourcePath, expectedValueSourceLine)); + Assert.True(false, GetAssertMessage(expected, actual, escapeQuotes)); } } @@ -414,25 +403,23 @@ internal static string NormalizeWhitespace(string input) return output.ToString(); } - public static string GetAssertMessage(string expected, string actual, bool escapeQuotes = false, string expectedValueSourcePath = null, int expectedValueSourceLine = 0) + public static string GetAssertMessage(string expected, string actual, bool escapeQuotes = false) { - return GetAssertMessage(DiffUtil.Lines(expected), DiffUtil.Lines(actual), escapeQuotes, expectedValueSourcePath, expectedValueSourceLine); + return GetAssertMessage(DiffUtil.Lines(expected), DiffUtil.Lines(actual), escapeQuotes); } - public static string GetAssertMessage(IEnumerable expected, IEnumerable actual, bool escapeQuotes, string expectedValueSourcePath = null, int expectedValueSourceLine = 0) + public static string GetAssertMessage(IEnumerable expected, IEnumerable actual, bool escapeQuotes) { - Func itemInspector = escapeQuotes ? new Func(t => t.ToString().Replace("\"", "\"\"")) : null; - return GetAssertMessage(expected, actual, itemInspector: itemInspector, itemSeparator: "\r\n", expectedValueSourcePath: expectedValueSourcePath, expectedValueSourceLine: expectedValueSourceLine); + Func? itemInspector = escapeQuotes ? new Func(t => t?.ToString().Replace("\"", "\"\"") ?? "null") : null; + return GetAssertMessage(expected, actual, itemInspector: itemInspector, itemSeparator: "\r\n"); } public static string GetAssertMessage( IEnumerable expected, IEnumerable actual, - Func comparer = null, - Func itemInspector = null, - string itemSeparator = null, - string expectedValueSourcePath = null, - int expectedValueSourceLine = 0) + Func? comparer = null, + Func? itemInspector = null, + string? itemSeparator = null) { if (itemInspector == null) { @@ -474,7 +461,7 @@ public static string GetAssertMessage( return message.ToString(); } - public static void AssertLinesEqual(string expected, string actual, string message = null, Func comparer = null) + public static void AssertLinesEqual(string expected, string actual, string? message = null, Func? comparer = null) { if (expected == actual) { @@ -484,12 +471,12 @@ public static void AssertLinesEqual(string expected, string actual, string messa Assert.NotNull(expected); Assert.NotNull(actual); - IEnumerable GetLines(string str) => + static IEnumerable getLines(string str) => str.Trim().Replace("\r\n", "\n").Split(new[] { '\r', '\n' }, StringSplitOptions.None); Equal( - GetLines(expected), - GetLines(actual), + getLines(expected), + getLines(actual), message: message, comparer: comparer ?? new Func((left, right) => left.Trim() == right.Trim()), itemInspector: line => line.Replace("\"", "\"\""), diff --git a/src/TestUtilities/DiffUtil.cs b/src/TestUtilities/DiffUtil.cs index 7f6719fc..5bcbc94c 100644 --- a/src/TestUtilities/DiffUtil.cs +++ b/src/TestUtilities/DiffUtil.cs @@ -70,10 +70,10 @@ public IEnumerable CalculateDiff(IList sequenceA, IList sequenceB, } } - public static string DiffReport(IEnumerable expected, IEnumerable actual, Func comparer = null, Func toString = null, string separator = ",\r\n") + public static string DiffReport(IEnumerable expected, IEnumerable actual, Func? comparer = null, Func? toString = null, string? separator = ",\r\n") { var lcs = (comparer != null) ? new LCS(comparer) : LCS.Default; - toString = toString ?? new Func(obj => obj.ToString()); + toString ??= new Func(obj => obj?.ToString() ?? ""); IList expectedList = expected as IList ?? new List(expected); IList actualList = actual as IList ?? new List(actual); diff --git a/src/TestUtilities/DotNetSdk/DotNetSdkTestBase.cs b/src/TestUtilities/DotNetSdk/DotNetSdkTestBase.cs index 6023c7ab..d66b208c 100644 --- a/src/TestUtilities/DotNetSdk/DotNetSdkTestBase.cs +++ b/src/TestUtilities/DotNetSdk/DotNetSdkTestBase.cs @@ -21,19 +21,19 @@ public sealed class DotNetSdkAvailable : ExecutionCondition public readonly TempRoot Temp = new TempRoot(); - private static readonly string s_dotnetExeName; - private static readonly string s_dotnetInstallDir; + private static readonly string? s_dotnetExeName; + private static readonly string? s_dotnetInstallDir; private static readonly BuildInfoAttribute s_buildInfo; - private static readonly string s_dotnetSdkPath; + private static readonly string? s_dotnetSdkPath; - private static string s_projectSource = + private static readonly string s_projectSource = @" netstandard2.0 "; - private static string s_classSource = + private static readonly string s_classSource = @"using System; public class TestClass @@ -84,7 +84,7 @@ static DotNetSdkTestBase() var minSdkVersion = SemanticVersion.Parse(s_buildInfo.SdkVersion); - bool isDotNetInstallDirectory(string dir) + static bool isDotNetInstallDirectory(string dir) => dir != null && File.Exists(Path.Combine(dir, s_dotnetExeName)); var dotnetInstallDir = @@ -203,16 +203,16 @@ protected void VerifyValues( string customTargets, string[] targets, string[] expressions, - string[] expectedResults = null, - string[] expectedErrors = null, - string[] expectedWarnings = null, - string additionalCommandLineArgs = null, + string[]? expectedResults = null, + string[]? expectedErrors = null, + string[]? expectedWarnings = null, + string? additionalCommandLineArgs = null, string buildVerbosity = "minimal", - Func expectedBuildOutputFilter = null) + Func? expectedBuildOutputFilter = null) { - Debug.Assert(targets != null); - Debug.Assert(expressions != null); - Debug.Assert(expectedResults == null ^ expectedErrors == null); + NullableDebug.Assert(targets != null); + NullableDebug.Assert(expressions != null); + NullableDebug.Assert(expectedResults == null ^ expectedErrors == null); var evaluationResultsFile = Path.Combine(ProjectOutDir.Path, "EvaluationResult.txt"); diff --git a/src/TestUtilities/ProcessUtilities.cs b/src/TestUtilities/ProcessUtilities.cs index ce0082f6..83406fb8 100644 --- a/src/TestUtilities/ProcessUtilities.cs +++ b/src/TestUtilities/ProcessUtilities.cs @@ -16,9 +16,9 @@ public static class ProcessUtilities public static ProcessResult Run( string fileName, string arguments, - string workingDirectory = null, - IEnumerable> additionalEnvironmentVars = null, - string stdInput = null) + string? workingDirectory = null, + IEnumerable>? additionalEnvironmentVars = null, + string? stdInput = null) { if (fileName == null) throw new ArgumentNullException(nameof(fileName)); @@ -88,7 +88,7 @@ public static ProcessResult Run( /// Launch a process, and return Process object. The process continues to run asynchronously. /// You cannot capture the output. /// - public static Process StartProcess(string fileName, string arguments, string workingDirectory = null) + public static Process StartProcess(string fileName, string arguments, string? workingDirectory = null) { if (fileName == null) { @@ -111,14 +111,14 @@ public static Process StartProcess(string fileName, string arguments, string wor return p; } - public static string RunAndGetOutput(string exeFileName, string arguments = null, int expectedRetCode = 0, string startFolder = null) + public static string RunAndGetOutput(string exeFileName, string? arguments = null, int expectedRetCode = 0, string? startFolder = null) { ProcessStartInfo startInfo = new ProcessStartInfo(exeFileName); if (arguments != null) { startInfo.Arguments = arguments; } - string result = null; + string? result = null; startInfo.CreateNoWindow = true; startInfo.RedirectStandardOutput = true; diff --git a/src/TestUtilities/TempFiles/DisposableFile.cs b/src/TestUtilities/TempFiles/DisposableFile.cs index 8767069b..cd7669d9 100644 --- a/src/TestUtilities/TempFiles/DisposableFile.cs +++ b/src/TestUtilities/TempFiles/DisposableFile.cs @@ -14,7 +14,7 @@ public DisposableFile(string path) { } - public DisposableFile(string prefix = null, string extension = null, string directory = null, string callerSourcePath = null, int callerLineNumber = 0) + public DisposableFile(string? prefix = null, string? extension = null, string? directory = null, string? callerSourcePath = null, int callerLineNumber = 0) : base(prefix, extension, directory, callerSourcePath, callerLineNumber) { } diff --git a/src/TestUtilities/TempFiles/TempDirectory.cs b/src/TestUtilities/TempFiles/TempDirectory.cs index a0a30fbd..e6860d82 100644 --- a/src/TestUtilities/TempFiles/TempDirectory.cs +++ b/src/TestUtilities/TempFiles/TempDirectory.cs @@ -18,8 +18,8 @@ protected TempDirectory(TempRoot root) private TempDirectory(string path, TempRoot root) { - Debug.Assert(path != null); - Debug.Assert(root != null); + NullableDebug.Assert(path != null); + NullableDebug.Assert(root != null); _path = path; _root = root; @@ -61,7 +61,7 @@ public TempFile CreateFile(string name) /// /// Creates a file in this directory that is a copy of the specified file. /// - public TempFile CopyFile(string originalPath, string name = null) + public TempFile CopyFile(string originalPath, string? name = null) { string filePath = System.IO.Path.Combine(_path, name ?? System.IO.Path.GetFileName(originalPath)); File.Copy(originalPath, filePath); diff --git a/src/TestUtilities/TempFiles/TempFile.cs b/src/TestUtilities/TempFiles/TempFile.cs index 94e0241b..0bd9dc32 100644 --- a/src/TestUtilities/TempFiles/TempFile.cs +++ b/src/TestUtilities/TempFiles/TempFile.cs @@ -18,7 +18,7 @@ internal TempFile(string path) _path = path; } - internal TempFile(string prefix, string extension, string directory, string callerSourcePath, int callerLineNumber) + internal TempFile(string? prefix, string? extension, string? directory, string? callerSourcePath, int callerLineNumber) { while (true) { diff --git a/src/TestUtilities/TempFiles/TempRoot.cs b/src/TestUtilities/TempFiles/TempRoot.cs index 1e803d55..5e5083ea 100644 --- a/src/TestUtilities/TempFiles/TempRoot.cs +++ b/src/TestUtilities/TempFiles/TempRoot.cs @@ -65,7 +65,7 @@ public TempDirectory CreateDirectory() return dir; } - public TempFile CreateFile(string prefix = null, string extension = null, string directory = null, [CallerFilePath]string callerSourcePath = null, [CallerLineNumber]int callerLineNumber = 0) + public TempFile CreateFile(string? prefix = null, string? extension = null, string? directory = null, [CallerFilePath]string? callerSourcePath = null, [CallerLineNumber]int callerLineNumber = 0) { CheckDisposed(); return AddFile(new DisposableFile(prefix, extension, directory, callerSourcePath, callerLineNumber)); diff --git a/src/TestUtilities/TestUtilities.csproj b/src/TestUtilities/TestUtilities.csproj index 168f8768..20953914 100644 --- a/src/TestUtilities/TestUtilities.csproj +++ b/src/TestUtilities/TestUtilities.csproj @@ -16,6 +16,10 @@ <_DotNetSdkBuildIInfoFile>BuildInfo.g.cs + + + + <_Attribute Include="TestUtilities.BuildInfoAttribute"> From 39c946a79b0daca4e47ec6d23554fea69f33253d Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 27 Jan 2020 13:21:33 +0000 Subject: [PATCH 053/161] Update dependencies from https://github.com/dotnet/arcade build 20200126.3 (#569) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.20076.3 --- eng/Version.Details.xml | 4 +-- eng/common/cross/build-rootfs.sh | 19 +++++++++++--- eng/common/enable-cross-org-publishing.ps1 | 9 +++++-- .../templates/post-build/post-build.yml | 26 +++++++++++++++++++ eng/common/tools.ps1 | 3 +++ global.json | 2 +- 6 files changed, 54 insertions(+), 9 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 36084ef8..1a8adbef 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - ebc71292545adda514c1e6a5657cd9ab753240bc + cc8fe69635c7c6e791c87540851aea75946945fa diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 8d61377a..a23f895b 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -25,8 +25,9 @@ __UbuntuPackages="build-essential" __AlpinePackages="alpine-base" __AlpinePackages+=" build-base" __AlpinePackages+=" linux-headers" -__AlpinePackages+=" lldb-dev" -__AlpinePackages+=" llvm-dev" +__AlpinePackagesEdgeTesting=" lldb-dev" +__AlpinePackagesEdgeMain=" llvm9-libs" +__AlpinePackagesEdgeMain+=" python3" # symlinks fixer __UbuntuPackages+=" symlinks" @@ -199,13 +200,23 @@ if [[ "$__LinuxCodeName" == "alpine" ]]; then tar -xf $__ApkToolsDir/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -C $__ApkToolsDir mkdir -p $__RootfsDir/usr/bin cp -v /usr/bin/qemu-$__QEMUArch-static $__RootfsDir/usr/bin + $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \ -X http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/main \ -X http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/community \ - -X http://dl-cdn.alpinelinux.org/alpine/edge/testing \ - -X http://dl-cdn.alpinelinux.org/alpine/edge/main \ -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ add $__AlpinePackages + + $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \ + -X http://dl-cdn.alpinelinux.org/alpine/edge/main \ + -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ + add $__AlpinePackagesEdgeMain + + $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \ + -X http://dl-cdn.alpinelinux.org/alpine/edge/testing \ + -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ + add $__AlpinePackagesEdgeTesting + rm -r $__ApkToolsDir elif [[ -n $__LinuxCodeName ]]; then qemu-debootstrap --arch $__UbuntuArch $__LinuxCodeName $__RootfsDir $__UbuntuRepo diff --git a/eng/common/enable-cross-org-publishing.ps1 b/eng/common/enable-cross-org-publishing.ps1 index efa26621..da09da4f 100644 --- a/eng/common/enable-cross-org-publishing.ps1 +++ b/eng/common/enable-cross-org-publishing.ps1 @@ -2,7 +2,12 @@ param( [string] $token ) + . $PSScriptRoot\pipeline-logging-functions.ps1 -Write-PipelineSetVariable -Name 'VSS_NUGET_ACCESSTOKEN' -Value $token -Write-PipelineSetVariable -Name 'VSS_NUGET_URI_PREFIXES' -Value 'https://dnceng.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/dnceng/;https://devdiv.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/devdiv/' +# Write-PipelineSetVariable will no-op if a variable named $ci is not defined +# Since this script is only ever called in AzDO builds, just universally set it +$ci = $true + +Write-PipelineSetVariable -Name 'VSS_NUGET_ACCESSTOKEN' -Value $token -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'VSS_NUGET_URI_PREFIXES' -Value 'https://dnceng.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/dnceng/;https://devdiv.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/devdiv/' -IsMultiJobVariable $false diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 8a8d84f2..33295ba1 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -221,3 +221,29 @@ stages: transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental-symbols/nuget/v3/index.json' + +- template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'Net_Eng_Services_Int_Publish' + channelName: '.NET Eng Services - Int' + channelId: 678 + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' + +- template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'Net_Eng_Services_Prod_Publish' + channelName: '.NET Eng Services - Prod' + channelId: 679 + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index d3a43287..23a3fd53 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -655,6 +655,9 @@ Write-PipelineSetVariable -Name 'Artifacts.Log' -Value $LogDir Write-PipelineSetVariable -Name 'TEMP' -Value $TempDir Write-PipelineSetVariable -Name 'TMP' -Value $TempDir +$env:TEMP=$TempDir +$env:TMP=$TempDir + # Import custom tools configuration, if present in the repo. # Note: Import in global scope so that the script set top-level variables without qualification. if (!$disableConfigureToolsetImport) { diff --git a/global.json b/global.json index 0d065575..50b8a8cd 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.1.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20068.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20076.3" } } From 67a41641083881dc5e690f3ac193fba9fc035025 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 3 Feb 2020 13:26:52 +0000 Subject: [PATCH 054/161] Update dependencies from https://github.com/dotnet/arcade build 20200201.2 (#571) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.20101.2 --- eng/Version.Details.xml | 4 +- eng/common/build.sh | 10 + eng/common/darc-init.ps1 | 4 +- eng/common/generate-graph-files.ps1 | 2 +- eng/common/performance/performance-setup.ps1 | 2 +- eng/common/performance/performance-setup.sh | 2 +- ...ote-build.ps1 => add-build-to-channel.ps1} | 2 +- eng/common/templates/job/execute-sdl.yml | 20 +- .../channels/generic-internal-channel.yml | 30 ++- .../channels/generic-public-channel.yml | 30 ++- .../templates/post-build/post-build.yml | 219 +++++++++++------- .../templates/post-build/promote-build.yml | 25 -- .../post-build/setup-maestro-vars.yml | 51 +++- ...ote-build.yml => add-build-to-channel.yml} | 4 +- eng/common/tools.ps1 | 3 - global.json | 2 +- 16 files changed, 265 insertions(+), 145 deletions(-) rename eng/common/post-build/{promote-build.ps1 => add-build-to-channel.ps1} (95%) delete mode 100644 eng/common/templates/post-build/promote-build.yml rename eng/common/templates/steps/{promote-build.yml => add-build-to-channel.yml} (68%) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 1a8adbef..95121939 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - cc8fe69635c7c6e791c87540851aea75946945fa + 779323758d3e8aa7e4ee3dc2619d57e696078089 diff --git a/eng/common/build.sh b/eng/common/build.sh index 36f9aa04..6e395455 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -175,6 +175,16 @@ function Build { InitializeCustomToolset if [[ ! -z "$projects" ]]; then + # Split project paths by semi-colon, find full-paths using readlink, + # finally and splice back as a semi-colon separated list + IFS=';' read -r -a projs <<< "$projects" + len=${#projs[@]} + for ((i=0; i<$len; i++)); + do + projs[$i]=$(readlink -f "${projs[$i]}"); + done + projects=$(IFS=\; ; echo "${projs[*]}") + echo Updated projects: $projects properties="$properties /p:Projects=$projects" fi diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 index fc219036..2a30ccfd 100644 --- a/eng/common/darc-init.ps1 +++ b/eng/common/darc-init.ps1 @@ -24,13 +24,15 @@ function InstallDarcCli ($darcVersion) { $darcVersion = $(Invoke-WebRequest -Uri $versionEndpoint -UseBasicParsing).Content } - $arcadeServicesSource = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + $arcadeServicesSource = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' Write-Host "Installing Darc CLI version $darcVersion..." Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.' if (-not $toolpath) { + Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity -g" & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g }else { + Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity -g --tool-path '$toolpath'" & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath" } } diff --git a/eng/common/generate-graph-files.ps1 b/eng/common/generate-graph-files.ps1 index 7ad26afa..0728b1a8 100644 --- a/eng/common/generate-graph-files.ps1 +++ b/eng/common/generate-graph-files.ps1 @@ -3,7 +3,7 @@ Param( [Parameter(Mandatory=$true)][string] $gitHubPat, # GitHub personal access token from https://github.com/settings/tokens (no auth scopes needed) [Parameter(Mandatory=$true)][string] $azdoPat, # Azure Dev Ops tokens from https://dev.azure.com/dnceng/_details/security/tokens (code read scope needed) [Parameter(Mandatory=$true)][string] $outputFolder, # Where the graphviz.txt file will be created - [string] $darcVersion = '1.1.0-beta.19175.6', # darc's version + [string] $darcVersion, # darc's version [string] $graphvizVersion = '2.38', # GraphViz version [switch] $includeToolset # Whether the graph should include toolset dependencies or not. i.e. arcade, optimization. For more about # toolset dependencies see https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#toolset-vs-product-dependencies diff --git a/eng/common/performance/performance-setup.ps1 b/eng/common/performance/performance-setup.ps1 index f6e87829..3c2adf8c 100644 --- a/eng/common/performance/performance-setup.ps1 +++ b/eng/common/performance/performance-setup.ps1 @@ -9,7 +9,7 @@ Param( [string] $Branch=$env:BUILD_SOURCEBRANCH, [string] $CommitSha=$env:BUILD_SOURCEVERSION, [string] $BuildNumber=$env:BUILD_BUILDNUMBER, - [string] $RunCategories="coreclr corefx", + [string] $RunCategories="Libraries Runtime", [string] $Csproj="src\benchmarks\micro\MicroBenchmarks.csproj", [string] $Kind="micro", [switch] $Internal, diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index 7ef15514..604aebff 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -13,7 +13,7 @@ build_number=$BUILD_BUILDNUMBER internal=false compare=false kind="micro" -run_categories="coreclr corefx" +run_categories="Libraries Runtime" csproj="src\benchmarks\micro\MicroBenchmarks.csproj" configurations= run_from_perf_repo=false diff --git a/eng/common/post-build/promote-build.ps1 b/eng/common/post-build/add-build-to-channel.ps1 similarity index 95% rename from eng/common/post-build/promote-build.ps1 rename to eng/common/post-build/add-build-to-channel.ps1 index ce45635f..de2d9579 100644 --- a/eng/common/post-build/promote-build.ps1 +++ b/eng/common/post-build/add-build-to-channel.ps1 @@ -17,7 +17,7 @@ try { ExitWithExitCode 1 } - # Get info about which channels the build has already been promoted to + # Get info about which channel(s) the build has already been promoted to $buildInfo = Get-MaestroBuild -BuildId $BuildId if (!$buildInfo) { diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index 2973bcaf..640f2b04 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -1,4 +1,5 @@ parameters: + enable: 'false' # Whether the SDL validation job should execute or not overrideParameters: '' # Optional: to override values for parameters. additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")' # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named @@ -16,8 +17,15 @@ jobs: - job: Run_SDL dependsOn: ${{ parameters.dependsOn }} displayName: Run SDL tool + condition: eq( ${{ parameters.enable }}, 'true') variables: - group: DotNet-VSTS-Bot + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] pool: name: Hosted VS2017 steps: @@ -28,14 +36,22 @@ jobs: - task: DownloadBuildArtifacts@0 displayName: Download Build Artifacts inputs: - buildType: current + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) artifactName: ${{ artifactName }} downloadPath: $(Build.ArtifactStagingDirectory)\artifacts - ${{ if eq(parameters.artifactNames, '') }}: - task: DownloadBuildArtifacts@0 displayName: Download Build Artifacts inputs: - buildType: current + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) downloadType: specific files itemPattern: "**" downloadPath: $(Build.ArtifactStagingDirectory)\artifacts diff --git a/eng/common/templates/post-build/channels/generic-internal-channel.yml b/eng/common/templates/post-build/channels/generic-internal-channel.yml index 70021104..29d8d46e 100644 --- a/eng/common/templates/post-build/channels/generic-internal-channel.yml +++ b/eng/common/templates/post-build/channels/generic-internal-channel.yml @@ -23,9 +23,15 @@ stages: - job: publish_symbols displayName: Symbol Publishing dependsOn: setupMaestroVars - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )) + condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )), eq(dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'], ${{ parameters.channelId }})) variables: - group: DotNet-Symbol-Server-Pats + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] pool: vmImage: 'windows-2019' steps: @@ -33,7 +39,11 @@ stages: displayName: Download Build Assets continueOnError: true inputs: - buildType: 'current' + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) downloadType: 'specific' itemPattern: | PDBArtifacts/** @@ -79,7 +89,13 @@ stages: value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] - name: IsStableBuild value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ] - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }})) + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )), eq(dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'], ${{ parameters.channelId }})) pool: vmImage: 'windows-2019' steps: @@ -87,7 +103,11 @@ stages: displayName: Download Build Assets continueOnError: true inputs: - buildType: 'current' + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) downloadType: 'specific' itemPattern: | PackageArtifacts/** @@ -146,6 +166,6 @@ stages: StageLabel: '${{ parameters.stageName }}' JobLabel: 'AssetsPublishing' - - template: ../../steps/promote-build.yml + - template: ../../steps/add-build-to-channel.yml parameters: ChannelId: ${{ parameters.channelId }} diff --git a/eng/common/templates/post-build/channels/generic-public-channel.yml b/eng/common/templates/post-build/channels/generic-public-channel.yml index fbb5a19b..883e033c 100644 --- a/eng/common/templates/post-build/channels/generic-public-channel.yml +++ b/eng/common/templates/post-build/channels/generic-public-channel.yml @@ -23,9 +23,15 @@ stages: - job: publish_symbols displayName: Symbol Publishing dependsOn: setupMaestroVars - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )) + condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )), eq(dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'], ${{ parameters.channelId }})) variables: - group: DotNet-Symbol-Server-Pats + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] pool: vmImage: 'windows-2019' steps: @@ -33,7 +39,11 @@ stages: displayName: Download Build Assets continueOnError: true inputs: - buildType: 'current' + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) downloadType: 'specific' itemPattern: | PDBArtifacts/** @@ -78,7 +88,13 @@ stages: value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] - name: IsStableBuild value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ] - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }})) + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )), eq(dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'], ${{ parameters.channelId }})) pool: vmImage: 'windows-2019' steps: @@ -86,7 +102,11 @@ stages: displayName: Download Build Assets continueOnError: true inputs: - buildType: 'current' + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) downloadType: 'specific' itemPattern: | PackageArtifacts/** @@ -145,6 +165,6 @@ stages: StageLabel: '${{ parameters.stageName }}' JobLabel: 'AssetsPublishing' - - template: ../../steps/promote-build.yml + - template: ../../steps/add-build-to-channel.yml parameters: ChannelId: ${{ parameters.channelId }} diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 33295ba1..70cb65f5 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -26,97 +26,136 @@ stages: - stage: Validate dependsOn: ${{ parameters.validateDependsOn }} displayName: Validate + variables: + - template: common-variables.yml jobs: - - ${{ if eq(parameters.enableNugetValidation, 'true') }}: - - job: - displayName: NuGet Validation - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: current - artifactName: PackageArtifacts - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 - arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ - -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ - - - ${{ if eq(parameters.enableSigningValidation, 'true') }}: - - job: - displayName: Signing Validation - variables: - - template: common-variables.yml - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: current - artifactName: PackageArtifacts - - # This is necessary whenever we want to publish/restore to an AzDO private feed - # Since sdk-task.ps1 tries to restore packages we need to do this authentication here - # otherwise it'll complain about accessing a private feed. - - task: NuGetAuthenticate@0 - displayName: 'Authenticate to AzDO Feeds' - - - task: PowerShell@2 - displayName: Enable cross-org publishing - inputs: - filePath: eng\common\enable-cross-org-publishing.ps1 - arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task SigningValidation -restore -msbuildEngine dotnet - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' - /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' - ${{ parameters.signingValidationAdditionalParameters }} - - - template: ../steps/publish-logs.yml - parameters: - StageLabel: 'Validation' - JobLabel: 'Signing' - - - ${{ if eq(parameters.enableSourceLinkValidation, 'true') }}: - - job: - displayName: SourceLink Validation - variables: - - template: common-variables.yml - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: current - artifactName: BlobArtifacts - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -ExtractPath $(Agent.BuildDirectory)/Extract/ - -GHRepoName $(Build.Repository.Name) - -GHCommit $(Build.SourceVersion) - -SourcelinkCliVersion $(SourceLinkCLIVersion) - continueOnError: true - - - ${{ if eq(parameters.SDLValidationParameters.enable, 'true') }}: - - template: /eng/common/templates/job/execute-sdl.yml - parameters: - additionalParameters: ${{ parameters.SDLValidationParameters.params }} - continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} - artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }} + - template: setup-maestro-vars.yml + + - job: + displayName: NuGet Validation + dependsOn: setupMaestroVars + condition: eq( ${{ parameters.enableNugetValidation }}, 'true') + pool: + vmImage: 'windows-2019' + variables: + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ + -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ + + - job: + displayName: Signing Validation + dependsOn: setupMaestroVars + condition: eq( ${{ parameters.enableSigningValidation }}, 'true') + variables: + - template: common-variables.yml + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + + # This is necessary whenever we want to publish/restore to an AzDO private feed + # Since sdk-task.ps1 tries to restore packages we need to do this authentication here + # otherwise it'll complain about accessing a private feed. + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + + - task: PowerShell@2 + displayName: Enable cross-org publishing + inputs: + filePath: eng\common\enable-cross-org-publishing.ps1 + arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task SigningValidation -restore -msbuildEngine dotnet + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' + /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' + ${{ parameters.signingValidationAdditionalParameters }} + + - template: ../steps/publish-logs.yml + parameters: + StageLabel: 'Validation' + JobLabel: 'Signing' + + - job: + displayName: SourceLink Validation + dependsOn: setupMaestroVars + condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') + variables: + - template: common-variables.yml + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BlobArtifacts + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) + -GHCommit $(Build.SourceVersion) + -SourcelinkCliVersion $(SourceLinkCLIVersion) + continueOnError: true + + - template: /eng/common/templates/job/execute-sdl.yml + parameters: + enable: ${{ parameters.SDLValidationParameters.enable }} + dependsOn: setupMaestroVars + additionalParameters: ${{ parameters.SDLValidationParameters.params }} + continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} + artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }} - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: diff --git a/eng/common/templates/post-build/promote-build.yml b/eng/common/templates/post-build/promote-build.yml deleted file mode 100644 index 6b479c3b..00000000 --- a/eng/common/templates/post-build/promote-build.yml +++ /dev/null @@ -1,25 +0,0 @@ -parameters: - ChannelId: 0 - -jobs: -- job: - displayName: Promote Build - dependsOn: setupMaestroVars - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.ChannelId }})) - variables: - - name: BARBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] - - name: ChannelId - value: ${{ parameters.ChannelId }} - pool: - vmImage: 'windows-2019' - steps: - - task: PowerShell@2 - displayName: Add Build to Channel - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/promote-build.ps1 - arguments: -BuildId $(BARBuildId) - -ChannelId $(ChannelId) - -MaestroApiAccessToken $(MaestroApiAccessToken) - -MaestroApiEndPoint $(MaestroApiEndPoint) - -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml index 716b53f7..05e611ed 100644 --- a/eng/common/templates/post-build/setup-maestro-vars.yml +++ b/eng/common/templates/post-build/setup-maestro-vars.yml @@ -1,6 +1,12 @@ jobs: - job: setupMaestroVars displayName: Setup Maestro Vars + variables: + - template: common-variables.yml + - name: BuildId + value: $[ coalesce(variables.BARBuildId, 0) ] + - name: PromoteToChannelId + value: $[ coalesce(variables.PromoteToMaestroChannelId, 0) ] pool: vmImage: 'windows-2019' steps: @@ -8,6 +14,7 @@ jobs: - task: DownloadBuildArtifacts@0 displayName: Download Release Configs + condition: eq(variables.PromoteToChannelId, 0) inputs: buildType: current artifactName: ReleaseConfigs @@ -19,18 +26,50 @@ jobs: targetType: inline script: | try { - $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt + if ($Env:PromoteToChannelId -eq 0) { + $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt - $BarId = $Content | Select -Index 0 + $BarId = $Content | Select -Index 0 - $Channels = "" - $Content | Select -Index 1 | ForEach-Object { $Channels += "$_ ," } + $Channels = "" + $Content | Select -Index 1 | ForEach-Object { $Channels += "$_ ," } + + $IsStableBuild = $Content | Select -Index 2 - $IsStableBuild = $Content | Select -Index 2 + $AzureDevOpsProject = $Env:System_TeamProject + $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId + $AzureDevOpsBuildId = $Env:Build_BuildId + $PromoteToMaestroChannelId = 0 + } + else { + $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" + + $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' + $apiHeaders.Add('Accept', 'application/json') + $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") + + $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } + + $BarId = $Env:BARBuildId + $Channels = 'None' + + #TODO: Fix this once this issue is done: https://github.com/dotnet/arcade/issues/3834 + $IsStableBuild = 'False' + + $AzureDevOpsProject = $buildInfo.azureDevOpsProject + $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId + $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId + $PromoteToMaestroChannelId = $Env:PromoteToMaestroChannelId + } Write-Host "##vso[task.setvariable variable=BARBuildId;isOutput=true]$BarId" Write-Host "##vso[task.setvariable variable=InitialChannels;isOutput=true]$Channels" Write-Host "##vso[task.setvariable variable=IsStableBuild;isOutput=true]$IsStableBuild" + + Write-Host "##vso[task.setvariable variable=AzDOProjectName;isOutput=true]$AzureDevOpsProject" + Write-Host "##vso[task.setvariable variable=AzDOPipelineId;isOutput=true]$AzureDevOpsBuildDefinitionId" + Write-Host "##vso[task.setvariable variable=AzDOBuildId;isOutput=true]$AzureDevOpsBuildId" + Write-Host "##vso[task.setvariable variable=PromoteToMaestroChannelId;isOutput=true]$PromoteToMaestroChannelId" } catch { Write-Host $_ @@ -38,3 +77,5 @@ jobs: Write-Host $_.ScriptStackTrace exit 1 } + env: + MAESTRO_API_TOKEN: $(MaestroApiAccessToken) diff --git a/eng/common/templates/steps/promote-build.yml b/eng/common/templates/steps/add-build-to-channel.yml similarity index 68% rename from eng/common/templates/steps/promote-build.yml rename to eng/common/templates/steps/add-build-to-channel.yml index b9040443..f67a210d 100644 --- a/eng/common/templates/steps/promote-build.yml +++ b/eng/common/templates/steps/add-build-to-channel.yml @@ -5,9 +5,9 @@ steps: - task: PowerShell@2 displayName: Add Build to Channel inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/promote-build.ps1 + filePath: $(Build.SourcesDirectory)/eng/common/post-build/add-build-to-channel.ps1 arguments: -BuildId $(BARBuildId) -ChannelId ${{ parameters.ChannelId }} -MaestroApiAccessToken $(MaestroApiAccessToken) -MaestroApiEndPoint $(MaestroApiEndPoint) - -MaestroApiVersion $(MaestroApiVersion) + -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 23a3fd53..d3a43287 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -655,9 +655,6 @@ Write-PipelineSetVariable -Name 'Artifacts.Log' -Value $LogDir Write-PipelineSetVariable -Name 'TEMP' -Value $TempDir Write-PipelineSetVariable -Name 'TMP' -Value $TempDir -$env:TEMP=$TempDir -$env:TMP=$TempDir - # Import custom tools configuration, if present in the repo. # Note: Import in global scope so that the script set top-level variables without qualification. if (!$disableConfigureToolsetImport) { diff --git a/global.json b/global.json index 50b8a8cd..146e0615 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.1.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20076.3" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20101.2" } } From 2d4ea7104ce7ee2b48776134af200f40df0db4ca Mon Sep 17 00:00:00 2001 From: Rich Lander Date: Mon, 3 Feb 2020 23:32:37 -0800 Subject: [PATCH 055/161] Update link (#570) --- docs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/README.md b/docs/README.md index f4dbb050..9487e869 100644 --- a/docs/README.md +++ b/docs/README.md @@ -71,7 +71,7 @@ By setting `DeterministicSourcePaths` to true the project opts into mapping all Only set `DeterministicSourcePaths` to true on a build/CI server, never for local builds. In order for the debugger to find source files when debugging a locally built binary, the PDB must contain original, unmapped local paths. -Starting with .NET Core SDK 2.1.300, a fully deterministic build is [turned on](https://github.com/dotnet/roslyn/blob/dev15.7.x/src/Compilers/Core/MSBuildTask/Microsoft.Managed.Core.targets#L45-L55) when both `Deterministic` and `ContinuousIntegrationBuild` properties are set to `true`. +Starting with .NET Core SDK 2.1.300, a fully deterministic build is [turned on](https://github.com/dotnet/roslyn/blob/master/src/Compilers/Core/MSBuildTask/Microsoft.Managed.Core.targets#L45-L55) when both `Deterministic` and `ContinuousIntegrationBuild` properties are set to `true`. ## Example From 00b7a360ff21ccf94729359f0a51ef502d40eab5 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 10 Feb 2020 13:11:49 +0000 Subject: [PATCH 056/161] Update dependencies from https://github.com/dotnet/arcade build 20200209.1 (#573) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.20109.1 --- eng/Version.Details.xml | 4 +- eng/common/build.sh | 10 ---- eng/common/performance/performance-setup.ps1 | 4 +- eng/common/performance/performance-setup.sh | 4 +- .../channels/generic-internal-channel.yml | 2 +- .../channels/generic-public-channel.yml | 2 +- eng/common/tools.ps1 | 48 ++++++++++++++----- eng/common/tools.sh | 38 +++++++++++---- global.json | 2 +- 9 files changed, 76 insertions(+), 38 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 95121939..b6627d7a 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 779323758d3e8aa7e4ee3dc2619d57e696078089 + b0e8d3944155f94f83deea8afe025debe369e69f diff --git a/eng/common/build.sh b/eng/common/build.sh index 6e395455..36f9aa04 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -175,16 +175,6 @@ function Build { InitializeCustomToolset if [[ ! -z "$projects" ]]; then - # Split project paths by semi-colon, find full-paths using readlink, - # finally and splice back as a semi-colon separated list - IFS=';' read -r -a projs <<< "$projects" - len=${#projs[@]} - for ((i=0; i<$len; i++)); - do - projs[$i]=$(readlink -f "${projs[$i]}"); - done - projects=$(IFS=\; ; echo "${projs[*]}") - echo Updated projects: $projects properties="$properties /p:Projects=$projects" fi diff --git a/eng/common/performance/performance-setup.ps1 b/eng/common/performance/performance-setup.ps1 index 3c2adf8c..e3376699 100644 --- a/eng/common/performance/performance-setup.ps1 +++ b/eng/common/performance/performance-setup.ps1 @@ -14,7 +14,7 @@ Param( [string] $Kind="micro", [switch] $Internal, [switch] $Compare, - [string] $Configurations="CompilationMode=$CompilationMode" + [string] $Configurations="CompilationMode=$CompilationMode RunKind=$Kind" ) $RunFromPerformanceRepo = ($Repository -eq "dotnet/performance") -or ($Repository -eq "dotnet-performance") @@ -50,7 +50,7 @@ if ($Internal) { } # FIX ME: This is a workaround until we get this from the actual pipeline -$CommonSetupArguments="--channel master --queue $Queue --build-number $BuildNumber --build-configs $Configurations" +$CommonSetupArguments="--channel master --queue $Queue --build-number $BuildNumber --build-configs $Configurations --architecture $Architecture" $SetupArguments = "--repository https://github.com/$Repository --branch $Branch --get-perf-hash --commit-sha $CommitSha $CommonSetupArguments" if ($RunFromPerformanceRepo) { diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index 604aebff..94a04e0f 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -15,7 +15,7 @@ compare=false kind="micro" run_categories="Libraries Runtime" csproj="src\benchmarks\micro\MicroBenchmarks.csproj" -configurations= +configurations="CompliationMode=$compilation_mode RunKind=$kind" run_from_perf_repo=false use_core_run=true use_baseline_core_run=true @@ -164,7 +164,7 @@ if [[ "$internal" == true ]]; then fi fi -common_setup_arguments="--channel master --queue $queue --build-number $build_number --build-configs $configurations" +common_setup_arguments="--channel master --queue $queue --build-number $build_number --build-configs $configurations --architecture $architecture" setup_arguments="--repository https://github.com/$repository --branch $branch --get-perf-hash --commit-sha $commit_sha $common_setup_arguments" if [[ "$run_from_perf_repo" = true ]]; then diff --git a/eng/common/templates/post-build/channels/generic-internal-channel.yml b/eng/common/templates/post-build/channels/generic-internal-channel.yml index 29d8d46e..38012290 100644 --- a/eng/common/templates/post-build/channels/generic-internal-channel.yml +++ b/eng/common/templates/post-build/channels/generic-internal-channel.yml @@ -46,7 +46,7 @@ stages: buildId: $(AzDOBuildId) downloadType: 'specific' itemPattern: | - PDBArtifacts/** + PdbArtifacts/** BlobArtifacts/** downloadPath: '$(Build.ArtifactStagingDirectory)' diff --git a/eng/common/templates/post-build/channels/generic-public-channel.yml b/eng/common/templates/post-build/channels/generic-public-channel.yml index 883e033c..ae8c6349 100644 --- a/eng/common/templates/post-build/channels/generic-public-channel.yml +++ b/eng/common/templates/post-build/channels/generic-public-channel.yml @@ -46,7 +46,7 @@ stages: buildId: $(AzDOBuildId) downloadType: 'specific' itemPattern: | - PDBArtifacts/** + PdbArtifacts/** BlobArtifacts/** downloadPath: '$(Build.ArtifactStagingDirectory)' diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index d3a43287..fef7e5a3 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -155,12 +155,12 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { # The following code block is protecting against concurrent access so that this function can # be called in parallel. if ($createSdkLocationFile) { - do { + do { $sdkCacheFileTemp = Join-Path $ToolsetDir $([System.IO.Path]::GetRandomFileName()) - } + } until (!(Test-Path $sdkCacheFileTemp)) Set-Content -Path $sdkCacheFileTemp -Value $dotnetRoot - + try { Rename-Item -Force -Path $sdkCacheFileTemp 'sdk.txt' } catch { @@ -188,7 +188,33 @@ function GetDotNetInstallScript([string] $dotnetRoot) { if (!(Test-Path $installScript)) { Create-Directory $dotnetRoot $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit - Invoke-WebRequest "https://dot.net/$dotnetInstallScriptVersion/dotnet-install.ps1" -OutFile $installScript + + $maxRetries = 5 + $retries = 1 + + $uri = "https://dot.net/$dotnetInstallScriptVersion/dotnet-install.ps1" + + while($true) { + try { + Write-Host "GET $uri" + Invoke-WebRequest $uri -OutFile $installScript + break + } + catch { + Write-Host "Failed to download '$uri'" + Write-Error $_.Exception.Message -ErrorAction Continue + } + + if (++$retries -le $maxRetries) { + $delayInSeconds = [math]::Pow(2, $retries) - 1 # Exponential backoff + Write-Host "Retrying. Waiting for $delayInSeconds seconds before next attempt ($retries of $maxRetries)." + Start-Sleep -Seconds $delayInSeconds + } + else { + throw "Unable to download file in $maxRetries attempts." + } + + } } return $installScript @@ -198,12 +224,12 @@ function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $arc InstallDotNet $dotnetRoot $version $architecture } -function InstallDotNet([string] $dotnetRoot, - [string] $version, - [string] $architecture = '', - [string] $runtime = '', - [bool] $skipNonVersionedFiles = $false, - [string] $runtimeSourceFeed = '', +function InstallDotNet([string] $dotnetRoot, + [string] $version, + [string] $architecture = '', + [string] $runtime = '', + [bool] $skipNonVersionedFiles = $false, + [string] $runtimeSourceFeed = '', [string] $runtimeSourceFeedKey = '') { $installScript = GetDotNetInstallScript $dotnetRoot @@ -298,7 +324,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = $vsMajorVersion = $vsMinVersion.Major $xcopyMSBuildVersion = "$vsMajorVersion.$($vsMinVersion.Minor).0-alpha" } - + $vsInstallDir = $null if ($xcopyMSBuildVersion.Trim() -ine "none") { $vsInstallDir = InitializeXCopyMSBuild $xcopyMSBuildVersion $install diff --git a/eng/common/tools.sh b/eng/common/tools.sh index e071af4e..c26525e5 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -41,7 +41,7 @@ fi # Configures warning treatment in msbuild. warn_as_error=${warn_as_error:-true} -# True to attempt using .NET Core already that meets requirements specified in global.json +# True to attempt using .NET Core already that meets requirements specified in global.json # installed on the machine instead of downloading one. use_installed_dotnet_cli=${use_installed_dotnet_cli:-true} @@ -172,7 +172,7 @@ function InstallDotNetSdk { function InstallDotNet { local root=$1 local version=$2 - + GetDotNetInstallScript "$root" local install_script=$_GetDotNetInstallScript @@ -218,6 +218,28 @@ function InstallDotNet { } } +function with_retries { + local maxRetries=5 + local retries=1 + echo "Trying to run '$@' for maximum of $maxRetries attempts." + while [[ $((retries++)) -le $maxRetries ]]; do + "$@" + + if [[ $? == 0 ]]; then + echo "Ran '$@' successfully." + return 0 + fi + + timeout=$((2**$retries-1)) + echo "Failed to execute '$@'. Waiting $timeout seconds before next attempt ($retries out of $maxRetries)." 1>&2 + sleep $timeout + done + + echo "Failed to execute '$@' for $maxRetries times." 1>&2 + + return 1 +} + function GetDotNetInstallScript { local root=$1 local install_script="$root/dotnet-install.sh" @@ -230,13 +252,13 @@ function GetDotNetInstallScript { # Use curl if available, otherwise use wget if command -v curl > /dev/null; then - curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || { + with_retries curl "$install_script_url" -isSLv --retry 10 --create-dirs -o "$install_script" || { local exit_code=$? Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." ExitWithExitCode $exit_code } - else - wget -q -O "$install_script" "$install_script_url" || { + else + with_retries wget -v -O "$install_script" "$install_script_url" || { local exit_code=$? Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." ExitWithExitCode $exit_code @@ -251,11 +273,11 @@ function InitializeBuildTool { if [[ -n "${_InitializeBuildTool:-}" ]]; then return fi - + InitializeDotNetCli $restore # return values - _InitializeBuildTool="$_InitializeDotNetCli/dotnet" + _InitializeBuildTool="$_InitializeDotNetCli/dotnet" _InitializeBuildToolCommand="msbuild" _InitializeBuildToolFramework="netcoreapp2.1" } @@ -319,7 +341,7 @@ function InitializeToolset { if [[ "$binary_log" == true ]]; then bl="/bl:$log_dir/ToolsetRestore.binlog" fi - + echo '' > "$proj" MSBuild-Core "$proj" $bl /t:__WriteToolsetLocation /clp:ErrorsOnly\;NoSummary /p:__ToolsetLocationOutputFile="$toolset_location_file" diff --git a/global.json b/global.json index 146e0615..d9868069 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.1.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20101.2" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20109.1" } } From bdc85b7525cfa0f0b918891b5e07f5a1306c5bf8 Mon Sep 17 00:00:00 2001 From: Glen Nicol <58054391+Glen-Nicol-Garmin@users.noreply.github.com> Date: Sun, 16 Feb 2020 12:41:06 -0800 Subject: [PATCH 057/161] Feature: GitWeb Source Link provider (#505) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Feature: GitWeb Source Link provider GitWeb only currently supports SSH repository URLs. https://github.com/dotnet/sourcelink/pull/505 * Remove Resources.Designer.cs * Remove comment Co-authored-by: Tomáš Matoušek --- SourceLink.sln | 12 ++ src/Common/TranslateRepositoryUrlGitTask.cs | 41 ++++-- .../GitWebTests.cs | 130 ++++++++++++++++++ ...oft.SourceLink.Git.IntegrationTests.csproj | 1 + .../GetSourceLinkUrlTests.cs | 56 ++++++++ ...crosoft.SourceLink.GitWeb.UnitTests.csproj | 9 ++ .../TranslateRepositoryUrlsTests.cs | 51 +++++++ src/SourceLink.GitWeb/GetSourceLinkUrl.cs | 37 +++++ .../Microsoft.SourceLink.GitWeb.csproj | 35 +++++ .../Microsoft.SourceLink.GitWeb.nuspec | 17 +++ src/SourceLink.GitWeb/Resources.resx | 123 +++++++++++++++++ .../TranslateRepositoryUrls.cs | 31 +++++ .../build/Microsoft.SourceLink.GitWeb.targets | 65 +++++++++ .../Microsoft.SourceLink.GitWeb.targets | 4 + src/SourceLink.GitWeb/xlf/Resources.cs.xlf | 12 ++ src/SourceLink.GitWeb/xlf/Resources.de.xlf | 12 ++ src/SourceLink.GitWeb/xlf/Resources.es.xlf | 12 ++ src/SourceLink.GitWeb/xlf/Resources.fr.xlf | 12 ++ src/SourceLink.GitWeb/xlf/Resources.it.xlf | 12 ++ src/SourceLink.GitWeb/xlf/Resources.ja.xlf | 12 ++ src/SourceLink.GitWeb/xlf/Resources.ko.xlf | 12 ++ src/SourceLink.GitWeb/xlf/Resources.pl.xlf | 12 ++ src/SourceLink.GitWeb/xlf/Resources.pt-BR.xlf | 12 ++ src/SourceLink.GitWeb/xlf/Resources.ru.xlf | 12 ++ src/SourceLink.GitWeb/xlf/Resources.tr.xlf | 12 ++ .../xlf/Resources.zh-Hans.xlf | 12 ++ .../xlf/Resources.zh-Hant.xlf | 12 ++ 27 files changed, 758 insertions(+), 10 deletions(-) create mode 100644 src/SourceLink.Git.IntegrationTests/GitWebTests.cs create mode 100644 src/SourceLink.GitWeb.UnitTests/GetSourceLinkUrlTests.cs create mode 100644 src/SourceLink.GitWeb.UnitTests/Microsoft.SourceLink.GitWeb.UnitTests.csproj create mode 100644 src/SourceLink.GitWeb.UnitTests/TranslateRepositoryUrlsTests.cs create mode 100644 src/SourceLink.GitWeb/GetSourceLinkUrl.cs create mode 100644 src/SourceLink.GitWeb/Microsoft.SourceLink.GitWeb.csproj create mode 100644 src/SourceLink.GitWeb/Microsoft.SourceLink.GitWeb.nuspec create mode 100644 src/SourceLink.GitWeb/Resources.resx create mode 100644 src/SourceLink.GitWeb/TranslateRepositoryUrls.cs create mode 100644 src/SourceLink.GitWeb/build/Microsoft.SourceLink.GitWeb.targets create mode 100644 src/SourceLink.GitWeb/buildMultiTargeting/Microsoft.SourceLink.GitWeb.targets create mode 100644 src/SourceLink.GitWeb/xlf/Resources.cs.xlf create mode 100644 src/SourceLink.GitWeb/xlf/Resources.de.xlf create mode 100644 src/SourceLink.GitWeb/xlf/Resources.es.xlf create mode 100644 src/SourceLink.GitWeb/xlf/Resources.fr.xlf create mode 100644 src/SourceLink.GitWeb/xlf/Resources.it.xlf create mode 100644 src/SourceLink.GitWeb/xlf/Resources.ja.xlf create mode 100644 src/SourceLink.GitWeb/xlf/Resources.ko.xlf create mode 100644 src/SourceLink.GitWeb/xlf/Resources.pl.xlf create mode 100644 src/SourceLink.GitWeb/xlf/Resources.pt-BR.xlf create mode 100644 src/SourceLink.GitWeb/xlf/Resources.ru.xlf create mode 100644 src/SourceLink.GitWeb/xlf/Resources.tr.xlf create mode 100644 src/SourceLink.GitWeb/xlf/Resources.zh-Hans.xlf create mode 100644 src/SourceLink.GitWeb/xlf/Resources.zh-Hant.xlf diff --git a/SourceLink.sln b/SourceLink.sln index 49098d14..b185660e 100644 --- a/SourceLink.sln +++ b/SourceLink.sln @@ -54,6 +54,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.SourceLink.AzureD EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.SourceLink.AzureDevOpsServer.Git.UnitTests", "src\SourceLink.AzureDevOpsServer.Git.UnitTests\Microsoft.SourceLink.AzureDevOpsServer.Git.UnitTests.csproj", "{79371F26-FB84-408D-A4A1-B142B247C288}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.SourceLink.GitWeb", "src\SourceLink.GitWeb\Microsoft.SourceLink.GitWeb.csproj", "{C78DD3EF-9D20-4E00-8237-E871BB53F840}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.SourceLink.GitWeb.UnitTests", "src\SourceLink.GitWeb.UnitTests\Microsoft.SourceLink.GitWeb.UnitTests.csproj", "{50503A43-08C0-493B-B8CC-F368983644C1}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -136,6 +140,14 @@ Global {79371F26-FB84-408D-A4A1-B142B247C288}.Debug|Any CPU.Build.0 = Debug|Any CPU {79371F26-FB84-408D-A4A1-B142B247C288}.Release|Any CPU.ActiveCfg = Release|Any CPU {79371F26-FB84-408D-A4A1-B142B247C288}.Release|Any CPU.Build.0 = Release|Any CPU + {C78DD3EF-9D20-4E00-8237-E871BB53F840}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C78DD3EF-9D20-4E00-8237-E871BB53F840}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C78DD3EF-9D20-4E00-8237-E871BB53F840}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C78DD3EF-9D20-4E00-8237-E871BB53F840}.Release|Any CPU.Build.0 = Release|Any CPU + {50503A43-08C0-493B-B8CC-F368983644C1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {50503A43-08C0-493B-B8CC-F368983644C1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {50503A43-08C0-493B-B8CC-F368983644C1}.Release|Any CPU.ActiveCfg = Release|Any CPU + {50503A43-08C0-493B-B8CC-F368983644C1}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/Common/TranslateRepositoryUrlGitTask.cs b/src/Common/TranslateRepositoryUrlGitTask.cs index b87b2d70..02c0e032 100644 --- a/src/Common/TranslateRepositoryUrlGitTask.cs +++ b/src/Common/TranslateRepositoryUrlGitTask.cs @@ -1,4 +1,5 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See +// License.txt in the project root for license information. using System; using System.Collections.Generic; @@ -48,13 +49,13 @@ private void ExecuteImpl() } static bool isMatchingHostUri(Uri hostUri, Uri uri) - => uri.GetHost().Equals(hostUri.GetHost(), StringComparison.OrdinalIgnoreCase) || + => uri.GetHost().Equals(hostUri.GetHost(), StringComparison.OrdinalIgnoreCase) || uri.GetHost().EndsWith("." + hostUri.GetHost(), StringComparison.OrdinalIgnoreCase); // only need to translate valid ssh URLs that match one of our hosts: string? translate(string? url) { - if (Uri.TryCreate(url, UriKind.Absolute, out var uri) && + if (Uri.TryCreate(url, UriKind.Absolute, out var uri) && hostUris.Any(h => isMatchingHostUri(h, uri))) { return (uri.Scheme.Equals("http", StringComparison.OrdinalIgnoreCase) ? TranslateHttpUrl(uri) : @@ -66,7 +67,16 @@ static bool isMatchingHostUri(Uri hostUri, Uri uri) return url; } - TranslatedRepositoryUrl = translate(RepositoryUrl); + try + { + TranslatedRepositoryUrl = translate(RepositoryUrl); + } + catch (NotSupportedException e) + { + Log.LogError(e.Message); + return; + } + TranslatedSourceRoots = SourceRoots; if (TranslatedSourceRoots != null) @@ -78,12 +88,23 @@ static bool isMatchingHostUri(Uri hostUri, Uri uri) continue; } - // Item metadata are stored msbuild-escaped. GetMetadata unescapes, SetMetadata stores the value as specified. - // When initializing the URL metadata from git information we msbuild-escaped the URL to preserve any URL escapes in it. - // Here, GetMetadata unescapes the msbuild escapes, then we translate the URL and finally msbuild-escape - // the resulting URL to preserve any URL escapes. - sourceRoot.SetMetadata(Names.SourceRoot.ScmRepositoryUrl, - Evaluation.ProjectCollection.Escape(translate(sourceRoot.GetMetadata(Names.SourceRoot.ScmRepositoryUrl)))); + string? translatedUrl; + try + { + translatedUrl = translate(sourceRoot.GetMetadata(Names.SourceRoot.ScmRepositoryUrl)); + } + catch (NotSupportedException e) + { + Log.LogError(e.Message); + continue; + } + + // Item metadata are stored msbuild-escaped. GetMetadata unescapes, SetMetadata + // stores the value as specified. When initializing the URL metadata from git + // information we msbuild-escaped the URL to preserve any URL escapes in it. + // Here, GetMetadata unescapes the msbuild escapes, then we translate the URL + // and finally msbuild-escape the resulting URL to preserve any URL escapes. + sourceRoot.SetMetadata(Names.SourceRoot.ScmRepositoryUrl, Evaluation.ProjectCollection.Escape(translatedUrl)); } } } diff --git a/src/SourceLink.Git.IntegrationTests/GitWebTests.cs b/src/SourceLink.Git.IntegrationTests/GitWebTests.cs new file mode 100644 index 00000000..b0958d59 --- /dev/null +++ b/src/SourceLink.Git.IntegrationTests/GitWebTests.cs @@ -0,0 +1,130 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See +// License.txt in the project root for license information. + +using Microsoft.Build.Tasks.SourceControl; +using System; +using System.IO; +using TestUtilities; + +namespace Microsoft.SourceLink.IntegrationTests +{ + public class GitWebTests : DotNetSdkTestBase + { + public GitWebTests() + : base("Microsoft.SourceLink.GitWeb") + { + } + + [ConditionalFact(typeof(DotNetSdkAvailable))] + public void FullValidation_Ssh() + { + // Test non-ascii characters and escapes in the URL. Escaped URI reserved characters + // should remain escaped, non-reserved characters unescaped in the results. + var repoUrl = $"ssh://git@噸.com/test-%72epo\u1234%24%2572%2F.git"; + var repoName = "test-repo\u1234%24%2572%2F.git"; + + var repo = GitUtilities.CreateGitRepositoryWithSingleCommit(ProjectDir.Path, new[] { ProjectFileName }, repoUrl); + var commitSha = repo.Head.Tip.Sha; + + VerifyValues( + customProps: @" + + true + + + + +", + customTargets: "", + targets: new[] + { + "Build", "Pack" + }, + expressions: new[] + { + "@(SourceRoot)", + "@(SourceRoot->'%(SourceLinkUrl)')", + "$(SourceLink)", + "$(PrivateRepositoryUrl)", + "$(RepositoryUrl)" + }, + expectedResults: new[] + { + ProjectSourceRoot, + $"https://噸.com/gitweb/?p={repoName};a=blob_plain;hb={commitSha};f=*", + s_relativeSourceLinkJsonPath, + $"ssh://git@噸.com/{repoName}", + $"ssh://git@噸.com/{repoName}" + }); + + AssertEx.AreEqual( + $@"{{""documents"":{{""{ProjectSourceRoot.Replace(@"\", @"\\")}*"":""https://噸.com/gitweb/?p={repoName};a=blob_plain;hb={commitSha};f=*""}}}}", + File.ReadAllText(Path.Combine(ProjectDir.Path, s_relativeSourceLinkJsonPath))); + + TestUtilities.ValidateAssemblyInformationalVersion( + Path.Combine(ProjectDir.Path, s_relativeOutputFilePath), + "1.0.0+" + commitSha); + + TestUtilities.ValidateNuSpecRepository( + Path.Combine(ProjectDir.Path, s_relativePackagePath), + type: "git", + commit: commitSha, + url: $"ssh://git@噸.com/{repoName}"); + } + + [ConditionalFact(typeof(DotNetSdkAvailable))] + public void Issues_error_on_git_url() + { + var repoUrl = "git://噸.com/invalid_url_protocol.git"; + var repo = GitUtilities.CreateGitRepositoryWithSingleCommit(ProjectDir.Path, new[] { ProjectFileName }, repoUrl); + var commitSha = repo.Head.Tip.Sha; + + VerifyValues( + customProps: @" + + true + + + + +", + customTargets: "", + targets: new[] + { + "Build", "Pack" + }, + expressions: Array.Empty(), + expectedErrors: new[]{ + string.Format(GitWeb.Resources.RepositoryUrlIsNotSupportedByProvider, "GIT") + }); + } + + [ConditionalFact(typeof(DotNetSdkAvailable))] + public void Issues_error_on_https_url() + { + var repoUrl = "https://噸.com/invalid_url_protocol.git"; + var repo = GitUtilities.CreateGitRepositoryWithSingleCommit(ProjectDir.Path, new[] { ProjectFileName }, repoUrl); + var commitSha = repo.Head.Tip.Sha; + + VerifyValues( + customProps: @" + + true + + + + +", + customTargets: "", + targets: new[] + { + "Build", "Pack" + }, + expressions: Array.Empty(), + expectedErrors: new[] + { + string.Format(GitWeb.Resources.RepositoryUrlIsNotSupportedByProvider, "HTTP") + }); + } + } +} diff --git a/src/SourceLink.Git.IntegrationTests/Microsoft.SourceLink.Git.IntegrationTests.csproj b/src/SourceLink.Git.IntegrationTests/Microsoft.SourceLink.Git.IntegrationTests.csproj index efc7ea5d..a3f1755e 100644 --- a/src/SourceLink.Git.IntegrationTests/Microsoft.SourceLink.Git.IntegrationTests.csproj +++ b/src/SourceLink.Git.IntegrationTests/Microsoft.SourceLink.Git.IntegrationTests.csproj @@ -7,6 +7,7 @@ + diff --git a/src/SourceLink.GitWeb.UnitTests/GetSourceLinkUrlTests.cs b/src/SourceLink.GitWeb.UnitTests/GetSourceLinkUrlTests.cs new file mode 100644 index 00000000..b215d1a0 --- /dev/null +++ b/src/SourceLink.GitWeb.UnitTests/GetSourceLinkUrlTests.cs @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See +// License.txt in the project root for license information. +using Microsoft.Build.Tasks.SourceControl; +using TestUtilities; +using Xunit; +using static TestUtilities.KeyValuePairUtils; + +namespace Microsoft.SourceLink.GitWeb.UnitTests +{ + public class GetSourceLinkUrlTests + { + [Fact] + public void EmptyHosts() + { + var engine = new MockEngine(); + + var task = new GetSourceLinkUrl() + { + BuildEngine = engine, + SourceRoot = new MockItem("x", KVP("RepositoryUrl", "http://abc.com"), KVP("SourceControl", "git")), + }; + + bool result = task.Execute(); + + AssertEx.AssertEqualToleratingWhitespaceDifferences( + "ERROR : " + string.Format(CommonResources.AtLeastOneRepositoryHostIsRequired, "SourceLinkGitWebHost", "GitWeb"), engine.Log); + + Assert.False(result); + } + + [Theory] + [InlineData("", "")] + [InlineData("", "/")] + [InlineData("/", "")] + [InlineData("/", "/")] + public void BuildSourceLinkUrl(string s1, string s2) + { + var engine = new MockEngine(); + + var task = new GetSourceLinkUrl() + { + BuildEngine = engine, + SourceRoot = new MockItem("/src/", KVP("RepositoryUrl", "ssh://git@src.intranet.company.com/root_dir_name/sub_dirs/reponame.git" + s1), KVP("SourceControl", "git"), KVP("RevisionId", "0123456789abcdefABCDEF000000000000000000")), + Hosts = new[] + { + new MockItem("src.intranet.company.com", KVP("ContentUrl", "https://src.intranet.company.com/gitweb" + s2)), + } + }; + + bool result = task.Execute(); + AssertEx.AssertEqualToleratingWhitespaceDifferences("", engine.Log); + AssertEx.AreEqual("https://src.intranet.company.com/gitweb/?p=root_dir_name/sub_dirs/reponame.git;a=blob_plain;hb=0123456789abcdefABCDEF000000000000000000;f=*", task.SourceLinkUrl); + Assert.True(result); + } + } +} diff --git a/src/SourceLink.GitWeb.UnitTests/Microsoft.SourceLink.GitWeb.UnitTests.csproj b/src/SourceLink.GitWeb.UnitTests/Microsoft.SourceLink.GitWeb.UnitTests.csproj new file mode 100644 index 00000000..16608152 --- /dev/null +++ b/src/SourceLink.GitWeb.UnitTests/Microsoft.SourceLink.GitWeb.UnitTests.csproj @@ -0,0 +1,9 @@ + + + net461;netcoreapp2.0 + + + + + + diff --git a/src/SourceLink.GitWeb.UnitTests/TranslateRepositoryUrlsTests.cs b/src/SourceLink.GitWeb.UnitTests/TranslateRepositoryUrlsTests.cs new file mode 100644 index 00000000..255fdb2f --- /dev/null +++ b/src/SourceLink.GitWeb.UnitTests/TranslateRepositoryUrlsTests.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See +// License.txt in the project root for license information. +using System.Linq; +using TestUtilities; +using Xunit; +using static TestUtilities.KeyValuePairUtils; + +namespace Microsoft.SourceLink.GitWeb.UnitTests +{ + public class TranslateRepositoryUrlsTests + { + [Fact] + public void Translate() + { + var engine = new MockEngine(); + + var task = new TranslateRepositoryUrls() + { + BuildEngine = engine, + RepositoryUrl = "ssh://git@src.intranet.company.com/root_dir_name/sub_dirs/reponame.git", + IsSingleProvider = true, + SourceRoots = new[] + { + new MockItem("/1/", KVP("SourceControl", "git"), KVP("ScmRepositoryUrl", "ssh://git@src.intranet.company.com/root_dir_name/sub_dirs/reponame.git")), + new MockItem("/2/", KVP("SourceControl", "tfvc"), KVP("ScmRepositoryUrl", "ssh://git@src.intranet.company1.com/root_dir_name/sub_dirs/reponame.git")), + new MockItem("/2/", KVP("SourceControl", "git"), KVP("ScmRepositoryUrl", "ssh://git@src.intranet.company1.com/root_dir_name/sub_dirs/reponame.git")), + new MockItem("/2/", KVP("SourceControl", "tfvc"), KVP("ScmRepositoryUrl", "ssh://git@src.intranet.company2.com/root_dir_name/sub_dirs/reponame.git")), + }, + Hosts = new[] + { + new MockItem("src.intranet.company1.com") + } + }; + + bool result = task.Execute(); + AssertEx.AssertEqualToleratingWhitespaceDifferences("", engine.Log); + + AssertEx.AreEqual("ssh://git@src.intranet.company.com/root_dir_name/sub_dirs/reponame.git", task.TranslatedRepositoryUrl); + + AssertEx.Equal(new[] + { + "ssh://git@src.intranet.company.com/root_dir_name/sub_dirs/reponame.git", + "ssh://git@src.intranet.company1.com/root_dir_name/sub_dirs/reponame.git", + "ssh://git@src.intranet.company1.com/root_dir_name/sub_dirs/reponame.git", + "ssh://git@src.intranet.company2.com/root_dir_name/sub_dirs/reponame.git", + }, task.TranslatedSourceRoots.Select(r => r.GetMetadata("ScmRepositoryUrl"))); + + Assert.True(result); + } + } +} diff --git a/src/SourceLink.GitWeb/GetSourceLinkUrl.cs b/src/SourceLink.GitWeb/GetSourceLinkUrl.cs new file mode 100644 index 00000000..da5df319 --- /dev/null +++ b/src/SourceLink.GitWeb/GetSourceLinkUrl.cs @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See +// License.txt in the project root for license information. + +using System; +using Microsoft.Build.Framework; +using Microsoft.Build.Tasks.SourceControl; + +namespace Microsoft.SourceLink.GitWeb +{ + /// + /// The task calculates SourceLink URL for a given SourceRoot. If the SourceRoot is associated + /// with a git repository with a recognized domain the output + /// property is set to the content URL corresponding to the domain, otherwise it is set to + /// string "N/A". + /// + public sealed class GetSourceLinkUrl : GetSourceLinkUrlGitTask + { + protected override string HostsItemGroupName => "SourceLinkGitWebHost"; + protected override string ProviderDisplayName => "GitWeb"; + + protected override Uri GetDefaultContentUriFromHostUri(string authority, Uri gitUri) + => new Uri($"https://{authority}/gitweb", UriKind.Absolute); + + protected override string BuildSourceLinkUrl(Uri contentUri, Uri gitUri, string relativeUrl, string revisionId, ITaskItem? hostItem) + { + var trimLeadingSlash = relativeUrl.TrimStart('/'); + var trimmedContentUrl = contentUri.ToString().TrimEnd('/', '\\'); + + // p = project/path + // a = action + // hb = SHA/revision + // f = repo file path + var gitwebRawUrl = UriUtilities.Combine(trimmedContentUrl, $"?p={trimLeadingSlash}.git;a=blob_plain;hb={revisionId};f=*"); + return gitwebRawUrl; + } + } +} diff --git a/src/SourceLink.GitWeb/Microsoft.SourceLink.GitWeb.csproj b/src/SourceLink.GitWeb/Microsoft.SourceLink.GitWeb.csproj new file mode 100644 index 00000000..d9fb3080 --- /dev/null +++ b/src/SourceLink.GitWeb/Microsoft.SourceLink.GitWeb.csproj @@ -0,0 +1,35 @@ + + + net461;netcoreapp2.0 + true + + + true + $(MSBuildProjectName).nuspec + $(OutputPath) + + Generates source link for Git repositories using a GitWeb server. + MSBuild Tasks GitWeb source link + true + + + + + + + + + Microsoft.Build.Tasks.SourceControl + true + + + + + + + + + + + + diff --git a/src/SourceLink.GitWeb/Microsoft.SourceLink.GitWeb.nuspec b/src/SourceLink.GitWeb/Microsoft.SourceLink.GitWeb.nuspec new file mode 100644 index 00000000..5acaea90 --- /dev/null +++ b/src/SourceLink.GitWeb/Microsoft.SourceLink.GitWeb.nuspec @@ -0,0 +1,17 @@ + + + + $CommonMetadataElements$ + + + + + + + $CommonFileElements$ + + + + + + diff --git a/src/SourceLink.GitWeb/Resources.resx b/src/SourceLink.GitWeb/Resources.resx new file mode 100644 index 00000000..143a5dc8 --- /dev/null +++ b/src/SourceLink.GitWeb/Resources.resx @@ -0,0 +1,123 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + + \ No newline at end of file diff --git a/src/SourceLink.GitWeb/TranslateRepositoryUrls.cs b/src/SourceLink.GitWeb/TranslateRepositoryUrls.cs new file mode 100644 index 00000000..591cd7d4 --- /dev/null +++ b/src/SourceLink.GitWeb/TranslateRepositoryUrls.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See +// License.txt in the project root for license information. + +using Microsoft.Build.Tasks.SourceControl; +using System; + +namespace Microsoft.SourceLink.GitWeb +{ + /// + /// This task normally translates all protocols to HTTPS git repo URLs for consistency. Not all + /// GitWebs support HTTPS URLs for clones. So instead we keep the SSH URL. The + /// Task converts the URLs to HTTP content URLs which GitWeb does + /// support. The output of this Task is independent of . + /// + public sealed class TranslateRepositoryUrls : TranslateRepositoryUrlsGitTask + { + /// + /// Keep the SSH URL. It is the only protocol currently known to work with GitWeb. + /// + /// + /// as a string + protected override string TranslateSshUrl(Uri uri) + => uri.ToString(); + + protected override string TranslateGitUrl(Uri uri) + => throw new NotSupportedException(string.Format(Resources.RepositoryUrlIsNotSupportedByProvider, "GIT")); + + protected override string TranslateHttpUrl(Uri uri) + => throw new NotSupportedException(string.Format(Resources.RepositoryUrlIsNotSupportedByProvider, "HTTP")); + } +} diff --git a/src/SourceLink.GitWeb/build/Microsoft.SourceLink.GitWeb.targets b/src/SourceLink.GitWeb/build/Microsoft.SourceLink.GitWeb.targets new file mode 100644 index 00000000..d5019e7a --- /dev/null +++ b/src/SourceLink.GitWeb/build/Microsoft.SourceLink.GitWeb.targets @@ -0,0 +1,65 @@ + + + + <_SourceLinkGitWebAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net461\Microsoft.SourceLink.GitWeb.dll + <_SourceLinkGitWebAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\netcoreapp2.0\Microsoft.SourceLink.GitWeb.dll + + + + + + + $(SourceLinkUrlInitializerTargets);_InitializeGitWebSourceLinkUrl + $(SourceControlManagerUrlTranslationTargets);TranslateGitWebUrlsInSourceControlInformation + + + + + + + + + + + + + + + + + + + <_TranslatedSourceRoot Remove="@(_TranslatedSourceRoot)" /> + + + + + + + + + + + + + diff --git a/src/SourceLink.GitWeb/buildMultiTargeting/Microsoft.SourceLink.GitWeb.targets b/src/SourceLink.GitWeb/buildMultiTargeting/Microsoft.SourceLink.GitWeb.targets new file mode 100644 index 00000000..84afbc37 --- /dev/null +++ b/src/SourceLink.GitWeb/buildMultiTargeting/Microsoft.SourceLink.GitWeb.targets @@ -0,0 +1,4 @@ + + + + diff --git a/src/SourceLink.GitWeb/xlf/Resources.cs.xlf b/src/SourceLink.GitWeb/xlf/Resources.cs.xlf new file mode 100644 index 00000000..a83a11e5 --- /dev/null +++ b/src/SourceLink.GitWeb/xlf/Resources.cs.xlf @@ -0,0 +1,12 @@ + + + + + + The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + + + + + \ No newline at end of file diff --git a/src/SourceLink.GitWeb/xlf/Resources.de.xlf b/src/SourceLink.GitWeb/xlf/Resources.de.xlf new file mode 100644 index 00000000..f041b7e6 --- /dev/null +++ b/src/SourceLink.GitWeb/xlf/Resources.de.xlf @@ -0,0 +1,12 @@ + + + + + + The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + + + + + \ No newline at end of file diff --git a/src/SourceLink.GitWeb/xlf/Resources.es.xlf b/src/SourceLink.GitWeb/xlf/Resources.es.xlf new file mode 100644 index 00000000..8e1afbd3 --- /dev/null +++ b/src/SourceLink.GitWeb/xlf/Resources.es.xlf @@ -0,0 +1,12 @@ + + + + + + The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + + + + + \ No newline at end of file diff --git a/src/SourceLink.GitWeb/xlf/Resources.fr.xlf b/src/SourceLink.GitWeb/xlf/Resources.fr.xlf new file mode 100644 index 00000000..afec8a14 --- /dev/null +++ b/src/SourceLink.GitWeb/xlf/Resources.fr.xlf @@ -0,0 +1,12 @@ + + + + + + The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + + + + + \ No newline at end of file diff --git a/src/SourceLink.GitWeb/xlf/Resources.it.xlf b/src/SourceLink.GitWeb/xlf/Resources.it.xlf new file mode 100644 index 00000000..08d11f5e --- /dev/null +++ b/src/SourceLink.GitWeb/xlf/Resources.it.xlf @@ -0,0 +1,12 @@ + + + + + + The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + + + + + \ No newline at end of file diff --git a/src/SourceLink.GitWeb/xlf/Resources.ja.xlf b/src/SourceLink.GitWeb/xlf/Resources.ja.xlf new file mode 100644 index 00000000..b70c68a7 --- /dev/null +++ b/src/SourceLink.GitWeb/xlf/Resources.ja.xlf @@ -0,0 +1,12 @@ + + + + + + The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + + + + + \ No newline at end of file diff --git a/src/SourceLink.GitWeb/xlf/Resources.ko.xlf b/src/SourceLink.GitWeb/xlf/Resources.ko.xlf new file mode 100644 index 00000000..d17cb1ad --- /dev/null +++ b/src/SourceLink.GitWeb/xlf/Resources.ko.xlf @@ -0,0 +1,12 @@ + + + + + + The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + + + + + \ No newline at end of file diff --git a/src/SourceLink.GitWeb/xlf/Resources.pl.xlf b/src/SourceLink.GitWeb/xlf/Resources.pl.xlf new file mode 100644 index 00000000..4531f455 --- /dev/null +++ b/src/SourceLink.GitWeb/xlf/Resources.pl.xlf @@ -0,0 +1,12 @@ + + + + + + The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + + + + + \ No newline at end of file diff --git a/src/SourceLink.GitWeb/xlf/Resources.pt-BR.xlf b/src/SourceLink.GitWeb/xlf/Resources.pt-BR.xlf new file mode 100644 index 00000000..864e07dd --- /dev/null +++ b/src/SourceLink.GitWeb/xlf/Resources.pt-BR.xlf @@ -0,0 +1,12 @@ + + + + + + The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + + + + + \ No newline at end of file diff --git a/src/SourceLink.GitWeb/xlf/Resources.ru.xlf b/src/SourceLink.GitWeb/xlf/Resources.ru.xlf new file mode 100644 index 00000000..240cb3c9 --- /dev/null +++ b/src/SourceLink.GitWeb/xlf/Resources.ru.xlf @@ -0,0 +1,12 @@ + + + + + + The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + + + + + \ No newline at end of file diff --git a/src/SourceLink.GitWeb/xlf/Resources.tr.xlf b/src/SourceLink.GitWeb/xlf/Resources.tr.xlf new file mode 100644 index 00000000..8581c836 --- /dev/null +++ b/src/SourceLink.GitWeb/xlf/Resources.tr.xlf @@ -0,0 +1,12 @@ + + + + + + The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + + + + + \ No newline at end of file diff --git a/src/SourceLink.GitWeb/xlf/Resources.zh-Hans.xlf b/src/SourceLink.GitWeb/xlf/Resources.zh-Hans.xlf new file mode 100644 index 00000000..44b283db --- /dev/null +++ b/src/SourceLink.GitWeb/xlf/Resources.zh-Hans.xlf @@ -0,0 +1,12 @@ + + + + + + The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + + + + + \ No newline at end of file diff --git a/src/SourceLink.GitWeb/xlf/Resources.zh-Hant.xlf b/src/SourceLink.GitWeb/xlf/Resources.zh-Hant.xlf new file mode 100644 index 00000000..878bd818 --- /dev/null +++ b/src/SourceLink.GitWeb/xlf/Resources.zh-Hant.xlf @@ -0,0 +1,12 @@ + + + + + + The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + + + + + \ No newline at end of file From f17578b85fd8b2acf2d530df6ac776f6154b1c1c Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 17 Feb 2020 13:11:21 +0000 Subject: [PATCH 058/161] Update dependencies from https://github.com/dotnet/arcade build 20200216.1 (#577) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.20116.1 --- eng/Version.Details.xml | 4 +- eng/common/cross/toolchain.cmake | 12 ++++- eng/common/post-build/darc-gather-drop.ps1 | 44 ------------------- eng/common/post-build/nuget-validation.ps1 | 2 +- .../templates/post-build/darc-gather-drop.yml | 23 ---------- .../templates/post-build/post-build.yml | 26 +++++++++++ eng/common/tools.ps1 | 7 ++- eng/common/tools.sh | 2 +- global.json | 2 +- 9 files changed, 48 insertions(+), 74 deletions(-) delete mode 100644 eng/common/post-build/darc-gather-drop.ps1 delete mode 100644 eng/common/templates/post-build/darc-gather-drop.yml diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index b6627d7a..a649aa64 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - b0e8d3944155f94f83deea8afe025debe369e69f + 2086e534f12e6116889ee480646ef54c1f781887 diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index 0eea7d1d..556da2e1 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -71,7 +71,17 @@ endif() if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$") add_compile_options(-mthumb) - add_compile_options(-mfpu=vfpv3) + if (NOT DEFINED CLR_ARM_FPU_TYPE) + set (CLR_ARM_FPU_TYPE vfpv3) + endif (NOT DEFINED CLR_ARM_FPU_TYPE) + + add_compile_options (-mfpu=${CLR_ARM_FPU_TYPE}) + if (NOT DEFINED CLR_ARM_FPU_CAPABILITY) + set (CLR_ARM_FPU_CAPABILITY 0x7) + endif (NOT DEFINED CLR_ARM_FPU_CAPABILITY) + + add_definitions (-DCLR_ARM_FPU_CAPABILITY=${CLR_ARM_FPU_CAPABILITY}) + if(TARGET_ARCH_NAME STREQUAL "armel") add_compile_options(-mfloat-abi=softfp) if(DEFINED TIZEN_TOOLCHAIN) diff --git a/eng/common/post-build/darc-gather-drop.ps1 b/eng/common/post-build/darc-gather-drop.ps1 deleted file mode 100644 index 81ff2a43..00000000 --- a/eng/common/post-build/darc-gather-drop.ps1 +++ /dev/null @@ -1,44 +0,0 @@ -param( - [Parameter(Mandatory=$true)][int] $BarBuildId, # ID of the build which assets should be downloaded - [Parameter(Mandatory=$true)][string] $DropLocation, # Where the assets should be downloaded to - [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken, # Token used to access Maestro API - [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com', # Maestro API URL - [Parameter(Mandatory=$false)][string] $MaestroApiVersion = '2019-01-16' # Version of Maestro API to use -) - -try { - . $PSScriptRoot\post-build-utils.ps1 - - Write-Host 'Installing DARC ...' - - . $PSScriptRoot\..\darc-init.ps1 - $exitCode = $LASTEXITCODE - - if ($exitCode -ne 0) { - Write-PipelineTelemetryError -Category "Darc" -Message "Something failed while running 'darc-init.ps1'. Check for errors above. Exiting now..." - ExitWithExitCode $exitCode - } - - # For now, only use a dry run. - # Ideally we would change darc to enable a quick request that - # would check whether the file exists that you can download it, - # and that it won't conflict with other files. - # https://github.com/dotnet/arcade/issues/3674 - # Right now we can't remove continue-on-error because we ocassionally will have - # dependencies that have no associated builds (e.g. an old dependency). - # We need to add an option to baseline specific dependencies away, or add them manually - # to the BAR. - darc gather-drop --non-shipping ` - --dry-run ` - --continue-on-error ` - --id $BarBuildId ` - --output-dir $DropLocation ` - --bar-uri $MaestroApiEndpoint ` - --password $MaestroApiAccessToken ` - --latest-location -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category "Darc" -Message $_ - ExitWithExitCode 1 -} diff --git a/eng/common/post-build/nuget-validation.ps1 b/eng/common/post-build/nuget-validation.ps1 index 3d6129d7..dab3534a 100644 --- a/eng/common/post-build/nuget-validation.ps1 +++ b/eng/common/post-build/nuget-validation.ps1 @@ -9,7 +9,7 @@ param( try { . $PSScriptRoot\post-build-utils.ps1 - $url = 'https://raw.githubusercontent.com/NuGet/NuGetGallery/jver-verify/src/VerifyMicrosoftPackage/verify.ps1' + $url = 'https://raw.githubusercontent.com/NuGet/NuGetGallery/3e25ad135146676bcab0050a516939d9958bfa5d/src/VerifyMicrosoftPackage/verify.ps1' New-Item -ItemType 'directory' -Path ${ToolDestinationPath} -Force diff --git a/eng/common/templates/post-build/darc-gather-drop.yml b/eng/common/templates/post-build/darc-gather-drop.yml deleted file mode 100644 index 3268ccaa..00000000 --- a/eng/common/templates/post-build/darc-gather-drop.yml +++ /dev/null @@ -1,23 +0,0 @@ -parameters: - ChannelId: 0 - -jobs: -- job: gatherDrop - displayName: Gather Drop - dependsOn: setupMaestroVars - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.ChannelId }})) - variables: - - name: BARBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] - pool: - vmImage: 'windows-2019' - steps: - - task: PowerShell@2 - displayName: Darc gather-drop - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/darc-gather-drop.ps1 - arguments: -BarBuildId $(BARBuildId) - -DropLocation $(Agent.BuildDirectory)/Temp/Drop/ - -MaestroApiAccessToken $(MaestroApiAccessToken) - -MaestroApiEndPoint $(MaestroApiEndPoint) - -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 70cb65f5..eff0d9ac 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -170,6 +170,32 @@ stages: shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' +- template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'Net5_Preview1_Publish' + channelName: '.NET 5 Preview 1' + channelId: 737 + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' + +- template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'Net5_Preview2_Publish' + channelName: '.NET 5 Preview 2' + channelId: 738 + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index fef7e5a3..a45302b3 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -399,7 +399,12 @@ function LocateVisualStudio([object]$vsRequirements = $null){ if (!(Test-Path $vsWhereExe)) { Create-Directory $vsWhereDir Write-Host 'Downloading vswhere' - Invoke-WebRequest "https://github.com/Microsoft/vswhere/releases/download/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe + try { + Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe + } + catch { + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_ + } } if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } diff --git a/eng/common/tools.sh b/eng/common/tools.sh index c26525e5..664ac105 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -252,7 +252,7 @@ function GetDotNetInstallScript { # Use curl if available, otherwise use wget if command -v curl > /dev/null; then - with_retries curl "$install_script_url" -isSLv --retry 10 --create-dirs -o "$install_script" || { + with_retries curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || { local exit_code=$? Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." ExitWithExitCode $exit_code diff --git a/global.json b/global.json index d9868069..4df094c4 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.1.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20109.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20116.1" } } From a7754c9c0a0fb14c10dae98138d127c3d7679ffd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Matou=C5=A1ek?= Date: Mon, 17 Feb 2020 10:56:49 -0800 Subject: [PATCH 059/161] Dispose TempRoot (#578) --- src/Microsoft.Build.Tasks.Git.UnitTests/GitDataTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.Build.Tasks.Git.UnitTests/GitDataTests.cs b/src/Microsoft.Build.Tasks.Git.UnitTests/GitDataTests.cs index d41ebdd0..f179ae64 100644 --- a/src/Microsoft.Build.Tasks.Git.UnitTests/GitDataTests.cs +++ b/src/Microsoft.Build.Tasks.Git.UnitTests/GitDataTests.cs @@ -10,13 +10,13 @@ namespace Microsoft.Build.Tasks.Git.UnitTests { public class GitDataTests { - public readonly TempRoot Temp = new TempRoot(); private static readonly char s = Path.DirectorySeparatorChar; [Fact] public void MinimalGitData() { - var repoDir = Temp.CreateDirectory(); + using var temp = new TempRoot(); + var repoDir = temp.CreateDirectory(); var gitDir = repoDir.CreateDirectory(".git"); gitDir.CreateFile("HEAD").WriteAllText("1111111111111111111111111111111111111111"); From 336c5b6d0f42e3c763f08b0f640677f2fd9474af Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 24 Feb 2020 13:26:44 +0000 Subject: [PATCH 060/161] Update dependencies from https://github.com/dotnet/arcade build 20200223.1 (#584) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.20123.1 --- eng/Version.Details.xml | 4 +- eng/common/CheckSymbols.ps1 | 159 -------------- eng/common/PublishToPackageFeed.proj | 83 -------- eng/common/PublishToSymbolServers.proj | 84 -------- eng/common/SourceLinkValidation.ps1 | 195 ------------------ .../post-build/check-channel-consistency.ps1 | 25 +++ .../channels/generic-public-channel.yml | 4 +- .../templates/post-build/post-build.yml | 58 ++++-- global.json | 2 +- 9 files changed, 77 insertions(+), 537 deletions(-) delete mode 100644 eng/common/CheckSymbols.ps1 delete mode 100644 eng/common/PublishToPackageFeed.proj delete mode 100644 eng/common/PublishToSymbolServers.proj delete mode 100644 eng/common/SourceLinkValidation.ps1 create mode 100644 eng/common/post-build/check-channel-consistency.ps1 diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index a649aa64..611d35ba 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 2086e534f12e6116889ee480646ef54c1f781887 + 85d76351b1f0245c9f331f72219d12e8e2d48e72 diff --git a/eng/common/CheckSymbols.ps1 b/eng/common/CheckSymbols.ps1 deleted file mode 100644 index 5442eff3..00000000 --- a/eng/common/CheckSymbols.ps1 +++ /dev/null @@ -1,159 +0,0 @@ -param( - [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored - [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation - [Parameter(Mandatory=$true)][string] $SymbolToolPath # Full path to directory where dotnet symbol-tool was installed -) - -Add-Type -AssemblyName System.IO.Compression.FileSystem -. $PSScriptRoot\pipeline-logging-functions.ps1 - -function FirstMatchingSymbolDescriptionOrDefault { - param( - [string] $FullPath, # Full path to the module that has to be checked - [string] $TargetServerParameter, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols - [string] $SymbolsPath - ) - - $FileName = [System.IO.Path]::GetFileName($FullPath) - $Extension = [System.IO.Path]::GetExtension($FullPath) - - # Those below are potential symbol files that the `dotnet symbol` might - # return. Which one will be returned depend on the type of file we are - # checking and which type of file was uploaded. - - # The file itself is returned - $SymbolPath = $SymbolsPath + '\' + $FileName - - # PDB file for the module - $PdbPath = $SymbolPath.Replace($Extension, '.pdb') - - # PDB file for R2R module (created by crossgen) - $NGenPdb = $SymbolPath.Replace($Extension, '.ni.pdb') - - # DBG file for a .so library - $SODbg = $SymbolPath.Replace($Extension, '.so.dbg') - - # DWARF file for a .dylib - $DylibDwarf = $SymbolPath.Replace($Extension, '.dylib.dwarf') - - .\dotnet-symbol.exe --symbols --modules --windows-pdbs $TargetServerParameter $FullPath -o $SymbolsPath | Out-Null - - if (Test-Path $PdbPath) { - return 'PDB' - } - elseif (Test-Path $NGenPdb) { - return 'NGen PDB' - } - elseif (Test-Path $SODbg) { - return 'DBG for SO' - } - elseif (Test-Path $DylibDwarf) { - return 'Dwarf for Dylib' - } - elseif (Test-Path $SymbolPath) { - return 'Module' - } - else { - return $null - } -} - -function CountMissingSymbols { - param( - [string] $PackagePath # Path to a NuGet package - ) - - # Ensure input file exist - if (!(Test-Path $PackagePath)) { - throw "Input file does not exist: $PackagePath" - } - - # Extensions for which we'll look for symbols - $RelevantExtensions = @('.dll', '.exe', '.so', '.dylib') - - # How many files are missing symbol information - $MissingSymbols = 0 - - $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) - $PackageGuid = New-Guid - $ExtractPath = Join-Path -Path $ExtractPath -ChildPath $PackageGuid - $SymbolsPath = Join-Path -Path $ExtractPath -ChildPath 'Symbols' - - [System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath) - - # Makes easier to reference `symbol tool` - Push-Location $SymbolToolPath - - Get-ChildItem -Recurse $ExtractPath | - Where-Object {$RelevantExtensions -contains $_.Extension} | - ForEach-Object { - if ($_.FullName -Match '\\ref\\') { - Write-Host "`t Ignoring reference assembly file" $_.FullName - return - } - - $SymbolsOnMSDL = FirstMatchingSymbolDescriptionOrDefault -FullPath $_.FullName -TargetServerParameter '--microsoft-symbol-server' -SymbolsPath $SymbolsPath - $SymbolsOnSymWeb = FirstMatchingSymbolDescriptionOrDefault -FullPath $_.FullName -TargetServerParameter '--internal-server' -SymbolsPath $SymbolsPath - - Write-Host -NoNewLine "`t Checking file" $_.FullName "... " - - if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) { - Write-Host "Symbols found on MSDL (${$SymbolsOnMSDL}) and SymWeb (${$SymbolsOnSymWeb})" - } - else { - $MissingSymbols++ - - if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) { - Write-Host 'No symbols found on MSDL or SymWeb!' - } - else { - if ($SymbolsOnMSDL -eq $null) { - Write-Host 'No symbols found on MSDL!' - } - else { - Write-Host 'No symbols found on SymWeb!' - } - } - } - } - - Pop-Location - - return $MissingSymbols -} - -function CheckSymbolsAvailable { - if (Test-Path $ExtractPath) { - Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue - } - - Get-ChildItem "$InputPath\*.nupkg" | - ForEach-Object { - $FileName = $_.Name - - # These packages from Arcade-Services include some native libraries that - # our current symbol uploader can't handle. Below is a workaround until - # we get issue: https://github.com/dotnet/arcade/issues/2457 sorted. - if ($FileName -Match 'Microsoft\.DotNet\.Darc\.') { - Write-Host "Ignoring Arcade-services file: $FileName" - Write-Host - return - } - elseif ($FileName -Match 'Microsoft\.DotNet\.Maestro\.Tasks\.') { - Write-Host "Ignoring Arcade-services file: $FileName" - Write-Host - return - } - - Write-Host "Validating $FileName " - $Status = CountMissingSymbols "$InputPath\$FileName" - - if ($Status -ne 0) { - Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $Status modules in the package $FileName" - } - - Write-Host - } -} - -CheckSymbolsAvailable diff --git a/eng/common/PublishToPackageFeed.proj b/eng/common/PublishToPackageFeed.proj deleted file mode 100644 index a1b13337..00000000 --- a/eng/common/PublishToPackageFeed.proj +++ /dev/null @@ -1,83 +0,0 @@ - - - - - - netcoreapp2.1 - - - - - - - - - - - - - - - - - - - - - - - - - https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json - https://dotnetfeed.blob.core.windows.net/arcade-validation/index.json - https://dotnetfeed.blob.core.windows.net/aspnet-aspnetcore/index.json - https://dotnetfeed.blob.core.windows.net/aspnet-aspnetcore-tooling/index.json - https://dotnetfeed.blob.core.windows.net/aspnet-entityframeworkcore/index.json - https://dotnetfeed.blob.core.windows.net/aspnet-extensions/index.json - https://dotnetfeed.blob.core.windows.net/dotnet-coreclr/index.json - https://dotnetfeed.blob.core.windows.net/dotnet-sdk/index.json - https://dotnetfeed.blob.core.windows.net/dotnet-tools-internal/index.json - https://dotnetfeed.blob.core.windows.net/dotnet-toolset/index.json - https://dotnetfeed.blob.core.windows.net/dotnet-windowsdesktop/index.json - https://dotnetfeed.blob.core.windows.net/nuget-nugetclient/index.json - https://dotnetfeed.blob.core.windows.net/aspnet-entityframework6/index.json - https://dotnetfeed.blob.core.windows.net/aspnet-blazor/index.json - - - - - - - - - - - - diff --git a/eng/common/PublishToSymbolServers.proj b/eng/common/PublishToSymbolServers.proj deleted file mode 100644 index 311e2bbe..00000000 --- a/eng/common/PublishToSymbolServers.proj +++ /dev/null @@ -1,84 +0,0 @@ - - - - - - netcoreapp2.1 - - - - - - - - - - - - - - - - 3650 - true - true - true - false - - - - - - - - - - - - - - - - - diff --git a/eng/common/SourceLinkValidation.ps1 b/eng/common/SourceLinkValidation.ps1 deleted file mode 100644 index 0e4c349e..00000000 --- a/eng/common/SourceLinkValidation.ps1 +++ /dev/null @@ -1,195 +0,0 @@ -param( - [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where Symbols.NuGet packages to be checked are stored - [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation - [Parameter(Mandatory=$true)][string] $SourceLinkToolPath, # Full path to directory where dotnet SourceLink CLI was installed - [Parameter(Mandatory=$true)][string] $GHRepoName, # GitHub name of the repo including the Org. E.g., dotnet/arcade - [Parameter(Mandatory=$true)][string] $GHCommit # GitHub commit SHA used to build the packages -) - -function ValidateSourceLinkLinks { - if (!($GHRepoName -Match "^[^\s\/]+/[^\s\/]+$")) { - Write-PipelineTelemetryError -Category "Build" -Message "GHRepoName should be in the format /" - $global:LASTEXITCODE = 1 - return - } - - if (!($GHCommit -Match "^[0-9a-fA-F]{40}$")) { - Write-PipelineTelemetryError -Category "Build" -Message "GHCommit should be a 40 chars hexadecimal string" - $global:LASTEXITCODE = 1 - return - } - - $RepoTreeURL = -Join("https://api.github.com/repos/", $GHRepoName, "/git/trees/", $GHCommit, "?recursive=1") - $CodeExtensions = @(".cs", ".vb", ".fs", ".fsi", ".fsx", ".fsscript") - - try { - # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash - $Data = Invoke-WebRequest $RepoTreeURL | ConvertFrom-Json | Select-Object -ExpandProperty tree - - foreach ($file in $Data) { - $Extension = [System.IO.Path]::GetExtension($file.path) - - if ($CodeExtensions.Contains($Extension)) { - $RepoFiles[$file.path] = 1 - } - } - } - catch { - Write-PipelineTelemetryError -Category "Build" -Message "Problems downloading the list of files from the repo. Url used: $RepoTreeURL" - $global:LASTEXITCODE = 1 - return - } - - if (Test-Path $ExtractPath) { - Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue - } - - # Process each NuGet package in parallel - $Jobs = @() - Get-ChildItem "$InputPath\*.symbols.nupkg" | - ForEach-Object { - $Jobs += Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName - } - - foreach ($Job in $Jobs) { - Wait-Job -Id $Job.Id | Receive-Job - } -} - -try { - . $PSScriptRoot\pipeline-logging-functions.ps1 - - # Cache/HashMap (File -> Exist flag) used to consult whether a file exist - # in the repository at a specific commit point. This is populated by inserting - # all files present in the repo at a specific commit point. - $global:RepoFiles = @{} - - $ValidatePackage = { - param( - [string] $PackagePath # Full path to a Symbols.NuGet package - ) - - # Ensure input file exist - if (!(Test-Path $PackagePath)) { - throw "Input file does not exist: $PackagePath" - } - - # Extensions for which we'll look for SourceLink information - # For now we'll only care about Portable & Embedded PDBs - $RelevantExtensions = @(".dll", ".exe", ".pdb") - - Write-Host -NoNewLine "Validating" ([System.IO.Path]::GetFileName($PackagePath)) "... " - - $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) - $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId - $FailedFiles = 0 - - Add-Type -AssemblyName System.IO.Compression.FileSystem - - [System.IO.Directory]::CreateDirectory($ExtractPath); - - $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath) - - $zip.Entries | - Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} | - ForEach-Object { - $FileName = $_.FullName - $Extension = [System.IO.Path]::GetExtension($_.Name) - $FakeName = -Join((New-Guid), $Extension) - $TargetFile = Join-Path -Path $ExtractPath -ChildPath $FakeName - - # We ignore resource DLLs - if ($FileName.EndsWith(".resources.dll")) { - return - } - - [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true) - - $ValidateFile = { - param( - [string] $FullPath, # Full path to the module that has to be checked - [string] $RealPath, - [ref] $FailedFiles - ) - - # Makes easier to reference `sourcelink cli` - Push-Location $using:SourceLinkToolPath - - $SourceLinkInfos = .\sourcelink.exe print-urls $FullPath | Out-String - - if ($LASTEXITCODE -eq 0 -and -not ([string]::IsNullOrEmpty($SourceLinkInfos))) { - $NumFailedLinks = 0 - - # We only care about Http addresses - $Matches = (Select-String '(http[s]?)(:\/\/)([^\s,]+)' -Input $SourceLinkInfos -AllMatches).Matches - - if ($Matches.Count -ne 0) { - $Matches.Value | - ForEach-Object { - $Link = $_ - $CommitUrl = -Join("https://raw.githubusercontent.com/", $using:GHRepoName, "/", $using:GHCommit, "/") - $FilePath = $Link.Replace($CommitUrl, "") - $Status = 200 - $Cache = $using:RepoFiles - - if ( !($Cache.ContainsKey($FilePath)) ) { - try { - $Uri = $Link -as [System.URI] - - # Only GitHub links are valid - if ($Uri.AbsoluteURI -ne $null -and $Uri.Host -match "github") { - $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode - } - else { - $Status = 0 - } - } - catch { - $Status = 0 - } - } - - if ($Status -ne 200) { - if ($NumFailedLinks -eq 0) { - if ($FailedFiles.Value -eq 0) { - Write-Host - } - - Write-Host "`tFile $RealPath has broken links:" - } - - Write-Host "`t`tFailed to retrieve $Link" - - $NumFailedLinks++ - } - } - } - - if ($NumFailedLinks -ne 0) { - $FailedFiles.value++ - $global:LASTEXITCODE = 1 - } - } - - Pop-Location - } - - &$ValidateFile $TargetFile $FileName ([ref]$FailedFiles) - } - - $zip.Dispose() - - if ($FailedFiles -eq 0) { - Write-Host "Passed." - } - } - - Measure-Command { ValidateSourceLinkLinks } -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'SourceLink' -Message $_ - ExitWithExitCode 1 -} - - diff --git a/eng/common/post-build/check-channel-consistency.ps1 b/eng/common/post-build/check-channel-consistency.ps1 new file mode 100644 index 00000000..7e6618d6 --- /dev/null +++ b/eng/common/post-build/check-channel-consistency.ps1 @@ -0,0 +1,25 @@ +param( + [Parameter(Mandatory=$true)][string] $PromoteToChannels, # List of channels that the build should be promoted to + [Parameter(Mandatory=$true)][array] $AvailableChannelIds # List of channel IDs available in the YAML implementation +) + +try { + . $PSScriptRoot\post-build-utils.ps1 + + # Check that every channel that Maestro told to promote the build to + # is available in YAML + $PromoteToChannelsIds = $PromoteToChannels -split "\D" | Where-Object { $_ } + + foreach ($id in $PromoteToChannelsIds) { + if (($id -ne 0) -and ($id -notin $AvailableChannelIds)) { + Write-PipelineTaskError -Type 'warning' -Message "Channel $id is not present in the post-build YAML configuration!" + } + } + + Write-Host 'done.' +} +catch { + Write-Host $_ + Write-PipelineTelemetryError -Category 'CheckChannelConsistency' -Message "There was an error while trying to check consistency of Maestro default channels for the build and post-build YAML configuration." + ExitWithExitCode 1 +} diff --git a/eng/common/templates/post-build/channels/generic-public-channel.yml b/eng/common/templates/post-build/channels/generic-public-channel.yml index ae8c6349..00ef36fe 100644 --- a/eng/common/templates/post-build/channels/generic-public-channel.yml +++ b/eng/common/templates/post-build/channels/generic-public-channel.yml @@ -94,6 +94,8 @@ stages: value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - name: AzDOBuildId value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + - name: ArtifactsCategory + value: ${{ coalesce(variables._DotNetArtifactsCategory, '.NETCore') }} condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )), eq(dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'], ${{ parameters.channelId }})) pool: vmImage: 'windows-2019' @@ -132,7 +134,7 @@ stages: inputs: filePath: eng\common\sdk-task.ps1 arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet - /p:ArtifactsCategory=$(_DotNetArtifactsCategory) + /p:ArtifactsCategory=$(ArtifactsCategory) /p:IsStableBuild=$(IsStableBuild) /p:IsInternalBuild=$(IsInternalBuild) /p:RepositoryName=$(Build.Repository.Name) diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index eff0d9ac..05ff3561 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -22,6 +22,22 @@ parameters: publishDependsOn: - Validate + # Channel ID's instantiated in this file. + # When adding a new channel implementation the call to `check-channel-consistency.ps1` + # needs to be updated with the new channel ID + NetEngLatestChannelId: 2 + NetEngValidationChannelId: 9 + NetCoreDev5ChannelId: 131 + GeneralTestingChannelId: 529 + NETCoreToolingDevChannelId: 548 + NETCoreToolingReleaseChannelId: 549 + NETInternalToolingChannelId: 551 + NETCoreExperimentalChannelId: 562 + NetEngServicesIntChannelId: 678 + NetEngServicesProdChannelId: 679 + Net5Preview1ChannelId: 737 + Net5Preview2ChannelId: 738 + stages: - stage: Validate dependsOn: ${{ parameters.validateDependsOn }} @@ -31,6 +47,24 @@ stages: jobs: - template: setup-maestro-vars.yml + - job: + displayName: Post-build Checks + dependsOn: setupMaestroVars + variables: + - name: InitialChannels + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'] ] + - name: PromoteToMaestroChannelId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'] ] + pool: + vmImage: 'windows-2019' + steps: + - task: PowerShell@2 + displayName: Maestro Channels Consistency + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 + arguments: -PromoteToChannels "$(InitialChannels)[$(PromoteToMaestroChannelId)]" + -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetCoreDev5ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview1ChannelId}},${{parameters.Net5Preview2ChannelId}} + - job: displayName: NuGet Validation dependsOn: setupMaestroVars @@ -165,7 +199,7 @@ stages: symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} stageName: 'NetCore_Dev5_Publish' channelName: '.NET Core 5 Dev' - channelId: 131 + channelId: ${{ parameters.NetCoreDev5ChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' @@ -178,7 +212,7 @@ stages: symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} stageName: 'Net5_Preview1_Publish' channelName: '.NET 5 Preview 1' - channelId: 737 + channelId: ${{ parameters.Net5Preview1ChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' @@ -191,7 +225,7 @@ stages: symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} stageName: 'Net5_Preview2_Publish' channelName: '.NET 5 Preview 2' - channelId: 738 + channelId: ${{ parameters.Net5Preview2ChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' @@ -204,7 +238,7 @@ stages: symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} stageName: 'Net_Eng_Latest_Publish' channelName: '.NET Eng - Latest' - channelId: 2 + channelId: ${{ parameters.NetEngLatestChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' @@ -217,7 +251,7 @@ stages: symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} stageName: 'Net_Eng_Validation_Publish' channelName: '.NET Eng - Validation' - channelId: 9 + channelId: ${{ parameters.NetEngValidationChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' @@ -230,7 +264,7 @@ stages: symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} stageName: 'General_Testing_Publish' channelName: 'General Testing' - channelId: 529 + channelId: ${{ parameters.GeneralTestingChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing-symbols/nuget/v3/index.json' @@ -243,7 +277,7 @@ stages: symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} stageName: 'NETCore_Tooling_Dev_Publishing' channelName: '.NET Core Tooling Dev' - channelId: 548 + channelId: ${{ parameters.NETCoreToolingDevChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' @@ -256,7 +290,7 @@ stages: symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} stageName: 'NETCore_Tooling_Release_Publishing' channelName: '.NET Core Tooling Release' - channelId: 549 + channelId: ${{ parameters.NETCoreToolingReleaseChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' @@ -269,7 +303,7 @@ stages: symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} stageName: 'NET_Internal_Tooling_Publishing' channelName: '.NET Internal Tooling' - channelId: 551 + channelId: ${{ parameters.NETInternalToolingChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal-symbols/nuget/v3/index.json' @@ -282,7 +316,7 @@ stages: symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} stageName: 'NETCore_Experimental_Publishing' channelName: '.NET Core Experimental' - channelId: 562 + channelId: ${{ parameters.NETCoreExperimentalChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental-symbols/nuget/v3/index.json' @@ -295,7 +329,7 @@ stages: symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} stageName: 'Net_Eng_Services_Int_Publish' channelName: '.NET Eng Services - Int' - channelId: 678 + channelId: ${{ parameters.NetEngServicesIntChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' @@ -308,7 +342,7 @@ stages: symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} stageName: 'Net_Eng_Services_Prod_Publish' channelName: '.NET Eng Services - Prod' - channelId: 679 + channelId: ${{ parameters.NetEngServicesProdChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' diff --git a/global.json b/global.json index 4df094c4..e16a73fd 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.1.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20116.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20123.1" } } From 0c453e25f3ff7bb08d0be4b8e9c5b68237c05ecc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Matou=C5=A1ek?= Date: Sun, 1 Mar 2020 16:12:20 -0800 Subject: [PATCH 061/161] Use submodule.name.url to determine the URL of a submodule (#587) --- docs/README.md | 12 ++- .../GitDataTests.cs | 9 +- .../GitOperationsTests.cs | 91 +++++++++---------- .../GitRepositoryTests.cs | 4 +- .../GitDataReader/GitRepository.cs | 6 +- .../GitDataReader/GitSubmodule.cs | 8 +- .../GitOperations.cs | 19 +++- 7 files changed, 81 insertions(+), 68 deletions(-) diff --git a/docs/README.md b/docs/README.md index 9487e869..49758ff9 100644 --- a/docs/README.md +++ b/docs/README.md @@ -220,14 +220,18 @@ for Source Link to operate properly. configuration file that specifies `origin` remote URL (e.g. `[remote "origin"] url="http://server.com/repo"`) -If the repository has submodules the file `.gitmodules` must be present in the repository root and must list the URLs and -relative paths of all submodules. - -For example, +If the repository has submodules the file `.gitmodules` must be present in the repository root and must list the +relative paths of all submodules: ``` [submodule "submodule-name"] path = submodule-path +``` + +The `.git/config` file must contain URLs of all initialized submodules: + +``` +[submodule "submodule-name"] url = https://server.com/subrepo ``` diff --git a/src/Microsoft.Build.Tasks.Git.UnitTests/GitDataTests.cs b/src/Microsoft.Build.Tasks.Git.UnitTests/GitDataTests.cs index f179ae64..809ef4ea 100644 --- a/src/Microsoft.Build.Tasks.Git.UnitTests/GitDataTests.cs +++ b/src/Microsoft.Build.Tasks.Git.UnitTests/GitDataTests.cs @@ -20,7 +20,12 @@ public void MinimalGitData() var gitDir = repoDir.CreateDirectory(".git"); gitDir.CreateFile("HEAD").WriteAllText("1111111111111111111111111111111111111111"); - gitDir.CreateFile("config").WriteAllText(@"[remote ""origin""]url=""http://github.com/test-org/test-repo"""); + gitDir.CreateFile("config").WriteAllText(@" +[remote ""origin""] +url=http://github.com/test-org/test-repo +[submodule ""my submodule""] +url=https://github.com/test-org/test-sub +"); gitDir.CreateDirectory("objects"); gitDir.CreateDirectory("refs"); repoDir.CreateFile(".gitignore").WriteAllText("ignore_this_*"); @@ -29,7 +34,7 @@ public void MinimalGitData() var gitModules = repoDir.CreateFile(".gitmodules").WriteAllText(@" [submodule ""my submodule""] path = sub - url = https://github.com/test-org/test-sub + url = xyz "); var subDir = repoDir.CreateDirectory("sub"); diff --git a/src/Microsoft.Build.Tasks.Git.UnitTests/GitOperationsTests.cs b/src/Microsoft.Build.Tasks.Git.UnitTests/GitOperationsTests.cs index eaef98e8..d4a25ca0 100644 --- a/src/Microsoft.Build.Tasks.Git.UnitTests/GitOperationsTests.cs +++ b/src/Microsoft.Build.Tasks.Git.UnitTests/GitOperationsTests.cs @@ -9,6 +9,8 @@ namespace Microsoft.Build.Tasks.Git.UnitTests { + using static KeyValuePairUtils; + public class GitOperationsTests { private static readonly bool IsUnix = Path.DirectorySeparatorChar == '/'; @@ -64,11 +66,11 @@ private static GitVariableName CreateVariableName(string str) private static GitConfig CreateConfig(params (string Name, string Value)[] variables) => new GitConfig(ImmutableDictionary.CreateRange( - variables.Select(v => new KeyValuePair>(CreateVariableName(v.Name), ImmutableArray.Create(v.Value))))); + variables.Select(v => KVP(CreateVariableName(v.Name), ImmutableArray.Create(v.Value))))); private static GitConfig CreateConfig(params (string Name, string[] Values)[] variables) => new GitConfig(ImmutableDictionary.CreateRange( - variables.Select(v => new KeyValuePair>(CreateVariableName(v.Name), ImmutableArray.CreateRange(v.Values))))); + variables.Select(v => KVP(CreateVariableName(v.Name), ImmutableArray.CreateRange(v.Values))))); [Fact] public void GetRepositoryUrl_NoRemotes() @@ -179,8 +181,8 @@ public void GetRepositoryUrl_InsteadOf() { var repo = CreateRepository(config: new GitConfig(ImmutableDictionary.CreateRange(new[] { - new KeyValuePair>(new GitVariableName("remote", "origin", "url"), ImmutableArray.Create("http://?")), - new KeyValuePair>(new GitVariableName("url", "git@github.com:org/repo", "insteadOf"), ImmutableArray.Create("http://?")) + KVP(new GitVariableName("remote", "origin", "url"), ImmutableArray.Create("http://?")), + KVP(new GitVariableName("url", "git@github.com:org/repo", "insteadOf"), ImmutableArray.Create("http://?")) }))); var warnings = new List<(string, object?[])>(); @@ -206,7 +208,7 @@ public void GetRepositoryUrl_Local(bool useFileUrl) var repo = CreateRepository(config: new GitConfig(ImmutableDictionary.CreateRange(new[] { - new KeyValuePair>(new GitVariableName("remote", "origin", "url"), + KVP(new GitVariableName("remote", "origin", "url"), ImmutableArray.Create(useFileUrl ? new Uri(mainWorkingDir.Path).AbsolutePath : mainWorkingDir.Path)), }))); @@ -231,7 +233,7 @@ public void GetRepositoryUrl_Local_CustomRemoteName() var repo = CreateRepository(config: new GitConfig(ImmutableDictionary.CreateRange(new[] { - new KeyValuePair>(new GitVariableName("remote", "origin", "url"), ImmutableArray.Create(mainWorkingDir.Path)), + KVP(new GitVariableName("remote", "origin", "url"), ImmutableArray.Create(mainWorkingDir.Path)), }))); var warnings = new List<(string, object?[])>(); @@ -253,7 +255,7 @@ public void GetRepositoryUrl_Local_NoRemoteOriginUrl() var repo = CreateRepository(config: new GitConfig(ImmutableDictionary.CreateRange(new[] { - new KeyValuePair>(new GitVariableName("remote", "origin", "url"), ImmutableArray.Create(mainWorkingDir.Path)), + KVP(new GitVariableName("remote", "origin", "url"), ImmutableArray.Create(mainWorkingDir.Path)), }))); var warnings = new List<(string, object?[])>(); @@ -281,7 +283,7 @@ public void GetRepositoryUrl_Local_NoWorkingDirectory() var repo = CreateRepository(config: new GitConfig(ImmutableDictionary.CreateRange(new[] { - new KeyValuePair>(new GitVariableName("remote", "origin", "url"), ImmutableArray.Create(gitDir2.Path)), + KVP(new GitVariableName("remote", "origin", "url"), ImmutableArray.Create(gitDir2.Path)), }))); var warnings = new List<(string, object?[])>(); @@ -302,7 +304,7 @@ public void GetRepositoryUrl_Local_BadRepo() var repo = CreateRepository(config: new GitConfig(ImmutableDictionary.CreateRange(new[] { - new KeyValuePair>(new GitVariableName("remote", "origin", "url"), ImmutableArray.Create(mainWorkingDir.Path)), + KVP(new GitVariableName("remote", "origin", "url"), ImmutableArray.Create(mainWorkingDir.Path)), }))); var warnings = new List<(string, object?[])>(); @@ -322,7 +324,7 @@ public void GetRepositoryUrl_LocalRecursion() var repo = CreateRepository(config: new GitConfig(ImmutableDictionary.CreateRange(new[] { - new KeyValuePair>(new GitVariableName("remote", "origin", "url"), + KVP(new GitVariableName("remote", "origin", "url"), ImmutableArray.Create(mainWorkingDir.Path)), }))); @@ -461,21 +463,37 @@ public void GetSourceRoots_RepoWithoutCommitsWithSubmodules() { var repo = CreateRepository( commitSha: null, - config: CreateConfig(("url.ssh://.insteadOf", "http://")), + config: CreateConfig( + ("url.ssh://.insteadOf", "http://"), + ("submodule.sub1.url", "http://github.com/sub-1"), + ("submodule.sub3.url", "https://github.com/sub-3"), + ("submodule.sub4.url", "https:///"), + ("submodule.sub6.url", "https://github.com/sub-6")), submodules: ImmutableArray.Create( - CreateSubmodule("1", "sub/1", "http://1.com", "1111111111111111111111111111111111111111"), - CreateSubmodule("1", "sub/2", "http://2.com", "2222222222222222222222222222222222222222"))); ; + CreateSubmodule("sub1", "sub/1", "http://1.com", "1111111111111111111111111111111111111111"), + CreateSubmodule("sub2", "sub/2", "http://2.com", "2222222222222222222222222222222222222222"), + CreateSubmodule("sub3", "sub/3", "http://3.com", "3333333333333333333333333333333333333333"), + CreateSubmodule("sub4", "sub/4", "http://4.com", "4444444444444444444444444444444444444444"), + CreateSubmodule("sub5", "sub/5", "http:///", "5555555555555555555555555555555555555555"), + CreateSubmodule("sub6", "sub/6", "", "6666666666666666666666666666666666666666"))); var warnings = new List<(string, object?[])>(); var items = GitOperations.GetSourceRoots(repo, remoteName: null, (message, args) => warnings.Add((message, args))); + // Module without a configuration entry is not initialized. + // URLs listed in .submodules are ignored (they are used by git submodule initialize to generate URLs stored in config). AssertEx.Equal(new[] { - $@"'{_workingDir}{s}sub{s}1{s}' SourceControl='git' RevisionId='1111111111111111111111111111111111111111' NestedRoot='sub/1/' ContainingRoot='{_workingDir}{s}' ScmRepositoryUrl='ssh://1.com/'", - $@"'{_workingDir}{s}sub{s}2{s}' SourceControl='git' RevisionId='2222222222222222222222222222222222222222' NestedRoot='sub/2/' ContainingRoot='{_workingDir}{s}' ScmRepositoryUrl='ssh://2.com/'", + $@"'{_workingDir}{s}sub{s}1{s}' SourceControl='git' RevisionId='1111111111111111111111111111111111111111' NestedRoot='sub/1/' ContainingRoot='{_workingDir}{s}' ScmRepositoryUrl='ssh://github.com/sub-1'", + $@"'{_workingDir}{s}sub{s}3{s}' SourceControl='git' RevisionId='3333333333333333333333333333333333333333' NestedRoot='sub/3/' ContainingRoot='{_workingDir}{s}' ScmRepositoryUrl='https://github.com/sub-3'", + $@"'{_workingDir}{s}sub{s}6{s}' SourceControl='git' RevisionId='6666666666666666666666666666666666666666' NestedRoot='sub/6/' ContainingRoot='{_workingDir}{s}' ScmRepositoryUrl='https://github.com/sub-6'", }, items.Select(TestUtilities.InspectSourceRoot)); - AssertEx.Equal(new[] { Resources.RepositoryHasNoCommit }, warnings.Select(TestUtilities.InspectDiagnostic)); + AssertEx.Equal(new[] + { + Resources.RepositoryHasNoCommit, + string.Format(Resources.SourceCodeWontBeAvailableViaSourceLink, string.Format(Resources.InvalidSubmoduleUrl, "sub4", "https:///")) + }, warnings.Select(TestUtilities.InspectDiagnostic)); ; } [Fact] @@ -483,9 +501,12 @@ public void GetSourceRoots_RepoWithCommitsWithSubmodules() { var repo = CreateRepository( commitSha: "0000000000000000000000000000000000000000", + config: CreateConfig( + ("submodule.1.url", "http://github.com/sub-1"), + ("submodule.2.url", "http://github.com/sub-2")), submodules: ImmutableArray.Create( CreateSubmodule("1", "sub/1", "http://1.com", headCommitSha: null), - CreateSubmodule("1", "sub/2", "http://2.com", "2222222222222222222222222222222222222222"))); + CreateSubmodule("2", "sub/2", "http://2.com", "2222222222222222222222222222222222222222"))); var warnings = new List<(string, object?[])>(); var items = GitOperations.GetSourceRoots(repo, remoteName: null, (message, args) => warnings.Add((message, args))); @@ -493,7 +514,7 @@ public void GetSourceRoots_RepoWithCommitsWithSubmodules() AssertEx.Equal(new[] { $@"'{_workingDir}{s}' SourceControl='git' RevisionId='0000000000000000000000000000000000000000'", - $@"'{_workingDir}{s}sub{s}2{s}' SourceControl='git' RevisionId='2222222222222222222222222222222222222222' NestedRoot='sub/2/' ContainingRoot='{_workingDir}{s}' ScmRepositoryUrl='http://2.com/'", + $@"'{_workingDir}{s}sub{s}2{s}' SourceControl='git' RevisionId='2222222222222222222222222222222222222222' NestedRoot='sub/2/' ContainingRoot='{_workingDir}{s}' ScmRepositoryUrl='http://github.com/sub-2'", }, items.Select(TestUtilities.InspectSourceRoot)); AssertEx.Equal(new[] { string.Format(Resources.SourceCodeWontBeAvailableViaSourceLink, string.Format(Resources.SubmoduleWithoutCommit, "1")) }, @@ -501,7 +522,7 @@ public void GetSourceRoots_RepoWithCommitsWithSubmodules() } [Fact] - public void GetSourceRoots_RelativeSubmodulePaths() + public void GetSourceRoots_RelativeSubmodulePath() { using var temp = new TempRoot(); @@ -518,8 +539,10 @@ public void GetSourceRoots_RelativeSubmodulePaths() var repo = CreateRepository( workingDir: repoDir.Path, commitSha: "0000000000000000000000000000000000000000", + config: CreateConfig( + ("submodule.1.url", "../1")), submodules: ImmutableArray.Create( - CreateSubmodule("1", "sub/1", "../1", "1111111111111111111111111111111111111111", containingRepositoryWorkingDir: repoDir.Path))); + CreateSubmodule("1", "sub/1", "---", "1111111111111111111111111111111111111111", containingRepositoryWorkingDir: repoDir.Path))); var warnings = new List<(string, object?[])>(); var items = GitOperations.GetSourceRoots(repo, remoteName: null, (message, args) => warnings.Add((message, args))); @@ -532,32 +555,8 @@ public void GetSourceRoots_RelativeSubmodulePaths() Assert.Empty(warnings); } - - [Fact] - public void GetSourceRoots_InvalidSubmoduleUrl() - { - var repo = CreateRepository( - commitSha: "0000000000000000000000000000000000000000", - submodules: ImmutableArray.Create( - CreateSubmodule("1", "sub/1", "http:///", "1111111111111111111111111111111111111111"), - CreateSubmodule("3", "sub/3", "http://3.com", "3333333333333333333333333333333333333333"))); - - var warnings = new List<(string, object?[])>(); - var items = GitOperations.GetSourceRoots(repo, remoteName: null, (message, args) => warnings.Add((message, args))); - - AssertEx.Equal(new[] - { - $@"'{s_root}{s}' SourceControl='git' RevisionId='0000000000000000000000000000000000000000'", - $@"'{s_root}{s}sub{s}3{s}' SourceControl='git' RevisionId='3333333333333333333333333333333333333333' NestedRoot='sub/3/' ContainingRoot='{s_root}{s}' ScmRepositoryUrl='http://3.com/'", - }, items.Select(TestUtilities.InspectSourceRoot)); - - AssertEx.Equal(new[] - { - string.Format(Resources.SourceCodeWontBeAvailableViaSourceLink, string.Format(Resources.InvalidSubmoduleUrl, "1", "http:///")), - }, warnings.Select(TestUtilities.InspectDiagnostic)); - } - - private static GitOperations.DirectoryNode CreateNode(string name, string? submoduleWorkingDirectory, List? children = null) + + private static GitOperations.DirectoryNode CreateNode(string name, string? submoduleWorkingDirectory, List? children = null) => new GitOperations.DirectoryNode(name, children ?? new List()) { Matcher = (submoduleWorkingDirectory != null) ? new Lazy(() => diff --git a/src/Microsoft.Build.Tasks.Git.UnitTests/GitRepositoryTests.cs b/src/Microsoft.Build.Tasks.Git.UnitTests/GitRepositoryTests.cs index 8d611bb4..442a8dcc 100644 --- a/src/Microsoft.Build.Tasks.Git.UnitTests/GitRepositoryTests.cs +++ b/src/Microsoft.Build.Tasks.Git.UnitTests/GitRepositoryTests.cs @@ -307,8 +307,8 @@ public void Submodules_Errors() string.Format(Resources.InvalidSubmodulePath, "S2", ""), // Could not find a part of the path 'sub3\.git'. TestUtilities.GetExceptionMessage(() => File.ReadAllText(Path.Combine(workingDir.Path, "sub3", ".git"))), - // The URL of submodule 'S4' is missing or invalid: ' ' - string.Format(Resources.InvalidSubmoduleUrl, "S4", " "), + // Could not find a part of the path 'sub4\.git'. + TestUtilities.GetExceptionMessage(() => File.ReadAllText(Path.Combine(workingDir.Path, "sub4", ".git"))), // Could not find a part of the path 'sub5\.git'. TestUtilities.GetExceptionMessage(() => File.ReadAllText(Path.Combine(workingDir.Path, "sub5", ".git"))), // Access to the path 'sub6\.git' is denied diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs index 9bab50f5..c2fbfeb5 100644 --- a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs +++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs @@ -261,11 +261,7 @@ void reportDiagnostic(string diagnostic) continue; } - if (NullableString.IsNullOrWhiteSpace(url)) - { - reportDiagnostic(string.Format(Resources.InvalidSubmoduleUrl, name, url)); - continue; - } + // Ignore unspecified URL - Source Link doesn't use it. string fullPath; try diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitSubmodule.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitSubmodule.cs index cba708d7..a953773e 100644 --- a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitSubmodule.cs +++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitSubmodule.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; -using System.Diagnostics.CodeAnalysis; namespace Microsoft.Build.Tasks.Git { @@ -21,21 +20,20 @@ internal readonly struct GitSubmodule public string WorkingDirectoryFullPath { get; } /// - /// An absolute URL or a relative path (if it starts with `./` or `../`) to the default remote of the containing repository. + /// An absolute URL or a relative path (if it starts with `./` or `../`) to the origin remote of the containing repository. /// - public string Url { get; } + public string? Url { get; } /// /// Head tip commit SHA. Null, if there is no commit. /// public string? HeadCommitSha { get; } - internal GitSubmodule(string name, string workingDirectoryRelativePath, string workingDirectoryFullPath, string url, string? headCommitSha) + internal GitSubmodule(string name, string workingDirectoryRelativePath, string workingDirectoryFullPath, string? url, string? headCommitSha) { NullableDebug.Assert(name != null); NullableDebug.Assert(workingDirectoryRelativePath != null); NullableDebug.Assert(workingDirectoryFullPath != null); - NullableDebug.Assert(url != null); Name = name; WorkingDirectoryRelativePath = workingDirectoryRelativePath; diff --git a/src/Microsoft.Build.Tasks.Git/GitOperations.cs b/src/Microsoft.Build.Tasks.Git/GitOperations.cs index e21948a4..2d029de4 100644 --- a/src/Microsoft.Build.Tasks.Git/GitOperations.cs +++ b/src/Microsoft.Build.Tasks.Git/GitOperations.cs @@ -18,6 +18,7 @@ internal static class GitOperations private const string SourceControlName = "git"; private const string RemoteSectionName = "remote"; + private const string SubmoduleSectionName = "submodule"; private const string RemoteOriginName = "origin"; private const string UrlSectionName = "url"; private const string UrlVariableName = "url"; @@ -275,12 +276,22 @@ public static ITaskItem[] GetSourceRoots(GitRepository repository, string? remot continue; } - // https://git-scm.com/docs/git-submodule - var submoduleUri = NormalizeUrl(repository, submodule.Url); + // submodule..url specifies where to find the submodule. + // This variable is calculated based on the entry in .gitmodules by git submodule init and will be present for initialized submodules. + // Uninitialized modules don't have source that should be considered during the build. + // Relative URLs are relative to the repository directory. + // See https://git-scm.com/docs/gitsubmodules. + var submoduleConfigUrl = repository.Config.GetVariableValue(SubmoduleSectionName, submodule.Name, UrlVariableName); + if (submoduleConfigUrl == null) + { + continue; + } + + var submoduleUri = NormalizeUrl(repository, submoduleConfigUrl); if (submoduleUri == null) { logWarning(Resources.SourceCodeWontBeAvailableViaSourceLink, - new[] { string.Format(Resources.InvalidSubmoduleUrl, submodule.Name, submodule.Url) }); + new[] { string.Format(Resources.InvalidSubmoduleUrl, submodule.Name, submoduleConfigUrl) }); continue; } @@ -289,7 +300,7 @@ public static ITaskItem[] GetSourceRoots(GitRepository repository, string? remot if (submoduleUrl == null) { logWarning(Resources.SourceCodeWontBeAvailableViaSourceLink, - new[] { string.Format(Resources.InvalidSubmoduleUrl, submodule.Name, submodule.Url) }); + new[] { string.Format(Resources.InvalidSubmoduleUrl, submodule.Name, submoduleConfigUrl) }); continue; } From e91c12496a532290573062cf8bf15420f979cf25 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 2 Mar 2020 13:21:52 +0000 Subject: [PATCH 062/161] Update dependencies from https://github.com/dotnet/arcade build 20200301.1 (#588) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.20151.1 --- eng/Version.Details.xml | 4 +- eng/common/dotnet-install.sh | 2 +- eng/common/init-tools-native.sh | 49 +++++++++++++++---- eng/common/performance/perfhelixpublish.proj | 21 +++++++- .../channels/generic-internal-channel.yml | 1 + .../channels/generic-public-channel.yml | 1 + global.json | 4 +- 7 files changed, 67 insertions(+), 15 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 611d35ba..31635bca 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 85d76351b1f0245c9f331f72219d12e8e2d48e72 + 8ccad075bbb0db445e03eed0a6073d27bdd4f31a diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh index 50bc5e47..ead6a1d9 100755 --- a/eng/common/dotnet-install.sh +++ b/eng/common/dotnet-install.sh @@ -63,7 +63,7 @@ case $cpuname in amd64|x86_64) buildarch=x64 ;; - armv7l) + armv*l) buildarch=arm ;; i686) diff --git a/eng/common/init-tools-native.sh b/eng/common/init-tools-native.sh index dbd7cbfd..29fc5db8 100755 --- a/eng/common/init-tools-native.sh +++ b/eng/common/init-tools-native.sh @@ -34,6 +34,14 @@ while (($# > 0)); do force=true shift 1 ;; + --donotabortonfailure) + donotabortonfailure=true + shift 1 + ;; + --donotdisplaywarnings) + donotdisplaywarnings=true + shift 1 + ;; --downloadretries) download_retries=$2 shift 2 @@ -52,6 +60,8 @@ while (($# > 0)); do echo " - (default) %USERPROFILE%/.netcoreeng/native" echo "" echo " --clean Switch specifying not to install anything, but cleanup native asset folders" + echo " --donotabortonfailure Switch specifiying whether to abort native tools installation on failure" + echo " --donotdisplaywarnings Switch specifiying whether to display warnings during native tools installation on failure" echo " --force Clean and then install tools" echo " --help Print help and exit" echo "" @@ -92,6 +102,7 @@ if [[ -z $install_directory ]]; then fi install_bin="${native_base_dir}/bin" +installed_any=false ReadGlobalJsonNativeTools @@ -103,8 +114,8 @@ else for tool in "${!native_assets[@]}" do tool_version=${native_assets[$tool]} - installer_name="install-$tool.sh" - installer_command="$native_installer_dir/$installer_name" + installer_path="$native_installer_dir/install-$tool.sh" + installer_command="$installer_path" installer_command+=" --baseuri $base_uri" installer_command+=" --installpath $install_bin" installer_command+=" --version $tool_version" @@ -118,11 +129,29 @@ else installer_command+=" --clean" fi - $installer_command - - if [[ $? != 0 ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed" - exit 1 + if [[ -a $installer_path ]]; then + $installer_command + if [[ $? != 0 ]]; then + if [[ $donotabortonfailure = true ]]; then + if [[ $donotdisplaywarnings != true ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed" + fi + else + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed" + exit 1 + fi + else + $installed_any = true + fi + else + if [[ $donotabortonfailure == true ]]; then + if [[ $donotdisplaywarnings != true ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed: no install script" + fi + else + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed: no install script" + exit 1 + fi fi done fi @@ -135,8 +164,10 @@ if [[ -d $install_bin ]]; then echo "Native tools are available from $install_bin" echo "##vso[task.prependpath]$install_bin" else - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Native tools install directory does not exist, installation failed" - exit 1 + if [[ $installed_any = true ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Native tools install directory does not exist, installation failed" + exit 1 + fi fi exit 0 diff --git a/eng/common/performance/perfhelixpublish.proj b/eng/common/performance/perfhelixpublish.proj index e5826b53..cf5941e1 100644 --- a/eng/common/performance/perfhelixpublish.proj +++ b/eng/common/performance/perfhelixpublish.proj @@ -6,7 +6,7 @@ py -3 %HELIX_CORRELATION_PAYLOAD%\Core_Root\CoreRun.exe %HELIX_CORRELATION_PAYLOAD%\Baseline_Core_Root\CoreRun.exe - $(HelixPreCommands);call %HELIX_CORRELATION_PAYLOAD%\performance\tools\machine-setup.cmd + $(HelixPreCommands);call %HELIX_CORRELATION_PAYLOAD%\performance\tools\machine-setup.cmd;set PYTHONPATH=%HELIX_WORKITEM_PAYLOAD%\scripts%3B%HELIX_WORKITEM_PAYLOAD% %HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts %HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts_Baseline %HELIX_CORRELATION_PAYLOAD%\performance\src\tools\ResultsComparer\ResultsComparer.csproj @@ -99,4 +99,23 @@ 4:00 + + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name System.Private.Xml.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name System.Linq.Expressions.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name Microsoft.CodeAnalysis.VisualBasic.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name Microsoft.CodeAnalysis.CSharp.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + \ No newline at end of file diff --git a/eng/common/templates/post-build/channels/generic-internal-channel.yml b/eng/common/templates/post-build/channels/generic-internal-channel.yml index 38012290..dde27800 100644 --- a/eng/common/templates/post-build/channels/generic-internal-channel.yml +++ b/eng/common/templates/post-build/channels/generic-internal-channel.yml @@ -84,6 +84,7 @@ stages: - job: publish_assets displayName: Publish Assets dependsOn: setupMaestroVars + timeoutInMinutes: 120 variables: - name: BARBuildId value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] diff --git a/eng/common/templates/post-build/channels/generic-public-channel.yml b/eng/common/templates/post-build/channels/generic-public-channel.yml index 00ef36fe..29bc1a94 100644 --- a/eng/common/templates/post-build/channels/generic-public-channel.yml +++ b/eng/common/templates/post-build/channels/generic-public-channel.yml @@ -83,6 +83,7 @@ stages: - job: publish_assets displayName: Publish Assets dependsOn: setupMaestroVars + timeoutInMinutes: 120 variables: - name: BARBuildId value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] diff --git a/global.json b/global.json index e16a73fd..014ace9d 100644 --- a/global.json +++ b/global.json @@ -1,8 +1,8 @@ { "tools": { - "dotnet": "3.1.100" + "dotnet": "3.1.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20123.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20151.1" } } From 0fadf1fa3a9c61f310244358627bd462d0e47c9c Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 9 Mar 2020 12:11:28 +0000 Subject: [PATCH 063/161] Update dependencies from https://github.com/dotnet/arcade build 20200308.2 (#589) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.20158.2 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 4 +- eng/common/SigningValidation.proj | 83 ------------------- eng/common/build.ps1 | 5 ++ eng/common/cross/android/arm/toolchain.cmake | 41 --------- .../cross/android/arm64/toolchain.cmake | 42 ---------- eng/common/cross/build-android-rootfs.sh | 69 ++++++--------- eng/common/cross/toolchain.cmake | 30 +++++-- eng/common/darc-init.ps1 | 6 +- eng/common/pipeline-logging-functions.ps1 | 4 +- .../channels/generic-public-channel.yml | 5 ++ .../templates/post-build/post-build.yml | 6 ++ eng/common/tools.ps1 | 2 +- global.json | 2 +- 13 files changed, 77 insertions(+), 222 deletions(-) delete mode 100644 eng/common/SigningValidation.proj delete mode 100644 eng/common/cross/android/arm/toolchain.cmake delete mode 100644 eng/common/cross/android/arm64/toolchain.cmake diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 31635bca..62e72933 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 8ccad075bbb0db445e03eed0a6073d27bdd4f31a + 527179abfbb5cd738035c4ac2d2566c571ce85db diff --git a/eng/common/SigningValidation.proj b/eng/common/SigningValidation.proj deleted file mode 100644 index 3d0ac80a..00000000 --- a/eng/common/SigningValidation.proj +++ /dev/null @@ -1,83 +0,0 @@ - - - - - - netcoreapp2.1 - - - - - - - - $(NuGetPackageRoot)Microsoft.DotNet.SignCheck\$(SignCheckVersion)\tools\Microsoft.DotNet.SignCheck.exe - - $(PackageBasePath) - signcheck.log - signcheck.errors.log - signcheck.exclusions.txt - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 88814514..813d440d 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -26,6 +26,11 @@ Param( [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties ) +# Unset 'Platform' environment variable to avoid unwanted collision in InstallDotNetCore.targets file +# some computer has this env var defined (e.g. Some HP) +if($env:Platform) { + $env:Platform="" +} function Print-Usage() { Write-Host "Common settings:" Write-Host " -configuration Build configuration: 'Debug' or 'Release' (short: -c)" diff --git a/eng/common/cross/android/arm/toolchain.cmake b/eng/common/cross/android/arm/toolchain.cmake deleted file mode 100644 index a7e1c735..00000000 --- a/eng/common/cross/android/arm/toolchain.cmake +++ /dev/null @@ -1,41 +0,0 @@ -set(CROSS_NDK_TOOLCHAIN $ENV{ROOTFS_DIR}/../) -set(CROSS_ROOTFS ${CROSS_NDK_TOOLCHAIN}/sysroot) -set(CLR_CMAKE_PLATFORM_ANDROID "Android") - -set(CMAKE_SYSTEM_NAME Linux) -set(CMAKE_SYSTEM_VERSION 1) -set(CMAKE_SYSTEM_PROCESSOR arm) - -## Specify the toolchain -set(TOOLCHAIN "arm-linux-androideabi") -set(CMAKE_PREFIX_PATH ${CROSS_NDK_TOOLCHAIN}) -set(TOOLCHAIN_PREFIX ${TOOLCHAIN}-) - -find_program(CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}clang) -find_program(CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}clang++) -find_program(CMAKE_ASM_COMPILER ${TOOLCHAIN_PREFIX}clang) -find_program(CMAKE_AR ${TOOLCHAIN_PREFIX}ar) -find_program(CMAKE_LD ${TOOLCHAIN_PREFIX}ar) -find_program(CMAKE_OBJCOPY ${TOOLCHAIN_PREFIX}objcopy) -find_program(CMAKE_OBJDUMP ${TOOLCHAIN_PREFIX}objdump) - -add_compile_options(--sysroot=${CROSS_ROOTFS}) -add_compile_options(-fPIE) -add_compile_options(-mfloat-abi=soft) -include_directories(SYSTEM ${CROSS_NDK_TOOLCHAIN}/include/c++/4.9.x/) -include_directories(SYSTEM ${CROSS_NDK_TOOLCHAIN}/include/c++/4.9.x/arm-linux-androideabi/) - -set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -B ${CROSS_ROOTFS}/usr/lib/gcc/${TOOLCHAIN}") -set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -L${CROSS_ROOTFS}/lib/${TOOLCHAIN}") -set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} --sysroot=${CROSS_ROOTFS}") -set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -fPIE -pie") - -set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE) -set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE) -set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE) - -set(CMAKE_FIND_ROOT_PATH "${CROSS_ROOTFS}") -set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) -set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) -set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) -set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) diff --git a/eng/common/cross/android/arm64/toolchain.cmake b/eng/common/cross/android/arm64/toolchain.cmake deleted file mode 100644 index 29415899..00000000 --- a/eng/common/cross/android/arm64/toolchain.cmake +++ /dev/null @@ -1,42 +0,0 @@ -set(CROSS_NDK_TOOLCHAIN $ENV{ROOTFS_DIR}/../) -set(CROSS_ROOTFS ${CROSS_NDK_TOOLCHAIN}/sysroot) -set(CLR_CMAKE_PLATFORM_ANDROID "Android") - -set(CMAKE_SYSTEM_NAME Linux) -set(CMAKE_SYSTEM_VERSION 1) -set(CMAKE_SYSTEM_PROCESSOR aarch64) - -## Specify the toolchain -set(TOOLCHAIN "aarch64-linux-android") -set(CMAKE_PREFIX_PATH ${CROSS_NDK_TOOLCHAIN}) -set(TOOLCHAIN_PREFIX ${TOOLCHAIN}-) - -find_program(CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}clang) -find_program(CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}clang++) -find_program(CMAKE_ASM_COMPILER ${TOOLCHAIN_PREFIX}clang) -find_program(CMAKE_AR ${TOOLCHAIN_PREFIX}ar) -find_program(CMAKE_LD ${TOOLCHAIN_PREFIX}ar) -find_program(CMAKE_OBJCOPY ${TOOLCHAIN_PREFIX}objcopy) -find_program(CMAKE_OBJDUMP ${TOOLCHAIN_PREFIX}objdump) - -add_compile_options(--sysroot=${CROSS_ROOTFS}) -add_compile_options(-fPIE) - -## Needed for Android or bionic specific conditionals -add_compile_options(-D__ANDROID__) -add_compile_options(-D__BIONIC__) - -set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -B ${CROSS_ROOTFS}/usr/lib/gcc/${TOOLCHAIN}") -set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -L${CROSS_ROOTFS}/lib/${TOOLCHAIN}") -set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} --sysroot=${CROSS_ROOTFS}") -set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -fPIE -pie") - -set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE) -set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE) -set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE) - -set(CMAKE_FIND_ROOT_PATH "${CROSS_ROOTFS}") -set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) -set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) -set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) -set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) diff --git a/eng/common/cross/build-android-rootfs.sh b/eng/common/cross/build-android-rootfs.sh index adceda87..5e74881e 100755 --- a/eng/common/cross/build-android-rootfs.sh +++ b/eng/common/cross/build-android-rootfs.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash set -e -__NDK_Version=r14 +__NDK_Version=r21 usage() { @@ -16,11 +16,11 @@ usage() echo. echo "By default, the NDK will be downloaded into the cross/android-rootfs/android-ndk-$__NDK_Version directory. If you already have an NDK installation," echo "you can set the NDK_DIR environment variable to have this script use that installation of the NDK." - echo "By default, this script will generate a file, android_platform, in the root of the ROOTFS_DIR directory that contains the RID for the supported and tested Android build: android.21-arm64. This file is to replace '/etc/os-release', which is not available for Android." + echo "By default, this script will generate a file, android_platform, in the root of the ROOTFS_DIR directory that contains the RID for the supported and tested Android build: android.28-arm64. This file is to replace '/etc/os-release', which is not available for Android." exit 1 } -__ApiLevel=21 # The minimum platform for arm64 is API level 21 +__ApiLevel=28 # The minimum platform for arm64 is API level 21 but the minimum version that support glob(3) is 28. See $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/glob.h __BuildArch=arm64 __AndroidArch=aarch64 __AndroidToolchain=aarch64-linux-android @@ -54,12 +54,11 @@ done # Obtain the location of the bash script to figure out where the root of the repo is. __CrossDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +__Android_Cross_Dir="$(cd "$__CrossDir/../../../.tools/android-rootfs" && pwd)" -__Android_Cross_Dir="$__CrossDir/android-rootfs" __NDK_Dir="$__Android_Cross_Dir/android-ndk-$__NDK_Version" -__libunwind_Dir="$__Android_Cross_Dir/libunwind" __lldb_Dir="$__Android_Cross_Dir/lldb" -__ToolchainDir="$__Android_Cross_Dir/toolchain/$__BuildArch" +__ToolchainDir="$__Android_Cross_Dir/android-ndk-$__NDK_Version" if [[ -n "$TOOLCHAIN_DIR" ]]; then __ToolchainDir=$TOOLCHAIN_DIR @@ -89,49 +88,33 @@ if [ ! -d $__lldb_Dir ]; then unzip -q $__Android_Cross_Dir/lldb-2.3.3614996-linux-x86_64.zip -d $__lldb_Dir fi -# Create the RootFS for both arm64 as well as aarch -rm -rf $__Android_Cross_Dir/toolchain +echo "Download dependencies..." +mkdir -p $__Android_Cross_Dir/tmp/$arch/ -echo Generating the $__BuildArch toolchain -$__NDK_Dir/build/tools/make_standalone_toolchain.py --arch $__BuildArch --api $__ApiLevel --install-dir $__ToolchainDir +# combined dependencies for coreclr, installer and libraries +__AndroidPackages="libicu" +__AndroidPackages+=" libandroid-glob" +__AndroidPackages+=" liblzma" +__AndroidPackages+=" krb5" +__AndroidPackages+=" openssl" -# Install the required packages into the toolchain -# TODO: Add logic to get latest pkg version instead of specific version number -rm -rf $__Android_Cross_Dir/deb/ -rm -rf $__Android_Cross_Dir/tmp +for path in $(wget -qO- http://termux.net/dists/stable/main/binary-$__AndroidArch/Packages |\ + grep -A15 "Package: \(${__AndroidPackages// /\\|}\)" | grep -v "static\|tool" | grep Filename); do -mkdir -p $__Android_Cross_Dir/deb/ -mkdir -p $__Android_Cross_Dir/tmp/$arch/ -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libicu_60.2_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libicu_60.2_$__AndroidArch.deb -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libicu-dev_60.2_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libicu-dev_60.2_$__AndroidArch.deb - -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libandroid-glob-dev_0.4_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libandroid-glob-dev_0.4_$__AndroidArch.deb -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libandroid-glob_0.4_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libandroid-glob_0.4_$__AndroidArch.deb -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libandroid-support-dev_22_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libandroid-support-dev_22_$__AndroidArch.deb -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libandroid-support_22_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libandroid-support_22_$__AndroidArch.deb -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/liblzma-dev_5.2.3_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/liblzma-dev_5.2.3_$__AndroidArch.deb -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/liblzma_5.2.3_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/liblzma_5.2.3_$__AndroidArch.deb -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libunwind-dev_1.2.20170304_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libunwind-dev_1.2.20170304_$__AndroidArch.deb -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libunwind_1.2.20170304_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libunwind_1.2.20170304_$__AndroidArch.deb - -echo Unpacking Termux packages -dpkg -x $__Android_Cross_Dir/deb/libicu_60.2_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/libicu-dev_60.2_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/libandroid-glob-dev_0.4_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/libandroid-glob_0.4_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/libandroid-support-dev_22_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/libandroid-support_22_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/liblzma-dev_5.2.3_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/liblzma_5.2.3_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/libunwind-dev_1.2.20170304_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/libunwind_1.2.20170304_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ + if [[ "$path" != "Filename:" ]]; then + echo "Working on: $path" + wget -qO- http://termux.net/$path | dpkg -x - $__Android_Cross_Dir/tmp/$__AndroidArch/ + fi +done cp -R $__Android_Cross_Dir/tmp/$__AndroidArch/data/data/com.termux/files/usr/* $__ToolchainDir/sysroot/usr/ # Generate platform file for build.sh script to assign to __DistroRid echo "Generating platform file..." +echo "RID=android.${__ApiLevel}-${__BuildArch}" > $__ToolchainDir/sysroot/android_platform -echo "RID=android.21-arm64" > $__ToolchainDir/sysroot/android_platform -echo Now run: -echo CONFIG_DIR=\`realpath cross/android/$__BuildArch\` ROOTFS_DIR=\`realpath $__ToolchainDir/sysroot\` ./build.sh cross $__BuildArch skipgenerateversion skipnuget cmakeargs -DENABLE_LLDBPLUGIN=0 - +echo "Now to build coreclr, libraries and installers; run:" +echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \ + --subsetCategory coreclr \ + --subsetCategory libraries \ + --subsetCategory installer diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index 556da2e1..1823804d 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -43,10 +43,30 @@ if(TARGET_ARCH_NAME STREQUAL "armel") endif() endif() -set(CMAKE_SYSROOT "${CROSS_ROOTFS}") -set(CMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr") -set(CMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr") -set(CMAKE_ASM_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr") +if("$ENV{__DistroRid}" MATCHES "android.*") + if(TARGET_ARCH_NAME STREQUAL "arm") + set(ANDROID_ABI armeabi-v7a) + elseif(TARGET_ARCH_NAME STREQUAL "arm64") + set(ANDROID_ABI arm64-v8a) + endif() + + # extract platform number required by the NDK's toolchain + string(REGEX REPLACE ".*\\.([0-9]+)-.*" "\\1" ANDROID_PLATFORM "$ENV{__DistroRid}") + + set(ANDROID_TOOLCHAIN clang) + set(FEATURE_EVENT_TRACE 0) # disable event trace as there is no lttng-ust package in termux repository + set(CMAKE_SYSTEM_LIBRARY_PATH "${CROSS_ROOTFS}/usr/lib") + set(CMAKE_SYSTEM_INCLUDE_PATH "${CROSS_ROOTFS}/usr/include") + + # include official NDK toolchain script + include(${CROSS_ROOTFS}/../build/cmake/android.toolchain.cmake) +else() + set(CMAKE_SYSROOT "${CROSS_ROOTFS}") + + set(CMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr") + set(CMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr") + set(CMAKE_ASM_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr") +endif() # Specify link flags @@ -63,7 +83,7 @@ endif() # Specify compile options -if(TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64)$") +if(TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64)$" AND NOT "$ENV{__DistroRid}" MATCHES "android.*") set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN}) diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 index 2a30ccfd..435e7641 100644 --- a/eng/common/darc-init.ps1 +++ b/eng/common/darc-init.ps1 @@ -7,7 +7,7 @@ param ( . $PSScriptRoot\tools.ps1 -function InstallDarcCli ($darcVersion) { +function InstallDarcCli ($darcVersion, $toolpath) { $darcCliPackageName = 'microsoft.dotnet.darc' $dotnetRoot = InitializeDotNetCli -install:$true @@ -32,13 +32,13 @@ function InstallDarcCli ($darcVersion) { Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity -g" & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g }else { - Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity -g --tool-path '$toolpath'" + Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity --tool-path '$toolpath'" & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath" } } try { - InstallDarcCli $darcVersion + InstallDarcCli $darcVersion $toolpath } catch { Write-Host $_.ScriptStackTrace diff --git a/eng/common/pipeline-logging-functions.ps1 b/eng/common/pipeline-logging-functions.ps1 index a3e1317a..5042baeb 100644 --- a/eng/common/pipeline-logging-functions.ps1 +++ b/eng/common/pipeline-logging-functions.ps1 @@ -31,7 +31,9 @@ function Write-PipelineTelemetryError { $PSBoundParameters.Remove('Category') | Out-Null - $Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message" + if($Force -Or ((Test-Path variable:ci) -And $ci)) { + $Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message" + } $PSBoundParameters.Remove('Message') | Out-Null $PSBoundParameters.Add('Message', $Message) Write-PipelineTaskError @PSBoundParameters diff --git a/eng/common/templates/post-build/channels/generic-public-channel.yml b/eng/common/templates/post-build/channels/generic-public-channel.yml index 29bc1a94..08853ec4 100644 --- a/eng/common/templates/post-build/channels/generic-public-channel.yml +++ b/eng/common/templates/post-build/channels/generic-public-channel.yml @@ -10,6 +10,8 @@ parameters: transportFeed: '' shippingFeed: '' symbolsFeed: '' + # If the channel name is empty, no links will be generated + akaMSChannelName: '' stages: - stage: ${{ parameters.stageName }} @@ -161,6 +163,9 @@ stages: /p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' /p:AzureDevOpsStaticSymbolsFeed='${{ parameters.symbolsFeed }}' /p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' + /p:LatestLinkShortUrlPrefix=dotnet/'${{ parameters.akaMSChannelName }}' + /p:AkaMSClientId=$(akams-client-id) + /p:AkaMSClientSecret=$(akams-client-secret) ${{ parameters.artifactsPublishingAdditionalParameters }} - template: ../../steps/publish-logs.yml diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 05ff3561..7be5b0bf 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -199,6 +199,7 @@ stages: symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} stageName: 'NetCore_Dev5_Publish' channelName: '.NET Core 5 Dev' + akaMSChannelName: 'net5/dev' channelId: ${{ parameters.NetCoreDev5ChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' @@ -212,6 +213,7 @@ stages: symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} stageName: 'Net5_Preview1_Publish' channelName: '.NET 5 Preview 1' + akaMSChannelName: 'net5/preview1' channelId: ${{ parameters.Net5Preview1ChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' @@ -225,6 +227,7 @@ stages: symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} stageName: 'Net5_Preview2_Publish' channelName: '.NET 5 Preview 2' + akaMSChannelName: 'net5/preview2' channelId: ${{ parameters.Net5Preview2ChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' @@ -238,6 +241,7 @@ stages: symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} stageName: 'Net_Eng_Latest_Publish' channelName: '.NET Eng - Latest' + akaMSChannelName: 'eng/daily' channelId: ${{ parameters.NetEngLatestChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' @@ -251,6 +255,7 @@ stages: symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} stageName: 'Net_Eng_Validation_Publish' channelName: '.NET Eng - Validation' + akaMSChannelName: 'eng/validation' channelId: ${{ parameters.NetEngValidationChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' @@ -264,6 +269,7 @@ stages: symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} stageName: 'General_Testing_Publish' channelName: 'General Testing' + akaMSChannelName: 'generaltesting' channelId: ${{ parameters.GeneralTestingChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json' diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index a45302b3..60c1cd89 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -635,7 +635,7 @@ function MSBuild-Core() { $exitCode = Exec-Process $buildTool.Path $cmdArgs if ($exitCode -ne 0) { - Write-PipelineTelemetryError Category 'Build' -Message 'Build failed.' + Write-PipelineTelemetryError -Category 'Build' -Message 'Build failed.' $buildLog = GetMSBuildBinaryLogCommandLineArgument $args if ($buildLog -ne $null) { diff --git a/global.json b/global.json index 014ace9d..6be46234 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.1.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20151.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20158.2" } } From 51237ede1990ec8b6f6a7eb8884ff71ff46df840 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2020 12:41:42 +0000 Subject: [PATCH 064/161] Update dependencies from https://github.com/dotnet/arcade build 20200312.3 (#591) - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.20162.3 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 62e72933..fafaac1f 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 527179abfbb5cd738035c4ac2d2566c571ce85db + 16d4350184cf362cd59807b589d1c93803025abc diff --git a/global.json b/global.json index 6be46234..86601843 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.1.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20158.2" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20162.3" } } From 30575c49201fbb62f93e6b9f36004f68943cd307 Mon Sep 17 00:00:00 2001 From: IgorKustov-ChathamFinancial <60136098+IgorKustov-ChathamFinancial@users.noreply.github.com> Date: Sat, 21 Mar 2020 13:08:56 -0400 Subject: [PATCH 065/161] Fixed 404 for Source Link specification link (#592) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index aa286f53..0a0250cb 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Source Link is a language- and source-control agnostic system for providing first-class source debugging experiences for binaries. The goal of the project is to enable anyone building [NuGet libraries to provide source debugging](https://github.com/dotnet/designs/blob/master/accepted/diagnostics/debugging-with-symbols-and-sources.md) for their users with almost no effort. Microsoft libraries, such as .NET Core and Roslyn have enabled Source Link. Source Link is supported by Microsoft. -Source Link is a [set of packages](https://www.nuget.org/packages?q=Microsoft.SourceLink) and a [specification](https://github.com/dotnet/designs/blob/master/accepted/diagnostics/source-link.md#source-link-file-specification) for describing source control metadata that can be embedded in symbols, binaries and packages. +Source Link is a [set of packages](https://www.nuget.org/packages?q=Microsoft.SourceLink) and a [specification](https://github.com/dotnet/designs/blob/master/accepted/2020/diagnostics/source-link.md#source-link-file-specification) for describing source control metadata that can be embedded in symbols, binaries and packages. Visual Studio 15.3+ supports reading Source Link information from symbols while debugging. It downloads and displays the appropriate commit-specific source for users, such as from [raw.githubusercontent](https://raw.githubusercontent.com/dotnet/roslyn/681cbc414542ffb9fb13ded613d26a88ea73a44b/src/VisualStudio/Core/Def/Implementation/ProjectSystem/AbstractProject.cs), enabling breakpoints and all other sources debugging experience on arbitrary NuGet dependencies. Visual Studio 15.7+ supports downloading source files from private GitHub and Azure DevOps (former VSTS) repositories that require authentication. From 837d0d1b713d7de671dbb3aa35cbac74176a5cb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Matou=C5=A1ek?= Date: Tue, 24 Mar 2020 11:28:26 -0700 Subject: [PATCH 066/161] Create README.md --- docs/StandardCI/README.md | 79 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 docs/StandardCI/README.md diff --git a/docs/StandardCI/README.md b/docs/StandardCI/README.md new file mode 100644 index 00000000..184cfce0 --- /dev/null +++ b/docs/StandardCI/README.md @@ -0,0 +1,79 @@ +# Standardized Environment Variables for CI Services + +The .NET Core SDK is implementing new scenarios that require source control manager information. We want to make it very easy to embed repo-specific information in built assets as a means of creating a strong relationship between binary and source. + +To make it possible to provide application platform provided experiences across CI services, we need a standardized set of environment variables that are supported across those same CI services. Ideally, this set of environment variables would be supported across multiple CI services and useful for multiple application environments, not just .NET. + +## Context + +The .NET Core SDK Team is implementing a set of scenarios that establish a strong relationship between binary and source. Some of these scenarios are purely for auditing/pedigree analysis and others are for debugging. + +The more simple scenarios simply record source repo information that is readily available into various informational fields in built assets. + +Source control information needed: + +* Commit ID/hash +* Repository URL +* Source Control Manager name (`git`, `tfvc`, etc.) + +Places to store this information: + +* AssemblyInformationalVersion attribute. +* NuGet nuspec file (manifest for NuGet package). + +This information doesn't need to be in a specific format, although accurate would be good. It should be descriptive and intuitive per the given source control system so that human readers can work their way back to source (assuming they have access). + +The following examples demonstrate how the Roslyn team has implemented this same scenario in their own build. It should be easy for anyone to include the same information in built artifacts. + +Example -- Microsoft.CodeAnalysis.CSharp.dll: + +```csharp +[assembly: AssemblyInformationalVersion("2.4.0-beta1-62122-04+ab56a4a6c32268d925014a3e45ddce61fba715cd")] +``` + +Example -- Microsoft.Net.Compilers.nupkg (NuGet package): + +```xml + +``` + +Note that the final implementation provided by the .NET Core SDK might look slightly differently from the Roslyn examples, but it will be similar. There will be a separate spec on this feature. This document is focussed on the CI service integration. + +## Proposed Standard Environment Variables + +The .NET Core SDK needs are oriented around source control. As a result, the initial list is source control oriented, but there is no affinity to source control on the general idea of standardized environment variables. + +It is important that these environment variables do not conflict with other variables. To avoid that, all environment variables will be prepended with `STANDARD_CI_`. This name is a first proposal for the idea and it may get changed based on feedback. + +* **STANDARD\_CI\_SOURCE\_REVISION\_ID** -- Commit hash / ID; Example: 2ba93796dcf132de447886d4d634414ee8cb069d +* **STANDARD\_CI\_REPOSITORY\_URL** -- URL for repository; Example: https://github.com/dotnet/corefx +* **STANDARD\_CI\_REPOSITORY\_TYPE** -- Source control manager name; Example: `git`, `TFVC`, `mercurial`, `svn` + +The following strings are well-known values for `STANDARD_CI_REPOSITORY_TYPE`. Other values can be used for source control managers not listed in this table. + +| `STANDARD_CI_REPOSITORY_TYPE` | Source Control Manager | +| ------------------------------ | ---------------------- | +| `git` | [git](https://git-scm.com) | +| `tfvc` | [Team Foundation Version Control](https://docs.microsoft.com/en-us/vsts/tfvc) | +| `svn` | [Apache Subversion](https://subversion.apache.org) | +| `mercurial` | [Mercurial](https://www.mercurial-scm.org) | + +## Support from CI Services + +This plan will only work if CI services decide to support these environment variables. An important question is whether CI services have similar environment variables already. The table below suggests that the information we need is already available. An arbitrary sample of CI services were picked for this exercise. + +| Environment Variable | VSTS | Travis CI| AppVeyor | Circle CI | AWS CodeBuild | Team City | OpenShift | +| -------------------- | ---- | -------- | -------- | --------- | ------------- | --------- | --------- | +|STANDARD\_CI\_SOURCE\_REVISION\_ID | BUILD\_SOURCEVERSION | TRAVIS\_COMMIT |APPVEYOR\_REPO\_COMMIT | CIRCLE\_SHA1 | CODEBUILD\_RESOLVED\_SOURCE\_VERSION | build.vcs.number | OPENSHIFT\_BUILD\_COMMIT | +|STANDARD\_CI\_REPOSITORY\_URL|BUILD\_REPOSITORY\_URI| | | CIRCLE\_REPOSITORY\_URL | CODEBUILD\_SOURCE\_REPO\_URL | vcsroot.url | OPENSHIFT\_BUILD\_SOURCE | +|STANDARD\_CI\_REPOSITORY\_TYPE | | |APPVEYOR\_REPO\_SCM | | + +The [VSTS](https://www.visualstudio.com/team-services/) team has graciously agreed to publish environment variables in the proposed STANDARD\_CI format. + +* [VSTS](https://www.visualstudio.com/team-services/) -- [environment variables](https://docs.microsoft.com/en-us/vsts/build-release/concepts/definitions/build/variables?tabs=batch#predefined-variables) +* [Travis CI](https://travis-ci.org/) -- [environment variables](https://docs.travis-ci.com/user/environment-variables/#Default-Environment-Variables) +* [AppVeyor](https://www.appveyor.com/) -- [environment variables](https://www.appveyor.com/docs/environment-variables/) +* [Circle CI](https://circleci.com) -- [environment variables](https://circleci.com/docs/2.0/env-vars) +* [AWS CodeBuild](https://aws.amazon.com/codebuild/) -- [environment variables](http://docs.aws.amazon.com/codebuild/latest/userguide/build-env-ref-env-vars.html) +* [Team City](https://confluence.jetbrains.com/display/TCDL/Predefined+Build+Parameters) +* [OpenShift](https://docs.openshift.com/enterprise/3.1/dev_guide/builds.html#output-image-environment-variables) From f90e67cd2a6aa3821fb1f3e735950dc94e19cf72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Matou=C5=A1ek?= Date: Tue, 24 Mar 2020 11:29:07 -0700 Subject: [PATCH 067/161] Update Microsoft.Build.StandardCI.props --- .../build/Microsoft.Build.StandardCI.props | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.Build.StandardCI/build/Microsoft.Build.StandardCI.props b/src/Microsoft.Build.StandardCI/build/Microsoft.Build.StandardCI.props index e85911ca..353bd638 100644 --- a/src/Microsoft.Build.StandardCI/build/Microsoft.Build.StandardCI.props +++ b/src/Microsoft.Build.StandardCI/build/Microsoft.Build.StandardCI.props @@ -2,7 +2,7 @@ $(STANDARD_CI_REPOSITORY_TYPE) - \ No newline at end of file + From 5986a9150d2281602b4b5664a4b36e67bfe03d5c Mon Sep 17 00:00:00 2001 From: Matt Mitchell Date: Sat, 28 Mar 2020 12:49:47 -0700 Subject: [PATCH 068/161] Feed cleanup (#594) Switch away from blob feeds. --- NuGet.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NuGet.config b/NuGet.config index 9edf4552..db2ee738 100644 --- a/NuGet.config +++ b/NuGet.config @@ -3,7 +3,7 @@ - + From 97b6bcf51bf6983449f8eae25dd3a6c19c631192 Mon Sep 17 00:00:00 2001 From: Immo Landwerth Date: Fri, 3 Apr 2020 10:52:24 -0700 Subject: [PATCH 069/161] Link Code of Conduct (#598) --- CODE-OF-CONDUCT.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 CODE-OF-CONDUCT.md diff --git a/CODE-OF-CONDUCT.md b/CODE-OF-CONDUCT.md new file mode 100644 index 00000000..775f221c --- /dev/null +++ b/CODE-OF-CONDUCT.md @@ -0,0 +1,6 @@ +# Code of Conduct + +This project has adopted the code of conduct defined by the Contributor Covenant +to clarify expected behavior in our community. + +For more information, see the [.NET Foundation Code of Conduct](https://dotnetfoundation.org/code-of-conduct). From 5f942d6ed59f123a369dac3c6a4152574d70a3b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Matou=C5=A1ek?= Date: Sat, 4 Apr 2020 10:52:29 -0700 Subject: [PATCH 070/161] Update README.md --- docs/README.md | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/docs/README.md b/docs/README.md index 49758ff9..bd80c61d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,10 +1,10 @@ ## Source Control and Source Link Packages MSBuild has no built-in knowledge of any source control system or provider. In order to retrieve information from the source control the project needs to include -a package reference to the appropriate source control package. Microsoft provides source control packages for Git and TFVC managers (currently as pre-release packages): +a package reference to the appropriate source control package. Microsoft provides source control packages for Git and TFVC managers: - - Microsoft.Build.Tasks.Git - - Microsoft.Build.Tasks.Tfvc + - Microsoft.Build.Tasks.Git (release) + - Microsoft.Build.Tasks.Tfvc (experimental) These packages implement a protocol defined by MSBuild that allows extraction of necessary information from the source control system during build. This protocol can be implemented by third party packages in order to support other source control systems. @@ -22,18 +22,22 @@ Having this information available enables the following features: To generate Source Link having just the source control package is not sufficient, since various source control providers (hosts) differ in the way how they expose the content of the hosted repositories. A package specific to the provider is needed. -The following Source Link pre-release packages are currently available: +The following Source Link packages have been released by Microsoft: - Microsoft.SourceLink.GitHub (depends on Microsoft.Build.Tasks.Git package) +- Microsoft.SourceLink.AzureRepos.Git (depends on Microsoft.Build.Tasks.Git package) +- Microsoft.SourceLink.AzureDevOpsServer.Git (depends on Microsoft.Build.Tasks.Git package) - Microsoft.SourceLink.GitLab (depends on Microsoft.Build.Tasks.Git package) -- Microsoft.SourceLink.Vsts.Git (depends on Microsoft.Build.Tasks.Git package) -- Microsoft.SourceLink.Vsts.Tfvc (depends on Microsoft.Build.Tasks.Tfvc package) +- Microsoft.SourceLink.Bitbucket.Git (depends on Microsoft.Build.Tasks.Git package) + +In addition an experimental package is available for TFVC server: +- Microsoft.SourceLink.AzureRepos.Tfvc (depends on experimental Microsoft.Build.Tasks.Tfvc package) The system is extensible and custom packages that handle other source control providers can be developed and used. See [Custom Source Link packages](#creating-custom-sourcelink-packages) for details. Each Source Link package depends on the corresponding source control package. Referencing a Source Link package makes the dependent source control package also referenced, thus providing the other source control features to the project. -Note that it is possible and supported to reference multiple Source Link packages in a single project provided they depend on the same source control package. This is necessary when the project sources are stored in mutliple submodules hosted by different providers (e.g. VSTS repository containing a GitHub submodule). See [Configuring Projects with Multiple Source Link Providers](#configuring-projects-with-multiple-sourcelink-providers) for details. +Note that it is possible and supported to reference multiple Source Link packages in a single project provided they depend on the same source control package. This is necessary when the project sources are stored in mutliple submodules hosted by different providers (e.g. Azure Repos repository containing a GitHub submodule). See [Configuring Projects with Multiple Source Link Providers](#configuring-projects-with-multiple-sourcelink-providers) for details. ## Basic Settings @@ -110,15 +114,16 @@ Each package defines an msbuild item group named `SourceLink{provider}Host`, whe *Content URL* is the URL where the raw source files can be downloaded from. Items of `SourceLink*Host` item group allow to specify the content URL if necessary. The content URL doesn't need to be specified in most cases as it is inferred from the domain. -The default content URLs for each package is listed below: +The default content URLs for each package is listed below ([GetSourceLinkUrlGitTask.GetDefaultContentUriFromHostUri](https://github.com/dotnet/sourcelink/blob/master/src/Common/GetSourceLinkUrlGitTask.cs#L50) API): -| | content URL | -|:----------:|:------------------:| -|**GitLab** |https://{domain} | -|**GitHub** |https://{domain}/raw| -|**Vsts.Git**|https://{domain} | +| | content URL | +|:----------------:|:------------------:| +|**GitLab** |https://{domain} | +|**GitHub** |https://{domain}/raw| +|**AzureRepos.Git**|https://{domain} | +|**BitBucket** |https://{domain} | -To override the above defaults specify `ContentUrl` metadata on the item. For example, GitHub.com server provides content on a CDN domain `raw.githubusercontent.com`: +To override the above defaults specify `ContentUrl` metadata on the item in [`build/*.props`](https://github.com/dotnet/sourcelink/blob/master/src/SourceLink.GitHub/build/Microsoft.SourceLink.GitHub.props) in the Source Link package. For example, GitHub.com server provides content on a CDN domain `raw.githubusercontent.com`: ```xml From b3cb6c4be71b67e3ef829a6cfde95e65d35f6ab7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Matou=C5=A1ek?= Date: Sat, 4 Apr 2020 11:06:12 -0700 Subject: [PATCH 071/161] Update README.md --- docs/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/README.md b/docs/README.md index bd80c61d..88299796 100644 --- a/docs/README.md +++ b/docs/README.md @@ -33,11 +33,11 @@ The following Source Link packages have been released by Microsoft: In addition an experimental package is available for TFVC server: - Microsoft.SourceLink.AzureRepos.Tfvc (depends on experimental Microsoft.Build.Tasks.Tfvc package) -The system is extensible and custom packages that handle other source control providers can be developed and used. See [Custom Source Link packages](#creating-custom-sourcelink-packages) for details. +The system is extensible and custom packages that handle other source control providers can be developed and used. See [Custom Source Link packages](#creating-custom-source-link-packages) for details. Each Source Link package depends on the corresponding source control package. Referencing a Source Link package makes the dependent source control package also referenced, thus providing the other source control features to the project. -Note that it is possible and supported to reference multiple Source Link packages in a single project provided they depend on the same source control package. This is necessary when the project sources are stored in mutliple submodules hosted by different providers (e.g. Azure Repos repository containing a GitHub submodule). See [Configuring Projects with Multiple Source Link Providers](#configuring-projects-with-multiple-sourcelink-providers) for details. +Note that it is possible and supported to reference multiple Source Link packages in a single project provided they depend on the same source control package. This is necessary when the project sources are stored in mutliple submodules hosted by different providers (e.g. Azure Repos repository containing a GitHub submodule). See [Configuring Projects with Multiple Source Link Providers](#configuring-projects-with-multiple-source-link-providers) for details. ## Basic Settings @@ -123,7 +123,7 @@ The default content URLs for each package is listed below ([GetSourceLinkUrlGitT |**AzureRepos.Git**|https://{domain} | |**BitBucket** |https://{domain} | -To override the above defaults specify `ContentUrl` metadata on the item in [`build/*.props`](https://github.com/dotnet/sourcelink/blob/master/src/SourceLink.GitHub/build/Microsoft.SourceLink.GitHub.props) in the Source Link package. For example, GitHub.com server provides content on a CDN domain `raw.githubusercontent.com`: +To override the above defaults specify `ContentUrl` metadata on the item in [`build/{PackageName}.props`](https://github.com/dotnet/sourcelink/blob/master/src/SourceLink.GitHub/build/Microsoft.SourceLink.GitHub.props) in the Source Link package. For example, GitHub.com server provides content on a CDN domain `raw.githubusercontent.com`: ```xml From db7c31800400b6203d2b162255fa46cbaf2f04aa Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 6 Apr 2020 12:26:59 +0000 Subject: [PATCH 072/161] [master] Update dependencies from dotnet/arcade (#593) * Update dependencies from https://github.com/dotnet/arcade build 20200321.1 - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.20171.1 * Update dependencies from https://github.com/dotnet/arcade build 20200401.2 - Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.20201.2 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 4 +- eng/common/cross/build-android-rootfs.sh | 39 +++++---- eng/common/cross/build-rootfs.sh | 79 +++++++++++++------ eng/common/cross/toolchain.cmake | 15 +++- eng/common/performance/performance-setup.ps1 | 7 ++ eng/common/performance/performance-setup.sh | 7 ++ eng/common/pipeline-logging-functions.ps1 | 6 +- eng/common/sdl/execute-all-sdl-tools.ps1 | 4 + eng/common/sdl/extract-artifact-packages.ps1 | 8 +- eng/common/sdl/init-sdl.ps1 | 6 +- eng/common/sdl/push-gdn.ps1 | 6 +- eng/common/sdl/run-sdl.ps1 | 8 +- eng/common/templates/job/job.yml | 4 + .../templates/job/publish-build-assets.yml | 6 ++ .../templates/post-build/common-variables.yml | 11 ++- .../templates/post-build/post-build.yml | 54 ++++++++++--- eng/common/tools.sh | 4 +- global.json | 2 +- 18 files changed, 201 insertions(+), 69 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index fafaac1f..3ad788b8 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 16d4350184cf362cd59807b589d1c93803025abc + bce0a98620c1c5a110b2bba9912f3d5929069c6b diff --git a/eng/common/cross/build-android-rootfs.sh b/eng/common/cross/build-android-rootfs.sh index 5e74881e..e7f12edb 100755 --- a/eng/common/cross/build-android-rootfs.sh +++ b/eng/common/cross/build-android-rootfs.sh @@ -53,12 +53,20 @@ for i in "$@" done # Obtain the location of the bash script to figure out where the root of the repo is. -__CrossDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -__Android_Cross_Dir="$(cd "$__CrossDir/../../../.tools/android-rootfs" && pwd)" +__ScriptBaseDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -__NDK_Dir="$__Android_Cross_Dir/android-ndk-$__NDK_Version" -__lldb_Dir="$__Android_Cross_Dir/lldb" -__ToolchainDir="$__Android_Cross_Dir/android-ndk-$__NDK_Version" +__CrossDir="$__ScriptBaseDir/../../../.tools/android-rootfs" + +if [[ ! -f "$__CrossDir" ]]; then + mkdir -p "$__CrossDir" +fi + +# Resolve absolute path to avoid `../` in build logs +__CrossDir="$( cd "$__CrossDir" && pwd )" + +__NDK_Dir="$__CrossDir/android-ndk-$__NDK_Version" +__lldb_Dir="$__CrossDir/lldb" +__ToolchainDir="$__CrossDir/android-ndk-$__NDK_Version" if [[ -n "$TOOLCHAIN_DIR" ]]; then __ToolchainDir=$TOOLCHAIN_DIR @@ -77,19 +85,20 @@ echo "Target Toolchain location: $__ToolchainDir" if [ ! -d $__NDK_Dir ]; then echo Downloading the NDK into $__NDK_Dir mkdir -p $__NDK_Dir - wget -nv -nc --show-progress https://dl.google.com/android/repository/android-ndk-$__NDK_Version-linux-x86_64.zip -O $__Android_Cross_Dir/android-ndk-$__NDK_Version-linux-x86_64.zip - unzip -q $__Android_Cross_Dir/android-ndk-$__NDK_Version-linux-x86_64.zip -d $__Android_Cross_Dir + wget -q --progress=bar:force:noscroll --show-progress https://dl.google.com/android/repository/android-ndk-$__NDK_Version-linux-x86_64.zip -O $__CrossDir/android-ndk-$__NDK_Version-linux-x86_64.zip + unzip -q $__CrossDir/android-ndk-$__NDK_Version-linux-x86_64.zip -d $__CrossDir fi if [ ! -d $__lldb_Dir ]; then mkdir -p $__lldb_Dir echo Downloading LLDB into $__lldb_Dir - wget -nv -nc --show-progress https://dl.google.com/android/repository/lldb-2.3.3614996-linux-x86_64.zip -O $__Android_Cross_Dir/lldb-2.3.3614996-linux-x86_64.zip - unzip -q $__Android_Cross_Dir/lldb-2.3.3614996-linux-x86_64.zip -d $__lldb_Dir + wget -q --progress=bar:force:noscroll --show-progress https://dl.google.com/android/repository/lldb-2.3.3614996-linux-x86_64.zip -O $__CrossDir/lldb-2.3.3614996-linux-x86_64.zip + unzip -q $__CrossDir/lldb-2.3.3614996-linux-x86_64.zip -d $__lldb_Dir fi echo "Download dependencies..." -mkdir -p $__Android_Cross_Dir/tmp/$arch/ +__TmpDir=$__CrossDir/tmp/$__BuildArch/ +mkdir -p "$__TmpDir" # combined dependencies for coreclr, installer and libraries __AndroidPackages="libicu" @@ -103,11 +112,11 @@ for path in $(wget -qO- http://termux.net/dists/stable/main/binary-$__AndroidArc if [[ "$path" != "Filename:" ]]; then echo "Working on: $path" - wget -qO- http://termux.net/$path | dpkg -x - $__Android_Cross_Dir/tmp/$__AndroidArch/ + wget -qO- http://termux.net/$path | dpkg -x - "$__TmpDir" fi done -cp -R $__Android_Cross_Dir/tmp/$__AndroidArch/data/data/com.termux/files/usr/* $__ToolchainDir/sysroot/usr/ +cp -R "$__TmpDir/data/data/com.termux/files/usr/"* "$__ToolchainDir/sysroot/usr/" # Generate platform file for build.sh script to assign to __DistroRid echo "Generating platform file..." @@ -115,6 +124,8 @@ echo "RID=android.${__ApiLevel}-${__BuildArch}" > $__ToolchainDir/sysroot/androi echo "Now to build coreclr, libraries and installers; run:" echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \ - --subsetCategory coreclr \ - --subsetCategory libraries \ + --subsetCategory coreclr +echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \ + --subsetCategory libraries +echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \ --subsetCategory installer diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index a23f895b..2cdd82d3 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -2,15 +2,16 @@ usage() { - echo "Usage: $0 [BuildArch] [LinuxCodeName] [lldbx.y] [--skipunmount] --rootfsdir ]" + echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [--skipunmount] --rootfsdir ]" echo "BuildArch can be: arm(default), armel, arm64, x86" - echo "LinuxCodeName - optional, Code name for Linux, can be: trusty, xenial(default), zesty, bionic, alpine. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." - echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine" + echo "CodeName - optional, Code name for Linux, can be: trusty, xenial(default), zesty, bionic, alpine. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." + echo " for FreeBSD can be: freebsd11 or freebsd12." + echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FReeBSD" echo "--skipunmount - optional, will skip the unmount of rootfs folder." exit 1 } -__LinuxCodeName=xenial +__CodeName=xenial __CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) __InitialDir=$PWD __BuildArch=arm @@ -53,6 +54,15 @@ __AlpinePackages+=" krb5-dev" __AlpinePackages+=" openssl-dev" __AlpinePackages+=" zlib-dev" +__FreeBSDBase="12.1-RELEASE" +__FreeBSDPkg="1.10.5" +__FreeBSDPackages="libunwind" +__FreeBSDPackages+=" icu" +__FreeBSDPackages+=" libinotify" +__FreeBSDPackages+=" lttng-ust" +__FreeBSDPackages+=" llvm-90" +__FreeBSDPackages+=" krb5" + __UnprocessedBuildArgs= while :; do if [ $# -le 0 ]; then @@ -81,7 +91,7 @@ while :; do __BuildArch=armel __UbuntuArch=armel __UbuntuRepo="http://ftp.debian.org/debian/" - __LinuxCodeName=jessie + __CodeName=jessie ;; x86) __BuildArch=x86 @@ -110,36 +120,36 @@ while :; do unset __LLDB_Package ;; trusty) # Ubuntu 14.04 - if [ "$__LinuxCodeName" != "jessie" ]; then - __LinuxCodeName=trusty + if [ "$__CodeName" != "jessie" ]; then + __CodeName=trusty fi ;; xenial) # Ubuntu 16.04 - if [ "$__LinuxCodeName" != "jessie" ]; then - __LinuxCodeName=xenial + if [ "$__CodeName" != "jessie" ]; then + __CodeName=xenial fi ;; zesty) # Ubuntu 17.04 - if [ "$__LinuxCodeName" != "jessie" ]; then - __LinuxCodeName=zesty + if [ "$__CodeName" != "jessie" ]; then + __CodeName=zesty fi ;; bionic) # Ubuntu 18.04 - if [ "$__LinuxCodeName" != "jessie" ]; then - __LinuxCodeName=bionic + if [ "$__CodeName" != "jessie" ]; then + __CodeName=bionic fi ;; jessie) # Debian 8 - __LinuxCodeName=jessie + __CodeName=jessie __UbuntuRepo="http://ftp.debian.org/debian/" ;; stretch) # Debian 9 - __LinuxCodeName=stretch + __CodeName=stretch __UbuntuRepo="http://ftp.debian.org/debian/" __LLDB_Package="liblldb-6.0-dev" ;; buster) # Debian 10 - __LinuxCodeName=buster + __CodeName=buster __UbuntuRepo="http://ftp.debian.org/debian/" __LLDB_Package="liblldb-6.0-dev" ;; @@ -149,14 +159,22 @@ while :; do usage; exit 1; fi - __LinuxCodeName= + __CodeName= __UbuntuRepo= __Tizen=tizen ;; alpine) - __LinuxCodeName=alpine + __CodeName=alpine __UbuntuRepo= ;; + freebsd11) + __FreeBSDBase="11.3-RELEASE" + ;& + freebsd12) + __CodeName=freebsd + __BuildArch=x64 + __SkipUnmount=1 + ;; --skipunmount) __SkipUnmount=1 ;; @@ -192,7 +210,7 @@ if [ -d "$__RootfsDir" ]; then rm -rf $__RootfsDir fi -if [[ "$__LinuxCodeName" == "alpine" ]]; then +if [[ "$__CodeName" == "alpine" ]]; then __ApkToolsVersion=2.9.1 __AlpineVersion=3.9 __ApkToolsDir=$(mktemp -d) @@ -218,9 +236,24 @@ if [[ "$__LinuxCodeName" == "alpine" ]]; then add $__AlpinePackagesEdgeTesting rm -r $__ApkToolsDir -elif [[ -n $__LinuxCodeName ]]; then - qemu-debootstrap --arch $__UbuntuArch $__LinuxCodeName $__RootfsDir $__UbuntuRepo - cp $__CrossDir/$__BuildArch/sources.list.$__LinuxCodeName $__RootfsDir/etc/apt/sources.list +elif [[ "$__CodeName" == "freebsd" ]]; then + mkdir -p $__RootfsDir/usr/local/etc + wget -O - https://download.freebsd.org/ftp/releases/amd64/${__FreeBSDBase}/base.txz | tar -C $__RootfsDir -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version + # For now, ask for 11 ABI even on 12. This can be revisited later. + echo "ABI = \"FreeBSD:11:amd64\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > ${__RootfsDir}/usr/local/etc/pkg.conf + echo "FreeBSD: { url: "pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"${__RootfsDir}/usr/share/keys/pkg\", enabled: yes }" > ${__RootfsDir}/etc/pkg/FreeBSD.conf + mkdir -p $__RootfsDir/tmp + # get and build package manager + wget -O - https://github.com/freebsd/pkg/archive/${__FreeBSDPkg}.tar.gz | tar -C $__RootfsDir/tmp -zxf - + cd $__RootfsDir/tmp/pkg-${__FreeBSDPkg} + ./autogen.sh && ./configure --prefix=$__RootfsDir/host && make install + rm -rf $__RootfsDir/tmp/pkg-${__FreeBSDPkg} + # install packages we need. + $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf update + $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf install --yes $__FreeBSDPackages +elif [[ -n $__CodeName ]]; then + qemu-debootstrap --arch $__UbuntuArch $__CodeName $__RootfsDir $__UbuntuRepo + cp $__CrossDir/$__BuildArch/sources.list.$__CodeName $__RootfsDir/etc/apt/sources.list chroot $__RootfsDir apt-get update chroot $__RootfsDir apt-get -f -y install chroot $__RootfsDir apt-get -y install $__UbuntuPackages @@ -230,7 +263,7 @@ elif [[ -n $__LinuxCodeName ]]; then umount $__RootfsDir/* fi - if [[ "$__BuildArch" == "arm" && "$__LinuxCodeName" == "trusty" ]]; then + if [[ "$__BuildArch" == "arm" && "$__CodeName" == "trusty" ]]; then pushd $__RootfsDir patch -p1 < $__CrossDir/$__BuildArch/trusty.patch patch -p1 < $__CrossDir/$__BuildArch/trusty-lttng-2.4.patch diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index 1823804d..534f1d19 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -1,7 +1,11 @@ set(CROSS_ROOTFS $ENV{ROOTFS_DIR}) set(TARGET_ARCH_NAME $ENV{TARGET_BUILD_ARCH}) -set(CMAKE_SYSTEM_NAME Linux) +if(EXISTS ${CROSS_ROOTFS}/bin/freebsd-version) + set(CMAKE_SYSTEM_NAME FreeBSD) +else() + set(CMAKE_SYSTEM_NAME Linux) +endif() set(CMAKE_SYSTEM_VERSION 1) if(TARGET_ARCH_NAME STREQUAL "armel") @@ -27,6 +31,9 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm64") elseif(TARGET_ARCH_NAME STREQUAL "x86") set(CMAKE_SYSTEM_PROCESSOR i686) set(TOOLCHAIN "i686-linux-gnu") +elseif (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") + set(CMAKE_SYSTEM_PROCESSOR "x86_64") + set(triple "x86_64-unknown-freebsd11") else() message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only armel, arm, arm64 and x86 are supported!") endif() @@ -60,6 +67,12 @@ if("$ENV{__DistroRid}" MATCHES "android.*") # include official NDK toolchain script include(${CROSS_ROOTFS}/../build/cmake/android.toolchain.cmake) +elseif (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") + # we cross-compile by instructing clang + set(CMAKE_C_COMPILER_TARGET ${triple}) + set(CMAKE_CXX_COMPILER_TARGET ${triple}) + set(CMAKE_ASM_COMPILER_TARGET ${triple}) + set(CMAKE_SYSROOT "${CROSS_ROOTFS}") else() set(CMAKE_SYSROOT "${CROSS_ROOTFS}") diff --git a/eng/common/performance/performance-setup.ps1 b/eng/common/performance/performance-setup.ps1 index e3376699..1763a1a9 100644 --- a/eng/common/performance/performance-setup.ps1 +++ b/eng/common/performance/performance-setup.ps1 @@ -53,6 +53,13 @@ if ($Internal) { $CommonSetupArguments="--channel master --queue $Queue --build-number $BuildNumber --build-configs $Configurations --architecture $Architecture" $SetupArguments = "--repository https://github.com/$Repository --branch $Branch --get-perf-hash --commit-sha $CommitSha $CommonSetupArguments" + +#This grabs the LKG version number of dotnet and passes it to our scripts +$VersionJSON = Get-Content global.json | ConvertFrom-Json +$DotNetVersion = $VersionJSON.tools.dotnet +$SetupArguments = "--dotnet-versions $DotNetVersion $SetupArguments" + + if ($RunFromPerformanceRepo) { $SetupArguments = "--perf-hash $CommitSha $CommonSetupArguments" diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index 94a04e0f..b9eecf94 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -167,6 +167,13 @@ fi common_setup_arguments="--channel master --queue $queue --build-number $build_number --build-configs $configurations --architecture $architecture" setup_arguments="--repository https://github.com/$repository --branch $branch --get-perf-hash --commit-sha $commit_sha $common_setup_arguments" + +# Get the tools section from the global.json. +# This grabs the LKG version number of dotnet and passes it to our scripts +dotnet_version=`cat global.json | python3 -c 'import json,sys;obj=json.load(sys.stdin);print(obj["tools"]["dotnet"])'` +setup_arguments="--dotnet-versions $dotnet_version $setup_arguments" + + if [[ "$run_from_perf_repo" = true ]]; then payload_directory= workitem_directory=$source_directory diff --git a/eng/common/pipeline-logging-functions.ps1 b/eng/common/pipeline-logging-functions.ps1 index 5042baeb..8484451f 100644 --- a/eng/common/pipeline-logging-functions.ps1 +++ b/eng/common/pipeline-logging-functions.ps1 @@ -67,12 +67,12 @@ function Write-PipelineTaskError { } if(($Type -ne 'error') -and ($Type -ne 'warning')) { - Write-Host $Message - return + Write-Host $Message + return } $PSBoundParameters.Remove('Force') | Out-Null if(-not $PSBoundParameters.ContainsKey('Type')) { - $PSBoundParameters.Add('Type', 'error') + $PSBoundParameters.Add('Type', 'error') } Write-LogIssue @PSBoundParameters } diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1 index 9db582f2..b7f61f9a 100644 --- a/eng/common/sdl/execute-all-sdl-tools.ps1 +++ b/eng/common/sdl/execute-all-sdl-tools.ps1 @@ -33,6 +33,10 @@ try { $disableConfigureToolsetImport = $true $LASTEXITCODE = 0 + # `tools.ps1` checks $ci to perform some actions. Since the SDL + # scripts don't necessarily execute in the same agent that run the + # build.ps1/sh script this variable isn't automatically set. + $ci = $true . $PSScriptRoot\..\tools.ps1 #Replace repo names to the format of org/repo diff --git a/eng/common/sdl/extract-artifact-packages.ps1 b/eng/common/sdl/extract-artifact-packages.ps1 index 3c9bf106..9e5f3cb4 100644 --- a/eng/common/sdl/extract-artifact-packages.ps1 +++ b/eng/common/sdl/extract-artifact-packages.ps1 @@ -6,10 +6,6 @@ param( $ErrorActionPreference = 'Stop' Set-StrictMode -Version 2.0 -# `tools.ps1` checks $ci to perform some actions. Since the post-build -# scripts don't necessarily execute in the same agent that run the -# build.ps1/sh script this variable isn't automatically set. -$ci = $true $disableConfigureToolsetImport = $true function ExtractArtifacts { @@ -29,6 +25,10 @@ function ExtractArtifacts { } try { + # `tools.ps1` checks $ci to perform some actions. Since the SDL + # scripts don't necessarily execute in the same agent that run the + # build.ps1/sh script this variable isn't automatically set. + $ci = $true . $PSScriptRoot\..\tools.ps1 $ExtractPackage = { diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1 index 285f1ccd..1a91bbbc 100644 --- a/eng/common/sdl/init-sdl.ps1 +++ b/eng/common/sdl/init-sdl.ps1 @@ -12,6 +12,10 @@ Set-StrictMode -Version 2.0 $disableConfigureToolsetImport = $true $LASTEXITCODE = 0 +# `tools.ps1` checks $ci to perform some actions. Since the SDL +# scripts don't necessarily execute in the same agent that run the +# build.ps1/sh script this variable isn't automatically set. +$ci = $true . $PSScriptRoot\..\tools.ps1 # Don't display the console progress UI - it's a huge perf hit @@ -58,6 +62,6 @@ try { } catch { Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'Sdl' -Message $_ + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ ExitWithExitCode 1 } diff --git a/eng/common/sdl/push-gdn.ps1 b/eng/common/sdl/push-gdn.ps1 index 79d3d355..d8fd2d82 100644 --- a/eng/common/sdl/push-gdn.ps1 +++ b/eng/common/sdl/push-gdn.ps1 @@ -12,6 +12,10 @@ $disableConfigureToolsetImport = $true $LASTEXITCODE = 0 try { + # `tools.ps1` checks $ci to perform some actions. Since the SDL + # scripts don't necessarily execute in the same agent that run the + # build.ps1/sh script this variable isn't automatically set. + $ci = $true . $PSScriptRoot\..\tools.ps1 # We create the temp directory where we'll store the sdl-config repository @@ -62,4 +66,4 @@ catch { Write-Host $_.ScriptStackTrace Write-PipelineTelemetryError -Category 'Sdl' -Message $_ ExitWithExitCode 1 -} \ No newline at end of file +} diff --git a/eng/common/sdl/run-sdl.ps1 b/eng/common/sdl/run-sdl.ps1 index 40a084f7..fe95ab35 100644 --- a/eng/common/sdl/run-sdl.ps1 +++ b/eng/common/sdl/run-sdl.ps1 @@ -16,6 +16,10 @@ $disableConfigureToolsetImport = $true $LASTEXITCODE = 0 try { + # `tools.ps1` checks $ci to perform some actions. Since the SDL + # scripts don't necessarily execute in the same agent that run the + # build.ps1/sh script this variable isn't automatically set. + $ci = $true . $PSScriptRoot\..\tools.ps1 # We store config files in the r directory of .gdn @@ -64,6 +68,6 @@ try { } catch { Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'Sdl' -Message $_ + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ ExitWithExitCode 1 -} \ No newline at end of file +} diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index ecebd0f0..536c15c4 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -24,6 +24,8 @@ parameters: enablePublishBuildAssets: false enablePublishTestResults: false enablePublishUsingPipelines: false + mergeTestResults: false + testRunTitle: $(AgentOsName)-$(BuildConfiguration)-xunit name: '' preSteps: [] runAsPublic: false @@ -194,6 +196,8 @@ jobs: testResultsFormat: 'xUnit' testResultsFiles: '*.xml' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + testRunTitle: ${{ parameters.testRunTitle }} + mergeTestResults: ${{ parameters.mergeTestResults }} continueOnError: true condition: always() diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index b722975f..055304ad 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -37,6 +37,12 @@ jobs: - name: _BuildConfig value: ${{ parameters.configuration }} - group: Publish-Build-Assets + # Skip component governance and codesign validation for SDL. These jobs + # create no content. + - name: skipComponentGovernanceDetection + value: true + - name: runCodesignValidationInjection + value: false steps: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml index 9505cf17..867f37cd 100644 --- a/eng/common/templates/post-build/common-variables.yml +++ b/eng/common/templates/post-build/common-variables.yml @@ -9,8 +9,8 @@ variables: - name: PublicDevRelease_31_Channel_Id value: 128 - # .NET Core 5 Dev - - name: NetCore_5_Dev_Channel_Id + # .NET 5 Dev + - name: Net_5_Dev_Channel_Id value: 131 # .NET Eng - Validation @@ -90,3 +90,10 @@ variables: value: https://dotnetclimsrc.blob.core.windows.net/dotnet/index.json - name: InternalInstallersBlobFeedKey value: $(dotnetclimsrc-access-key) + + # Skip component governance and codesign validation for SDL. These jobs + # create no content. + - name: skipComponentGovernanceDetection + value: true + - name: runCodesignValidationInjection + value: false diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 7be5b0bf..fbab4cb5 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -27,7 +27,7 @@ parameters: # needs to be updated with the new channel ID NetEngLatestChannelId: 2 NetEngValidationChannelId: 9 - NetCoreDev5ChannelId: 131 + NetDev5ChannelId: 131 GeneralTestingChannelId: 529 NETCoreToolingDevChannelId: 548 NETCoreToolingReleaseChannelId: 549 @@ -35,8 +35,10 @@ parameters: NETCoreExperimentalChannelId: 562 NetEngServicesIntChannelId: 678 NetEngServicesProdChannelId: 679 - Net5Preview1ChannelId: 737 Net5Preview2ChannelId: 738 + Net5Preview3ChannelId: 739 + NetCoreSDK313xxChannelId: 759 + NetCoreSDK313xxInternalChannelId: 760 stages: - stage: Validate @@ -63,7 +65,7 @@ stages: inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 arguments: -PromoteToChannels "$(InitialChannels)[$(PromoteToMaestroChannelId)]" - -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetCoreDev5ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview1ChannelId}},${{parameters.Net5Preview2ChannelId}} + -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview2ChannelId}},${{parameters.Net5Preview3ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}} - job: displayName: NuGet Validation @@ -198,9 +200,9 @@ stages: publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} stageName: 'NetCore_Dev5_Publish' - channelName: '.NET Core 5 Dev' + channelName: '.NET 5 Dev' akaMSChannelName: 'net5/dev' - channelId: ${{ parameters.NetCoreDev5ChannelId }} + channelId: ${{ parameters.NetDev5ChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' @@ -211,10 +213,10 @@ stages: dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net5_Preview1_Publish' - channelName: '.NET 5 Preview 1' - akaMSChannelName: 'net5/preview1' - channelId: ${{ parameters.Net5Preview1ChannelId }} + stageName: 'Net5_Preview2_Publish' + channelName: '.NET 5 Preview 2' + akaMSChannelName: 'net5/preview2' + channelId: ${{ parameters.Net5Preview2ChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' @@ -225,10 +227,10 @@ stages: dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net5_Preview2_Publish' - channelName: '.NET 5 Preview 2' - akaMSChannelName: 'net5/preview2' - channelId: ${{ parameters.Net5Preview2ChannelId }} + stageName: 'Net5_Preview3_Publish' + channelName: '.NET 5 Preview 3' + akaMSChannelName: 'net5/preview3' + channelId: ${{ parameters.Net5Preview3ChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' @@ -352,3 +354,29 @@ stages: transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' + +- template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NETCore_SDK_313xx_Publishing' + channelName: '.NET Core SDK 3.1.3xx' + channelId: ${{ parameters.NetCoreSDK313xxChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-symbols/nuget/v3/index.json' + +- template: \eng\common\templates\post-build\channels\generic-internal-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NETCore_SDK_313xx_Internal_Publishing' + channelName: '.NET Core SDK 3.1.3xx Internal' + channelId: ${{ parameters.NetCoreSDK313xxInternalChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-symbols/nuget/v3/index.json' diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 664ac105..caae1dbd 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -77,7 +77,7 @@ function ResolvePath { function ReadGlobalVersion { local key=$1 - local line=`grep -m 1 "$key" "$global_json_file"` + local line=$(awk "/$key/ {print; exit}" "$global_json_file") local pattern="\"$key\" *: *\"(.*)\"" if [[ ! $line =~ $pattern ]]; then @@ -438,7 +438,7 @@ temp_dir="$artifacts_dir/tmp/$configuration" global_json_file="$repo_root/global.json" # determine if global.json contains a "runtimes" entry global_json_has_runtimes=false -dotnetlocal_key=`grep -m 1 "runtimes" "$global_json_file"` || true +dotnetlocal_key=$(awk "/runtimes/ {print; exit}" "$global_json_file") || true if [[ -n "$dotnetlocal_key" ]]; then global_json_has_runtimes=true fi diff --git a/global.json b/global.json index 86601843..57a28d4c 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.1.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20162.3" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20201.2" } } From ee7c8d59b7afd3abc2c05a95dcb35bfd8953b6bd Mon Sep 17 00:00:00 2001 From: "Martin Chromecek (Moravia IT)" Date: Thu, 16 Apr 2020 20:20:22 +0200 Subject: [PATCH 073/161] LOC CHECKIN | dotnet/sourcelink master | 20200416 (#604) --- src/SourceLink.AzureDevOpsServer.Git/xlf/Resources.zh-Hans.xlf | 2 +- src/SourceLink.AzureDevOpsServer.Git/xlf/Resources.zh-Hant.xlf | 2 +- src/SourceLink.AzureRepos.Git/xlf/Resources.zh-Hans.xlf | 2 +- src/SourceLink.AzureRepos.Git/xlf/Resources.zh-Hant.xlf | 2 +- src/SourceLink.GitWeb/xlf/Resources.cs.xlf | 2 +- src/SourceLink.GitWeb/xlf/Resources.de.xlf | 2 +- src/SourceLink.GitWeb/xlf/Resources.es.xlf | 2 +- src/SourceLink.GitWeb/xlf/Resources.fr.xlf | 2 +- src/SourceLink.GitWeb/xlf/Resources.it.xlf | 2 +- src/SourceLink.GitWeb/xlf/Resources.ja.xlf | 2 +- src/SourceLink.GitWeb/xlf/Resources.ko.xlf | 2 +- src/SourceLink.GitWeb/xlf/Resources.pl.xlf | 2 +- src/SourceLink.GitWeb/xlf/Resources.pt-BR.xlf | 2 +- src/SourceLink.GitWeb/xlf/Resources.ru.xlf | 2 +- src/SourceLink.GitWeb/xlf/Resources.tr.xlf | 2 +- src/SourceLink.GitWeb/xlf/Resources.zh-Hans.xlf | 2 +- src/SourceLink.GitWeb/xlf/Resources.zh-Hant.xlf | 2 +- 17 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/SourceLink.AzureDevOpsServer.Git/xlf/Resources.zh-Hans.xlf b/src/SourceLink.AzureDevOpsServer.Git/xlf/Resources.zh-Hans.xlf index bc963ae6..05d2ad3b 100644 --- a/src/SourceLink.AzureDevOpsServer.Git/xlf/Resources.zh-Hans.xlf +++ b/src/SourceLink.AzureDevOpsServer.Git/xlf/Resources.zh-Hans.xlf @@ -1,6 +1,6 @@  - + The value of environment variable {0} is not a well-formed list of URL pairs: '{1}'" diff --git a/src/SourceLink.AzureDevOpsServer.Git/xlf/Resources.zh-Hant.xlf b/src/SourceLink.AzureDevOpsServer.Git/xlf/Resources.zh-Hant.xlf index 177fe576..b5bb875c 100644 --- a/src/SourceLink.AzureDevOpsServer.Git/xlf/Resources.zh-Hant.xlf +++ b/src/SourceLink.AzureDevOpsServer.Git/xlf/Resources.zh-Hant.xlf @@ -1,6 +1,6 @@  - + The value of environment variable {0} is not a well-formed list of URL pairs: '{1}'" diff --git a/src/SourceLink.AzureRepos.Git/xlf/Resources.zh-Hans.xlf b/src/SourceLink.AzureRepos.Git/xlf/Resources.zh-Hans.xlf index bc963ae6..05d2ad3b 100644 --- a/src/SourceLink.AzureRepos.Git/xlf/Resources.zh-Hans.xlf +++ b/src/SourceLink.AzureRepos.Git/xlf/Resources.zh-Hans.xlf @@ -1,6 +1,6 @@  - + The value of environment variable {0} is not a well-formed list of URL pairs: '{1}'" diff --git a/src/SourceLink.AzureRepos.Git/xlf/Resources.zh-Hant.xlf b/src/SourceLink.AzureRepos.Git/xlf/Resources.zh-Hant.xlf index 177fe576..b5bb875c 100644 --- a/src/SourceLink.AzureRepos.Git/xlf/Resources.zh-Hant.xlf +++ b/src/SourceLink.AzureRepos.Git/xlf/Resources.zh-Hant.xlf @@ -1,6 +1,6 @@  - + The value of environment variable {0} is not a well-formed list of URL pairs: '{1}'" diff --git a/src/SourceLink.GitWeb/xlf/Resources.cs.xlf b/src/SourceLink.GitWeb/xlf/Resources.cs.xlf index a83a11e5..7e49b4de 100644 --- a/src/SourceLink.GitWeb/xlf/Resources.cs.xlf +++ b/src/SourceLink.GitWeb/xlf/Resources.cs.xlf @@ -4,7 +4,7 @@ The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. - The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + Poskytovatel Microsoft.SourceLink.GitWeb nepodporuje překlad protokolu {0} na adresy URL obsahu. diff --git a/src/SourceLink.GitWeb/xlf/Resources.de.xlf b/src/SourceLink.GitWeb/xlf/Resources.de.xlf index f041b7e6..569975f8 100644 --- a/src/SourceLink.GitWeb/xlf/Resources.de.xlf +++ b/src/SourceLink.GitWeb/xlf/Resources.de.xlf @@ -4,7 +4,7 @@ The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. - The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + Der Microsoft.SourceLink.GitWeb-Anbieter unterstützt keine Übersetzung des Protokolls "{0}" in Inhalts-URLs. diff --git a/src/SourceLink.GitWeb/xlf/Resources.es.xlf b/src/SourceLink.GitWeb/xlf/Resources.es.xlf index 8e1afbd3..6a826dde 100644 --- a/src/SourceLink.GitWeb/xlf/Resources.es.xlf +++ b/src/SourceLink.GitWeb/xlf/Resources.es.xlf @@ -4,7 +4,7 @@ The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. - The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + El proveedor Microsoft.SourceLink.GitWeb no admite la traducción del protocolo "{0}" a direcciones URL de contenido. diff --git a/src/SourceLink.GitWeb/xlf/Resources.fr.xlf b/src/SourceLink.GitWeb/xlf/Resources.fr.xlf index afec8a14..dc53218a 100644 --- a/src/SourceLink.GitWeb/xlf/Resources.fr.xlf +++ b/src/SourceLink.GitWeb/xlf/Resources.fr.xlf @@ -4,7 +4,7 @@ The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. - The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + Le fournisseur Microsoft.SourceLink.GitWeb ne prend pas en charge la traduction du protocole '{0}' en URL de contenu. diff --git a/src/SourceLink.GitWeb/xlf/Resources.it.xlf b/src/SourceLink.GitWeb/xlf/Resources.it.xlf index 08d11f5e..ecbd6472 100644 --- a/src/SourceLink.GitWeb/xlf/Resources.it.xlf +++ b/src/SourceLink.GitWeb/xlf/Resources.it.xlf @@ -4,7 +4,7 @@ The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. - The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + Il provider Microsoft.SourceLink.GitWeb non supporta la conversione del protocollo '{0}' in URL di contenuto. diff --git a/src/SourceLink.GitWeb/xlf/Resources.ja.xlf b/src/SourceLink.GitWeb/xlf/Resources.ja.xlf index b70c68a7..d093e0e5 100644 --- a/src/SourceLink.GitWeb/xlf/Resources.ja.xlf +++ b/src/SourceLink.GitWeb/xlf/Resources.ja.xlf @@ -4,7 +4,7 @@ The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. - The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + Microsoft.SourceLink.GitWeb プロバイダーでは、'{0}' プロトコルからコンテンツ URL への変換はサポートされていません。 diff --git a/src/SourceLink.GitWeb/xlf/Resources.ko.xlf b/src/SourceLink.GitWeb/xlf/Resources.ko.xlf index d17cb1ad..df485c9f 100644 --- a/src/SourceLink.GitWeb/xlf/Resources.ko.xlf +++ b/src/SourceLink.GitWeb/xlf/Resources.ko.xlf @@ -4,7 +4,7 @@ The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. - The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + Microsoft.SourceLink.GitWeb 공급자는 '{0}' 프로토콜을 콘텐츠 URL로 변환하는 기능을 지원하지 않습니다. diff --git a/src/SourceLink.GitWeb/xlf/Resources.pl.xlf b/src/SourceLink.GitWeb/xlf/Resources.pl.xlf index 4531f455..6d2f961f 100644 --- a/src/SourceLink.GitWeb/xlf/Resources.pl.xlf +++ b/src/SourceLink.GitWeb/xlf/Resources.pl.xlf @@ -4,7 +4,7 @@ The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. - The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + Dostawca Microsoft.SourceLink.GitWeb nie obsługuje tłumaczenia protokołu „{0}” na adresy URL zawartości. diff --git a/src/SourceLink.GitWeb/xlf/Resources.pt-BR.xlf b/src/SourceLink.GitWeb/xlf/Resources.pt-BR.xlf index 864e07dd..1440f254 100644 --- a/src/SourceLink.GitWeb/xlf/Resources.pt-BR.xlf +++ b/src/SourceLink.GitWeb/xlf/Resources.pt-BR.xlf @@ -4,7 +4,7 @@ The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. - The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + O provedor Microsoft.SourceLink.GitWeb não dá suporte à conversão do protocolo '{0}' em URLs de conteúdo. diff --git a/src/SourceLink.GitWeb/xlf/Resources.ru.xlf b/src/SourceLink.GitWeb/xlf/Resources.ru.xlf index 240cb3c9..9f0862af 100644 --- a/src/SourceLink.GitWeb/xlf/Resources.ru.xlf +++ b/src/SourceLink.GitWeb/xlf/Resources.ru.xlf @@ -4,7 +4,7 @@ The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. - The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + Поставщик Microsoft.SourceLink.GitWeb не поддерживает преобразование протокола "{0}" в URL-адреса содержимого. diff --git a/src/SourceLink.GitWeb/xlf/Resources.tr.xlf b/src/SourceLink.GitWeb/xlf/Resources.tr.xlf index 8581c836..b46ea34d 100644 --- a/src/SourceLink.GitWeb/xlf/Resources.tr.xlf +++ b/src/SourceLink.GitWeb/xlf/Resources.tr.xlf @@ -4,7 +4,7 @@ The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. - The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + Microsoft.SourceLink.GitWeb sağlayıcısı '{0}' protokolünü içerik URL'lerine çevirmeyi desteklemiyor. diff --git a/src/SourceLink.GitWeb/xlf/Resources.zh-Hans.xlf b/src/SourceLink.GitWeb/xlf/Resources.zh-Hans.xlf index 44b283db..2d6553a8 100644 --- a/src/SourceLink.GitWeb/xlf/Resources.zh-Hans.xlf +++ b/src/SourceLink.GitWeb/xlf/Resources.zh-Hans.xlf @@ -4,7 +4,7 @@ The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. - The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + Microsoft.SourceLink.GitWeb 提供程序不支持将“{0}”协议翻译为内容 URL。 diff --git a/src/SourceLink.GitWeb/xlf/Resources.zh-Hant.xlf b/src/SourceLink.GitWeb/xlf/Resources.zh-Hant.xlf index 878bd818..efa8ded5 100644 --- a/src/SourceLink.GitWeb/xlf/Resources.zh-Hant.xlf +++ b/src/SourceLink.GitWeb/xlf/Resources.zh-Hant.xlf @@ -4,7 +4,7 @@ The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. - The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. + Microsoft.SourceLink.GitWeb 提供者不支援將 '{0}' 通訊協定轉譯成內容 URL。 From 5f466429a92451467bf50519a86b83ab1b80a3ae Mon Sep 17 00:00:00 2001 From: Immo Landwerth Date: Thu, 16 Apr 2020 18:12:18 -0700 Subject: [PATCH 074/161] Fix links to Code of Conduct (#602) --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 0a0250cb..2fccf1f4 100644 --- a/README.md +++ b/README.md @@ -186,8 +186,6 @@ Pre-release builds are available from Azure DevOps public feed: `https://pkgs.de [![Build Status](https://dnceng.visualstudio.com/public/_apis/build/status/SourceLink%20PR?branchName=master)](https://dnceng.visualstudio.com/public/_build/latest?definitionId=297?branchName=master) -This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. - ## Experience in Visual Studio The following screenshot demonstrates debugging a NuGet package referenced by an application, with source automatically downloaded from GitHub and used by Visual Studio 2017. From 0e7eed6d62f53d27e7aa36e19458f60402e0516c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Matou=C5=A1ek?= Date: Sat, 25 Apr 2020 09:10:02 -0700 Subject: [PATCH 075/161] List GitWeb in README (#600) --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 2fccf1f4..199245ff 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,16 @@ If your project is hosted by Bitbucket Server or Bitbucket Data Center older tha The item group `SourceLinkBitbucketGitHost` specifies the domain of the Bitbucket host and the version of Bitbucket. The version is important since URL format for accessing files changes with version 4.7. By default Source Link assumes new format (version 4.7+). +### gitweb (pre-release) + +For projects hosted on-prem via [gitweb](https://git-scm.com/docs/gitweb) reference [Microsoft.SourceLink.GitWeb](https://www.nuget.org/packages/Microsoft.SourceLink.GitWeb) package: + +```xml + + + +``` + ### Multiple providers, repositories with submodules If your repository contains submodules hosted by other git providers reference packages of all these providers. For example, projects in a repository hosted by Azure Repos that links a GitHub repository via a submodule should reference both [Microsoft.SourceLink.AzureRepos.Git](https://www.nuget.org/packages/Microsoft.SourceLink.AzureRepos.Git) and [Microsoft.SourceLink.GitHub](https://www.nuget.org/packages/Microsoft.SourceLink.GitHub) packages. [Additional configuration](https://github.com/dotnet/sourcelink/blob/master/docs/README.md#configuring-projects-with-multiple-sourcelink-providers) might be needed if multiple Source Link packages are used in the project. From a0ac3070561fd2c2e3214865c57226981b50aae5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Matou=C5=A1ek?= Date: Thu, 7 May 2020 11:13:46 -0700 Subject: [PATCH 076/161] Update NuGet.config --- NuGet.config | 1 + 1 file changed, 1 insertion(+) diff --git a/NuGet.config b/NuGet.config index db2ee738..502f9feb 100644 --- a/NuGet.config +++ b/NuGet.config @@ -4,6 +4,7 @@ + From 440f90b5422fc840b3e120f0162fa1adfc89e956 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 7 May 2020 18:37:10 +0000 Subject: [PATCH 077/161] [master] Update dependencies from dotnet/arcade (#607) * Update dependencies from https://github.com/dotnet/arcade build 20200421.14 - Microsoft.DotNet.Arcade.Sdk: 5.0.0-beta.20201.2 -> 5.0.0-beta.20221.14 * Update dependencies from https://github.com/dotnet/arcade build 20200428.4 - Microsoft.DotNet.Arcade.Sdk: 5.0.0-beta.20201.2 -> 5.0.0-beta.20228.4 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 4 +- eng/common/SetupNugetSources.ps1 | 14 +- eng/common/cross/build-rootfs.sh | 30 +++-- eng/common/native/find-native-compiler.sh | 121 ++++++++++++++++++ .../post-build/check-channel-consistency.ps1 | 7 +- eng/common/post-build/symbols-validation.ps1 | 18 ++- eng/common/sdk-task.ps1 | 12 ++ eng/common/sdl/extract-artifact-packages.ps1 | 4 +- eng/common/sdl/init-sdl.ps1 | 2 +- eng/common/sdl/packages.config | 2 +- eng/common/templates/job/execute-sdl.yml | 34 ++--- eng/common/templates/job/job.yml | 10 ++ .../channels/generic-internal-channel.yml | 4 +- .../channels/generic-public-channel.yml | 4 +- .../templates/post-build/common-variables.yml | 2 +- .../templates/post-build/post-build.yml | 76 +++++++---- .../post-build/setup-maestro-vars.yml | 26 ++-- eng/common/templates/steps/send-to-helix.yml | 6 +- eng/common/tools.ps1 | 9 +- eng/common/tools.sh | 23 +++- global.json | 2 +- 21 files changed, 311 insertions(+), 99 deletions(-) create mode 100644 eng/common/native/find-native-compiler.sh diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 3ad788b8..1dea3877 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - bce0a98620c1c5a110b2bba9912f3d5929069c6b + 590a102630c7efc7ca6f652f7c6c47dee4c4086c diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index a5a1e711..c3c473eb 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -83,7 +83,7 @@ function AddCredential($creds, $source, $username, $password) { $passwordElement.SetAttribute("value", $Password) } -function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Password) { +function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $Password) { $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]") Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds." @@ -123,19 +123,21 @@ if ($creds -eq $null) { $doc.DocumentElement.AppendChild($creds) | Out-Null } +$userName = "dn-bot" + # Insert credential nodes for Maestro's private feeds -InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Password $Password +InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -Password $Password $dotnet3Source = $sources.SelectSingleNode("add[@key='dotnet3']") if ($dotnet3Source -ne $null) { - AddPackageSource -Sources $sources -SourceName "dotnet3-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password - AddPackageSource -Sources $sources -SourceName "dotnet3-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal-transport/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet3-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet3-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password } $dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']") if ($dotnet31Source -ne $null) { - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password } $doc.Save($filename) diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 2cdd82d3..d780fefb 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +set -e + usage() { echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [--skipunmount] --rootfsdir ]" @@ -15,6 +17,8 @@ __CodeName=xenial __CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) __InitialDir=$PWD __BuildArch=arm +__AlpineArch=armv7 +__QEMUArch=arm __UbuntuArch=armhf __UbuntuRepo="http://ports.ubuntu.com/" __LLDB_Package="liblldb-3.9-dev" @@ -26,9 +30,10 @@ __UbuntuPackages="build-essential" __AlpinePackages="alpine-base" __AlpinePackages+=" build-base" __AlpinePackages+=" linux-headers" -__AlpinePackagesEdgeTesting=" lldb-dev" -__AlpinePackagesEdgeMain=" llvm9-libs" +__AlpinePackagesEdgeCommunity=" lldb-dev" +__AlpinePackagesEdgeMain=" llvm10-libs" __AlpinePackagesEdgeMain+=" python3" +__AlpinePackagesEdgeMain+=" libedit" # symlinks fixer __UbuntuPackages+=" symlinks" @@ -55,12 +60,11 @@ __AlpinePackages+=" openssl-dev" __AlpinePackages+=" zlib-dev" __FreeBSDBase="12.1-RELEASE" -__FreeBSDPkg="1.10.5" +__FreeBSDPkg="1.12.0" __FreeBSDPackages="libunwind" __FreeBSDPackages+=" icu" __FreeBSDPackages+=" libinotify" __FreeBSDPackages+=" lttng-ust" -__FreeBSDPackages+=" llvm-90" __FreeBSDPackages+=" krb5" __UnprocessedBuildArgs= @@ -78,7 +82,7 @@ while :; do arm) __BuildArch=arm __UbuntuArch=armhf - __AlpineArch=armhf + __AlpineArch=armv7 __QEMUArch=arm ;; arm64) @@ -205,7 +209,7 @@ fi if [ -d "$__RootfsDir" ]; then if [ $__SkipUnmount == 0 ]; then - umount $__RootfsDir/* + umount $__RootfsDir/* || true fi rm -rf $__RootfsDir fi @@ -231,9 +235,9 @@ if [[ "$__CodeName" == "alpine" ]]; then add $__AlpinePackagesEdgeMain $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \ - -X http://dl-cdn.alpinelinux.org/alpine/edge/testing \ + -X http://dl-cdn.alpinelinux.org/alpine/edge/community \ -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ - add $__AlpinePackagesEdgeTesting + add $__AlpinePackagesEdgeCommunity rm -r $__ApkToolsDir elif [[ "$__CodeName" == "freebsd" ]]; then @@ -246,11 +250,13 @@ elif [[ "$__CodeName" == "freebsd" ]]; then # get and build package manager wget -O - https://github.com/freebsd/pkg/archive/${__FreeBSDPkg}.tar.gz | tar -C $__RootfsDir/tmp -zxf - cd $__RootfsDir/tmp/pkg-${__FreeBSDPkg} - ./autogen.sh && ./configure --prefix=$__RootfsDir/host && make install + # needed for install to succeed + mkdir -p $__RootfsDir/host/etc + ./autogen.sh && ./configure --prefix=$__RootfsDir/host && make && make install rm -rf $__RootfsDir/tmp/pkg-${__FreeBSDPkg} # install packages we need. - $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf update - $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf install --yes $__FreeBSDPackages + INSTALL_AS_USER=$(whoami) $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf update + INSTALL_AS_USER=$(whoami) $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf install --yes $__FreeBSDPackages elif [[ -n $__CodeName ]]; then qemu-debootstrap --arch $__UbuntuArch $__CodeName $__RootfsDir $__UbuntuRepo cp $__CrossDir/$__BuildArch/sources.list.$__CodeName $__RootfsDir/etc/apt/sources.list @@ -260,7 +266,7 @@ elif [[ -n $__CodeName ]]; then chroot $__RootfsDir symlinks -cr /usr if [ $__SkipUnmount == 0 ]; then - umount $__RootfsDir/* + umount $__RootfsDir/* || true fi if [[ "$__BuildArch" == "arm" && "$__CodeName" == "trusty" ]]; then diff --git a/eng/common/native/find-native-compiler.sh b/eng/common/native/find-native-compiler.sh new file mode 100644 index 00000000..aed19d07 --- /dev/null +++ b/eng/common/native/find-native-compiler.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +# +# This file locates the native compiler with the given name and version and sets the environment variables to locate it. +# + +source="${BASH_SOURCE[0]}" + +# resolve $SOURCE until the file is no longer a symlink +while [[ -h $source ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +if [ $# -lt 0 ] +then + echo "Usage..." + echo "find-native-compiler.sh " + echo "Specify the name of compiler (clang or gcc)." + echo "Specify the major version of compiler." + echo "Specify the minor version of compiler." + exit 1 +fi + +. $scriptroot/../pipeline-logging-functions.sh + +compiler="$1" +cxxCompiler="$compiler++" +majorVersion="$2" +minorVersion="$3" + +if [ "$compiler" = "gcc" ]; then cxxCompiler="g++"; fi + +check_version_exists() { + desired_version=-1 + + # Set up the environment to be used for building with the desired compiler. + if command -v "$compiler-$1.$2" > /dev/null; then + desired_version="-$1.$2" + elif command -v "$compiler$1$2" > /dev/null; then + desired_version="$1$2" + elif command -v "$compiler-$1$2" > /dev/null; then + desired_version="-$1$2" + fi + + echo "$desired_version" +} + +if [ -z "$CLR_CC" ]; then + + # Set default versions + if [ -z "$majorVersion" ]; then + # note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero. + if [ "$compiler" = "clang" ]; then versions=( 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5 ) + elif [ "$compiler" = "gcc" ]; then versions=( 9 8 7 6 5 4.9 ); fi + + for version in "${versions[@]}"; do + parts=(${version//./ }) + desired_version="$(check_version_exists "${parts[0]}" "${parts[1]}")" + if [ "$desired_version" != "-1" ]; then majorVersion="${parts[0]}"; break; fi + done + + if [ -z "$majorVersion" ]; then + if command -v "$compiler" > /dev/null; then + if [ "$(uname)" != "Darwin" ]; then + Write-PipelineTelemetryError -category "Build" -type "warning" "Specific version of $compiler not found, falling back to use the one in PATH." + fi + export CC="$(command -v "$compiler")" + export CXX="$(command -v "$cxxCompiler")" + else + Write-PipelineTelemetryError -category "Build" "No usable version of $compiler found." + exit 1 + fi + else + if [ "$compiler" = "clang" ] && [ "$majorVersion" -lt 5 ]; then + if [ "$build_arch" = "arm" ] || [ "$build_arch" = "armel" ]; then + if command -v "$compiler" > /dev/null; then + Write-PipelineTelemetryError -category "Build" -type "warning" "Found clang version $majorVersion which is not supported on arm/armel architectures, falling back to use clang from PATH." + export CC="$(command -v "$compiler")" + export CXX="$(command -v "$cxxCompiler")" + else + Write-PipelineTelemetryError -category "Build" "Found clang version $majorVersion which is not supported on arm/armel architectures, and there is no clang in PATH." + exit 1 + fi + fi + fi + fi + else + desired_version="$(check_version_exists "$majorVersion" "$minorVersion")" + if [ "$desired_version" = "-1" ]; then + Write-PipelineTelemetryError -category "Build" "Could not find specific version of $compiler: $majorVersion $minorVersion." + exit 1 + fi + fi + + if [ -z "$CC" ]; then + export CC="$(command -v "$compiler$desired_version")" + export CXX="$(command -v "$cxxCompiler$desired_version")" + if [ -z "$CXX" ]; then export CXX="$(command -v "$cxxCompiler")"; fi + fi +else + if [ ! -f "$CLR_CC" ]; then + Write-PipelineTelemetryError -category "Build" "CLR_CC is set but path '$CLR_CC' does not exist" + exit 1 + fi + export CC="$CLR_CC" + export CXX="$CLR_CXX" +fi + +if [ -z "$CC" ]; then + Write-PipelineTelemetryError -category "Build" "Unable to find $compiler." + exit 1 +fi + +export CCC_CC="$CC" +export CCC_CXX="$CXX" +export SCAN_BUILD_COMMAND="$(command -v "scan-build$desired_version")" diff --git a/eng/common/post-build/check-channel-consistency.ps1 b/eng/common/post-build/check-channel-consistency.ps1 index 7e6618d6..38abc539 100644 --- a/eng/common/post-build/check-channel-consistency.ps1 +++ b/eng/common/post-build/check-channel-consistency.ps1 @@ -6,13 +6,18 @@ param( try { . $PSScriptRoot\post-build-utils.ps1 + if ($PromoteToChannels -eq "") { + Write-PipelineTaskError -Type 'warning' -Message "This build won't publish assets as it's not configured to any Maestro channel. If that wasn't intended use Darc to configure a default channel using add-default-channel for this branch or to promote it to a channel using add-build-to-channel. See https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#assigning-an-individual-build-to-a-channel for more info." + ExitWithExitCode 0 + } + # Check that every channel that Maestro told to promote the build to # is available in YAML $PromoteToChannelsIds = $PromoteToChannels -split "\D" | Where-Object { $_ } foreach ($id in $PromoteToChannelsIds) { if (($id -ne 0) -and ($id -notin $AvailableChannelIds)) { - Write-PipelineTaskError -Type 'warning' -Message "Channel $id is not present in the post-build YAML configuration!" + Write-PipelineTaskError -Message "Channel $id is not present in the post-build YAML configuration! This is an error scenario. Please contact @dnceng." } } diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 index f7cfe986..22eeb265 100644 --- a/eng/common/post-build/symbols-validation.ps1 +++ b/eng/common/post-build/symbols-validation.ps1 @@ -1,7 +1,8 @@ param( [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation - [Parameter(Mandatory=$true)][string] $DotnetSymbolVersion # Version of dotnet symbol to use + [Parameter(Mandatory=$true)][string] $DotnetSymbolVersion, # Version of dotnet symbol to use + [Parameter(Mandatory=$false)][switch] $ContinueOnError # If we should keep checking symbols after an error ) function FirstMatchingSymbolDescriptionOrDefault { @@ -125,6 +126,8 @@ function CheckSymbolsAvailable { Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue } + $TotalFailures = 0 + Get-ChildItem "$InputPath\*.nupkg" | ForEach-Object { $FileName = $_.Name @@ -148,11 +151,22 @@ function CheckSymbolsAvailable { if ($Status -ne 0) { Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $Status modules in the package $FileName" - ExitWithExitCode $exitCode + + if ($ContinueOnError) { + $TotalFailures++ + } + else { + ExitWithExitCode 1 + } } Write-Host } + + if ($TotalFailures -ne 0) { + Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Symbols missing for $TotalFailures packages" + ExitWithExitCode 1 + } } function InstallDotnetSymbol { diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index 3872af59..79c25e7f 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -57,6 +57,18 @@ try { ExitWithExitCode 1 } + if( $msbuildEngine -eq "vs") { + # Ensure desktop MSBuild is available for sdk tasks. + if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "vs" )) { + $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.4`" }") -MemberType NoteProperty + } + if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { + $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.4.0-alpha" -MemberType NoteProperty + } + + InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true + } + $taskProject = GetSdkTaskProject $task if (!(Test-Path $taskProject)) { Write-PipelineTelemetryError -Category 'Build' -Message "Unknown task: $task" -ForegroundColor Red diff --git a/eng/common/sdl/extract-artifact-packages.ps1 b/eng/common/sdl/extract-artifact-packages.ps1 index 9e5f3cb4..7f28d9c5 100644 --- a/eng/common/sdl/extract-artifact-packages.ps1 +++ b/eng/common/sdl/extract-artifact-packages.ps1 @@ -63,7 +63,7 @@ try { } } catch { - Write-Host $_.ScriptStackTrace + Write-Host $_ Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ ExitWithExitCode 1 } @@ -74,7 +74,7 @@ try { Measure-Command { ExtractArtifacts } } catch { - Write-Host $_.ScriptStackTrace + Write-Host $_ Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ ExitWithExitCode 1 } diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1 index 1a91bbbc..a68bf0b8 100644 --- a/eng/common/sdl/init-sdl.ps1 +++ b/eng/common/sdl/init-sdl.ps1 @@ -24,7 +24,7 @@ $ProgressPreference = 'SilentlyContinue' # Construct basic auth from AzDO access token; construct URI to the repository's gdn folder stored in that repository; construct location of zip file $encodedPat = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$AzureDevOpsAccessToken")) $escapedRepository = [Uri]::EscapeDataString("/$Repository/$BranchName/.gdn") -$uri = "https://dev.azure.com/dnceng/internal/_apis/git/repositories/sdl-tool-cfg/Items?path=$escapedRepository&versionDescriptor[versionOptions]=0&`$format=zip&api-version=5.0-preview.1" +$uri = "https://dev.azure.com/dnceng/internal/_apis/git/repositories/sdl-tool-cfg/Items?path=$escapedRepository&versionDescriptor[versionOptions]=0&`$format=zip&api-version=5.0" $zipFile = "$WorkingDirectory/gdn.zip" Add-Type -AssemblyName System.IO.Compression.FileSystem diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config index 256ffbfb..968b39be 100644 --- a/eng/common/sdl/packages.config +++ b/eng/common/sdl/packages.config @@ -1,4 +1,4 @@ - + diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index 640f2b04..c64c4f56 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -6,6 +6,7 @@ parameters: # 'continueOnError', the parameter value is not correctly picked up. # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter sdlContinueOnError: false # optional: determines whether to continue the build if the step errors; + downloadArtifacts: true # optional: determines if the artifacts should be dowloaded dependsOn: '' # Optional: dependencies of the job artifactNames: '' # Optional: patterns supplied to DownloadBuildArtifacts # Usage: @@ -31,8 +32,20 @@ jobs: steps: - checkout: self clean: true - - ${{ if ne(parameters.artifactNames, '') }}: - - ${{ each artifactName in parameters.artifactNames }}: + - ${{ if ne(parameters.downloadArtifacts, 'false')}}: + - ${{ if ne(parameters.artifactNames, '') }}: + - ${{ each artifactName in parameters.artifactNames }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Build Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: ${{ artifactName }} + downloadPath: $(Build.ArtifactStagingDirectory)\artifacts + - ${{ if eq(parameters.artifactNames, '') }}: - task: DownloadBuildArtifacts@0 displayName: Download Build Artifacts inputs: @@ -41,20 +54,9 @@ jobs: project: $(AzDOProjectName) pipeline: $(AzDOPipelineId) buildId: $(AzDOBuildId) - artifactName: ${{ artifactName }} + downloadType: specific files + itemPattern: "**" downloadPath: $(Build.ArtifactStagingDirectory)\artifacts - - ${{ if eq(parameters.artifactNames, '') }}: - - task: DownloadBuildArtifacts@0 - displayName: Download Build Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - downloadType: specific files - itemPattern: "**" - downloadPath: $(Build.ArtifactStagingDirectory)\artifacts - powershell: eng/common/sdl/extract-artifact-packages.ps1 -InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts @@ -81,7 +83,7 @@ jobs: continueOnError: ${{ parameters.sdlContinueOnError }} - ${{ if eq(parameters.overrideParameters, '') }}: - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 - -GuardianPackageName Microsoft.Guardian.Cli.0.7.2 + -GuardianPackageName Microsoft.Guardian.Cli.win10-x64.0.20.1 -NugetPackageDirectory $(Build.SourcesDirectory)\.packages -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) ${{ parameters.additionalParameters }} diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 536c15c4..fc39647f 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -24,6 +24,7 @@ parameters: enablePublishBuildAssets: false enablePublishTestResults: false enablePublishUsingPipelines: false + useBuildManifest: false mergeTestResults: false testRunTitle: $(AgentOsName)-$(BuildConfiguration)-xunit name: '' @@ -218,3 +219,12 @@ jobs: ArtifactName: AssetManifests continueOnError: ${{ parameters.continueOnError }} condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) + + - ${{ if eq(parameters.useBuildManifest, true) }}: + - task: PublishBuildArtifacts@1 + displayName: Publish Build Manifest + inputs: + PathToPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/manifest.props' + PublishLocation: Container + ArtifactName: BuildManifests + continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/templates/post-build/channels/generic-internal-channel.yml b/eng/common/templates/post-build/channels/generic-internal-channel.yml index dde27800..258ba4b7 100644 --- a/eng/common/templates/post-build/channels/generic-internal-channel.yml +++ b/eng/common/templates/post-build/channels/generic-internal-channel.yml @@ -23,7 +23,7 @@ stages: - job: publish_symbols displayName: Symbol Publishing dependsOn: setupMaestroVars - condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )), eq(dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'], ${{ parameters.channelId }})) + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} )) variables: - group: DotNet-Symbol-Server-Pats - name: AzDOProjectName @@ -96,7 +96,7 @@ stages: value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - name: AzDOBuildId value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] - condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )), eq(dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'], ${{ parameters.channelId }})) + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} )) pool: vmImage: 'windows-2019' steps: diff --git a/eng/common/templates/post-build/channels/generic-public-channel.yml b/eng/common/templates/post-build/channels/generic-public-channel.yml index 08853ec4..bf98d990 100644 --- a/eng/common/templates/post-build/channels/generic-public-channel.yml +++ b/eng/common/templates/post-build/channels/generic-public-channel.yml @@ -25,7 +25,7 @@ stages: - job: publish_symbols displayName: Symbol Publishing dependsOn: setupMaestroVars - condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )), eq(dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'], ${{ parameters.channelId }})) + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} )) variables: - group: DotNet-Symbol-Server-Pats - name: AzDOProjectName @@ -99,7 +99,7 @@ stages: value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] - name: ArtifactsCategory value: ${{ coalesce(variables._DotNetArtifactsCategory, '.NETCore') }} - condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )), eq(dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'], ${{ parameters.channelId }})) + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} )) pool: vmImage: 'windows-2019' steps: diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml index 867f37cd..c99fd750 100644 --- a/eng/common/templates/post-build/common-variables.yml +++ b/eng/common/templates/post-build/common-variables.yml @@ -63,7 +63,7 @@ variables: - name: MaestroApiAccessToken value: $(MaestroAccessToken) - name: MaestroApiVersion - value: "2019-01-16" + value: "2020-02-20" - name: SourceLinkCLIVersion value: 3.0.0 diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index fbab4cb5..47be0bed 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -9,12 +9,14 @@ parameters: continueOnError: false params: '' artifactNames: '' + downloadArtifacts: true # These parameters let the user customize the call to sdk-task.ps1 for publishing # symbols & general artifacts as well as for signing validation symbolPublishingAdditionalParameters: '' artifactsPublishingAdditionalParameters: '' signingValidationAdditionalParameters: '' + useBuildManifest: false # Which stages should finish execution before post-build stages start validateDependsOn: @@ -35,10 +37,11 @@ parameters: NETCoreExperimentalChannelId: 562 NetEngServicesIntChannelId: 678 NetEngServicesProdChannelId: 679 - Net5Preview2ChannelId: 738 Net5Preview3ChannelId: 739 - NetCoreSDK313xxChannelId: 759 - NetCoreSDK313xxInternalChannelId: 760 + Net5Preview4ChannelId: 856 + Net5Preview5ChannelId: 857 + NetCoreSDK314xxChannelId: 921 + NetCoreSDK314xxInternalChannelId: 922 stages: - stage: Validate @@ -53,10 +56,8 @@ stages: displayName: Post-build Checks dependsOn: setupMaestroVars variables: - - name: InitialChannels - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'] ] - - name: PromoteToMaestroChannelId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'] ] + - name: TargetChannels + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ] pool: vmImage: 'windows-2019' steps: @@ -64,8 +65,8 @@ stages: displayName: Maestro Channels Consistency inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 - arguments: -PromoteToChannels "$(InitialChannels)[$(PromoteToMaestroChannelId)]" - -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview2ChannelId}},${{parameters.Net5Preview3ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}} + arguments: -PromoteToChannels "$(TargetChannels)" + -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview3ChannelId}},${{parameters.Net5Preview4ChannelId}},${{parameters.Net5Preview5ChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}} - job: displayName: NuGet Validation @@ -113,6 +114,16 @@ stages: pool: vmImage: 'windows-2019' steps: + - ${{ if eq(parameters.useBuildManifest, true) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download build manifest + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BuildManifests - task: DownloadBuildArtifacts@0 displayName: Download Package Artifacts inputs: @@ -135,11 +146,13 @@ stages: filePath: eng\common\enable-cross-org-publishing.ps1 arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) + # Signing validation will optionally work with the buildmanifest file which is downloaded from + # Azure DevOps above. - task: PowerShell@2 displayName: Validate inputs: filePath: eng\common\sdk-task.ps1 - arguments: -task SigningValidation -restore -msbuildEngine dotnet + arguments: -task SigningValidation -restore -msbuildEngine vs /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' ${{ parameters.signingValidationAdditionalParameters }} @@ -192,6 +205,7 @@ stages: additionalParameters: ${{ parameters.SDLValidationParameters.params }} continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }} + downloadArtifacts: ${{ parameters.SDLValidationParameters.downloadArtifacts }} - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: @@ -213,10 +227,10 @@ stages: dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net5_Preview2_Publish' - channelName: '.NET 5 Preview 2' - akaMSChannelName: 'net5/preview2' - channelId: ${{ parameters.Net5Preview2ChannelId }} + stageName: 'Net5_Preview3_Publish' + channelName: '.NET 5 Preview 3' + akaMSChannelName: 'net5/preview3' + channelId: ${{ parameters.Net5Preview3ChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' @@ -227,10 +241,24 @@ stages: dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net5_Preview3_Publish' - channelName: '.NET 5 Preview 3' - akaMSChannelName: 'net5/preview3' - channelId: ${{ parameters.Net5Preview3ChannelId }} + stageName: 'Net5_Preview4_Publish' + channelName: '.NET 5 Preview 4' + akaMSChannelName: 'net5/preview4' + channelId: ${{ parameters.Net5Preview4ChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' + +- template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'Net5_Preview5_Publish' + channelName: '.NET 5 Preview 5' + akaMSChannelName: 'net5/preview5' + channelId: ${{ parameters.Net5Preview5ChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' @@ -361,9 +389,9 @@ stages: dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'NETCore_SDK_313xx_Publishing' - channelName: '.NET Core SDK 3.1.3xx' - channelId: ${{ parameters.NetCoreSDK313xxChannelId }} + stageName: 'NETCore_SDK_314xx_Publishing' + channelName: '.NET Core SDK 3.1.4xx' + channelId: ${{ parameters.NetCoreSDK314xxChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-symbols/nuget/v3/index.json' @@ -374,9 +402,9 @@ stages: dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'NETCore_SDK_313xx_Internal_Publishing' - channelName: '.NET Core SDK 3.1.3xx Internal' - channelId: ${{ parameters.NetCoreSDK313xxInternalChannelId }} + stageName: 'NETCore_SDK_314xx_Internal_Publishing' + channelName: '.NET Core SDK 3.1.4xx Internal' + channelId: ${{ parameters.NetCoreSDK314xxInternalChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-symbols/nuget/v3/index.json' diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml index 05e611ed..b3d29d44 100644 --- a/eng/common/templates/post-build/setup-maestro-vars.yml +++ b/eng/common/templates/post-build/setup-maestro-vars.yml @@ -5,7 +5,9 @@ jobs: - template: common-variables.yml - name: BuildId value: $[ coalesce(variables.BARBuildId, 0) ] - - name: PromoteToChannelId + - name: PromoteToMaestroChannels + value: $[ coalesce(variables.PromoteToChannelIds, 0) ] + - name: PromoteToMaestroChannel value: $[ coalesce(variables.PromoteToMaestroChannelId, 0) ] pool: vmImage: 'windows-2019' @@ -14,7 +16,7 @@ jobs: - task: DownloadBuildArtifacts@0 displayName: Download Release Configs - condition: eq(variables.PromoteToChannelId, 0) + condition: and(eq(variables.PromoteToMaestroChannels, 0), eq(variables.PromoteToMaestroChannel, 0)) inputs: buildType: current artifactName: ReleaseConfigs @@ -26,20 +28,16 @@ jobs: targetType: inline script: | try { - if ($Env:PromoteToChannelId -eq 0) { + if ($Env:PromoteToMaestroChannels -eq 0 -and $Env:PromoteToMaestroChannel -eq 0) { $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt $BarId = $Content | Select -Index 0 - - $Channels = "" - $Content | Select -Index 1 | ForEach-Object { $Channels += "$_ ," } - + $Channels = $Content | Select -Index 1 $IsStableBuild = $Content | Select -Index 2 $AzureDevOpsProject = $Env:System_TeamProject $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId $AzureDevOpsBuildId = $Env:Build_BuildId - $PromoteToMaestroChannelId = 0 } else { $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" @@ -51,25 +49,23 @@ jobs: $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } $BarId = $Env:BARBuildId - $Channels = 'None' - - #TODO: Fix this once this issue is done: https://github.com/dotnet/arcade/issues/3834 - $IsStableBuild = 'False' + $Channels = $Env:PromoteToMaestroChannels -split "," + $Channels = $Channels -join "][" + $Channels = "[$Channels][$Env:PromoteToMaestroChannel]" + $IsStableBuild = $buildInfo.stable $AzureDevOpsProject = $buildInfo.azureDevOpsProject $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId - $PromoteToMaestroChannelId = $Env:PromoteToMaestroChannelId } Write-Host "##vso[task.setvariable variable=BARBuildId;isOutput=true]$BarId" - Write-Host "##vso[task.setvariable variable=InitialChannels;isOutput=true]$Channels" + Write-Host "##vso[task.setvariable variable=TargetChannels;isOutput=true]$Channels" Write-Host "##vso[task.setvariable variable=IsStableBuild;isOutput=true]$IsStableBuild" Write-Host "##vso[task.setvariable variable=AzDOProjectName;isOutput=true]$AzureDevOpsProject" Write-Host "##vso[task.setvariable variable=AzDOPipelineId;isOutput=true]$AzureDevOpsBuildDefinitionId" Write-Host "##vso[task.setvariable variable=AzDOBuildId;isOutput=true]$AzureDevOpsBuildId" - Write-Host "##vso[task.setvariable variable=PromoteToMaestroChannelId;isOutput=true]$PromoteToMaestroChannelId" } catch { Write-Host $_ diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml index 30becf01..5eceb487 100644 --- a/eng/common/templates/steps/send-to-helix.yml +++ b/eng/common/templates/steps/send-to-helix.yml @@ -10,7 +10,7 @@ parameters: HelixPostCommands: '' # optional -- commands to run after Helix work item execution WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects - WorkItemTimeout: '' # optional -- a timeout in seconds for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects + WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload XUnitProjects: '' # optional -- semicolon delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true XUnitWorkItemTimeout: '' # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects @@ -18,8 +18,8 @@ parameters: XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion - DotNetCliPackageType: '' # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json - DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json + DotNetCliPackageType: '' # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases-index.json + DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases-index.json EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 60c1cd89..f31377a6 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -488,10 +488,11 @@ function GetNuGetPackageCachePath() { if ($env:NUGET_PACKAGES -eq $null) { # Use local cache on CI to ensure deterministic build, # use global cache in dev builds to avoid cost of downloading packages. + # For directory normalization, see also: https://github.com/NuGet/Home/issues/7968 if ($useGlobalNuGetCache) { - $env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages' + $env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages\' } else { - $env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages' + $env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages\' } } @@ -546,7 +547,7 @@ function InitializeToolset() { MSBuild-Core $proj $bl /t:__WriteToolsetLocation /clp:ErrorsOnly`;NoSummary /p:__ToolsetLocationOutputFile=$toolsetLocationFile - $path = Get-Content $toolsetLocationFile -TotalCount 1 + $path = Get-Content $toolsetLocationFile -Encoding UTF8 -TotalCount 1 if (!(Test-Path $path)) { throw "Invalid toolset path: $path" } @@ -632,6 +633,8 @@ function MSBuild-Core() { } } + $env:ARCADE_BUILD_TOOL_COMMAND = "$($buildTool.Path) $cmdArgs" + $exitCode = Exec-Process $buildTool.Path $cmdArgs if ($exitCode -ne 0) { diff --git a/eng/common/tools.sh b/eng/common/tools.sh index caae1dbd..a9dff440 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -201,7 +201,14 @@ function InstallDotNet { local runtimeSourceFeedKey='' if [[ -n "${7:-}" ]]; then - decodedFeedKey=`echo $7 | base64 --decode` + # The 'base64' binary on alpine uses '-d' and doesn't support '--decode' + # '-d'. To work around this, do a simple detection and switch the parameter + # accordingly. + decodeArg="--decode" + if base64 --help 2>&1 | grep -q "BusyBox"; then + decodeArg="-d" + fi + decodedFeedKey=`echo $7 | base64 $decodeArg` runtimeSourceFeedKey="--feed-credential $decodedFeedKey" fi @@ -415,11 +422,17 @@ function MSBuild-Core { warnaserror_switch="/warnaserror" fi - "$_InitializeBuildTool" "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" || { - local exit_code=$? - Write-PipelineTelemetryError -category 'Build' "Build failed (exit code '$exit_code')." - ExitWithExitCode $exit_code + function RunBuildTool { + export ARCADE_BUILD_TOOL_COMMAND="$_InitializeBuildTool $@" + + "$_InitializeBuildTool" "$@" || { + local exit_code=$? + Write-PipelineTaskError "Build failed (exit code '$exit_code')." + ExitWithExitCode $exit_code + } } + + RunBuildTool "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" } ResolvePath "${BASH_SOURCE[0]}" diff --git a/global.json b/global.json index 57a28d4c..77b92dd3 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "3.1.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20201.2" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20228.4" } } From 1c898bfa8c7f67405675488c34744dc2c0b804a2 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 11 May 2020 12:11:15 +0000 Subject: [PATCH 078/161] Update dependencies from https://github.com/dotnet/arcade build 20200506.5 (#611) - Microsoft.DotNet.Arcade.Sdk: 5.0.0-beta.20228.4 -> 5.0.0-beta.20256.5 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 4 ++-- eng/common/internal/Tools.csproj | 1 + eng/common/performance/performance-setup.ps1 | 5 +++-- eng/common/performance/performance-setup.sh | 2 +- eng/common/sdk-task.ps1 | 9 +++++---- global.json | 4 ++-- 6 files changed, 14 insertions(+), 11 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 1dea3877..f80ae772 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 590a102630c7efc7ca6f652f7c6c47dee4c4086c + 8547938aefa24475a04877285553f0b2663ae249 diff --git a/eng/common/internal/Tools.csproj b/eng/common/internal/Tools.csproj index 1a39a7ef..f46d5efe 100644 --- a/eng/common/internal/Tools.csproj +++ b/eng/common/internal/Tools.csproj @@ -4,6 +4,7 @@ net472 false + false diff --git a/eng/common/performance/performance-setup.ps1 b/eng/common/performance/performance-setup.ps1 index 1763a1a9..f5e6ca68 100644 --- a/eng/common/performance/performance-setup.ps1 +++ b/eng/common/performance/performance-setup.ps1 @@ -3,7 +3,7 @@ Param( [string] $CoreRootDirectory, [string] $BaselineCoreRootDirectory, [string] $Architecture="x64", - [string] $Framework="netcoreapp5.0", + [string] $Framework="net5.0", [string] $CompilationMode="Tiered", [string] $Repository=$env:BUILD_REPOSITORY_NAME, [string] $Branch=$env:BUILD_SOURCEBRANCH, @@ -31,7 +31,8 @@ $HelixSourcePrefix = "pr" $Queue = "Windows.10.Amd64.ClientRS4.DevEx.15.8.Open" -if ($Framework.StartsWith("netcoreapp")) { +# TODO: Implement a better logic to determine if Framework is .NET Core or >= .NET 5. +if ($Framework.StartsWith("netcoreapp") -or ($Framework -eq "net5.0")) { $Queue = "Windows.10.Amd64.ClientRS5.Open" } diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index b9eecf94..34eed8b6 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -4,7 +4,7 @@ source_directory=$BUILD_SOURCESDIRECTORY core_root_directory= baseline_core_root_directory= architecture=x64 -framework=netcoreapp5.0 +framework=net5.0 compilation_mode=tiered repository=$BUILD_REPOSITORY_NAME branch=$BUILD_SOURCEBRANCH diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index 79c25e7f..f997be43 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -59,14 +59,15 @@ try { if( $msbuildEngine -eq "vs") { # Ensure desktop MSBuild is available for sdk tasks. - if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "vs" )) { - $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.4`" }") -MemberType NoteProperty + if( -not ($GlobalJson.tools.PSObject.Properties.Name -contains "vs" )) { + $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty } if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { - $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.4.0-alpha" -MemberType NoteProperty + $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.5.0-alpha" -MemberType NoteProperty } - InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true + $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true + $global:_MSBuildExe = "$($xcopyMSBuildToolsFolder)\MSBuild\Current\Bin\MSBuild.exe" } $taskProject = GetSdkTaskProject $task diff --git a/global.json b/global.json index 77b92dd3..d6fc19c0 100644 --- a/global.json +++ b/global.json @@ -1,8 +1,8 @@ { "tools": { - "dotnet": "3.1.101" + "dotnet": "5.0.100-preview.5.20251.2" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20228.4" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20256.5" } } From 33ac6e296f886bd643bdac1e949ddc4f9d4901e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Matou=C5=A1ek?= Date: Mon, 11 May 2020 12:07:11 -0700 Subject: [PATCH 079/161] Delete old yaml (#610) --- azure-pipelines-microbuild.yml | 62 ---------------------------------- 1 file changed, 62 deletions(-) delete mode 100644 azure-pipelines-microbuild.yml diff --git a/azure-pipelines-microbuild.yml b/azure-pipelines-microbuild.yml deleted file mode 100644 index e4b8fd4a..00000000 --- a/azure-pipelines-microbuild.yml +++ /dev/null @@ -1,62 +0,0 @@ -resources: -- repo: self - clean: true -queue: - name: VSEng-MicroBuildVS2017 - demands: Cmd -variables: - BuildConfiguration: Release - TeamName: Roslyn - -steps: -- task: ms-vseng.MicroBuildTasks.30666190-6959-11e5-9f96-f56098202fef.MicroBuildSigningPlugin@1 - displayName: Install Signing Plugin - inputs: - signType: $(SignType) - esrpSigning: true - condition: and(succeeded(), ne(variables['SignType'], '')) - -- script: eng\CIBuild.cmd - -configuration $(BuildConfiguration) - /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - /p:DotNetSignType=$(SignType) - /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) - /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) - displayName: Build - -- task: NuGetPublisher@0 - displayName: Publish NuGet Packages to MyGet - inputs: - searchPattern: 'artifacts\packages\$(BuildConfiguration)\Shipping\*.nupkg' - connectedServiceName: 'SourceLink NuGet feed' - nuGetVersion: 4.0.0.2283 - condition: succeeded() - -- task: PublishBuildArtifacts@1 - displayName: Publish Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)\artifacts\log\$(_configuration)' - ArtifactName: 'Logs' - continueOnError: true - condition: not(succeeded()) - -- task: PublishTestResults@1 - displayName: Publish Test Results - inputs: - testRunner: XUnit - testResultsFiles: 'artifacts/TestResults/$(BuildConfiguration)/*.xml' - mergeTestResults: true - testRunTitle: 'Unit Tests' - condition: always() - -# Archive NuGet packages to DevOps. -- task: PublishBuildArtifacts@1 - displayName: Publish Artifact Packages - inputs: - PathtoPublish: 'artifacts\packages\$(BuildConfiguration)' - ArtifactName: 'Packages' - condition: succeeded() - -- task: ms-vseng.MicroBuildTasks.521a94ea-9e68-468a-8167-6dcf361ea776.MicroBuildCleanup@1 - displayName: Cleanup - condition: always() From 87fd214f1f9f82a338399459ead611baf48552f2 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 18 May 2020 12:36:23 +0000 Subject: [PATCH 080/161] Update dependencies from https://github.com/dotnet/arcade build 20200511.9 (#614) Microsoft.DotNet.Arcade.Sdk From Version 5.0.0-beta.20256.5 -> To Version 5.0.0-beta.20261.9 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 4 +-- eng/common/build.ps1 | 6 +++- eng/common/build.sh | 9 ++++- eng/common/native/CommonLibrary.psm1 | 24 +++++++++---- eng/common/performance/perfhelixpublish.proj | 8 +++++ eng/common/performance/performance-setup.ps1 | 27 +++++++++++++++ eng/common/performance/performance-setup.sh | 34 +++++++++++++++++++ eng/common/post-build/symbols-validation.ps1 | 18 ++++++++-- .../templates/post-build/post-build.yml | 30 +++++++++++++++- eng/common/tools.ps1 | 16 ++++----- eng/common/tools.sh | 16 ++++++--- global.json | 2 +- 12 files changed, 166 insertions(+), 28 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index f80ae772..1f5c04cc 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 8547938aefa24475a04877285553f0b2663ae249 + 898e51ed5fdcc4871087ac5754ca9056e58e575d diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 813d440d..67ee6d28 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -20,6 +20,7 @@ Param( [switch] $publish, [switch] $clean, [switch][Alias('bl')]$binaryLog, + [switch][Alias('nobl')]$excludeCIBinarylog, [switch] $ci, [switch] $prepareMachine, [switch] $help, @@ -58,6 +59,7 @@ function Print-Usage() { Write-Host "Advanced settings:" Write-Host " -projects Semi-colon delimited list of sln/proj's to build. Globbing is supported (*.sln)" Write-Host " -ci Set when running on CI server" + Write-Host " -excludeCIBinarylog Don't output binary log (short: -nobl)" Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build" Write-Host " -warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." @@ -134,7 +136,9 @@ try { } if ($ci) { - $binaryLog = $true + if (-not $excludeCIBinarylog) { + $binaryLog = $true + } $nodeReuse = $false } diff --git a/eng/common/build.sh b/eng/common/build.sh index 36f9aa04..6d7c5a1f 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -32,6 +32,7 @@ usage() echo "Advanced settings:" echo " --projects Project or solution file(s) to build" echo " --ci Set when running on CI server" + echo " --excludeCIBinarylog Don't output binary log (short: -nobl)" echo " --prepareMachine Prepare machine for CI run, clean up processes after build" echo " --nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')" echo " --warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" @@ -68,6 +69,7 @@ clean=false warn_as_error=true node_reuse=true binary_log=false +exclude_ci_binary_log=false pipelines_log=false projects='' @@ -98,6 +100,9 @@ while [[ $# > 0 ]]; do -binarylog|-bl) binary_log=true ;; + -excludeCIBinarylog|-nobl) + exclude_ci_binary_log=true + ;; -pipelineslog|-pl) pipelines_log=true ;; @@ -156,8 +161,10 @@ done if [[ "$ci" == true ]]; then pipelines_log=true - binary_log=true node_reuse=false + if [[ "$exclude_ci_binary_log" == false ]]; then + binary_log=true + fi fi . "$scriptroot/tools.sh" diff --git a/eng/common/native/CommonLibrary.psm1 b/eng/common/native/CommonLibrary.psm1 index 41416862..d7d1a651 100644 --- a/eng/common/native/CommonLibrary.psm1 +++ b/eng/common/native/CommonLibrary.psm1 @@ -145,9 +145,12 @@ function Get-File { New-Item -path $DownloadDirectory -force -itemType "Directory" | Out-Null } + $TempPath = "$Path.tmp" if (Test-Path -IsValid -Path $Uri) { - Write-Verbose "'$Uri' is a file path, copying file to '$Path'" - Copy-Item -Path $Uri -Destination $Path + Write-Verbose "'$Uri' is a file path, copying temporarily to '$TempPath'" + Copy-Item -Path $Uri -Destination $TempPath + Write-Verbose "Moving temporary file to '$Path'" + Move-Item -Path $TempPath -Destination $Path return $? } else { @@ -157,8 +160,10 @@ function Get-File { while($Attempt -Lt $DownloadRetries) { try { - Invoke-WebRequest -UseBasicParsing -Uri $Uri -OutFile $Path - Write-Verbose "Downloaded to '$Path'" + Invoke-WebRequest -UseBasicParsing -Uri $Uri -OutFile $TempPath + Write-Verbose "Downloaded to temporary location '$TempPath'" + Move-Item -Path $TempPath -Destination $Path + Write-Verbose "Moved temporary file to '$Path'" return $True } catch { @@ -359,16 +364,21 @@ function Expand-Zip { return $False } } - if (-Not (Test-Path $OutputDirectory)) { - New-Item -path $OutputDirectory -Force -itemType "Directory" | Out-Null + + $TempOutputDirectory = Join-Path "$(Split-Path -Parent $OutputDirectory)" "$(Split-Path -Leaf $OutputDirectory).tmp" + if (Test-Path $TempOutputDirectory) { + Remove-Item $TempOutputDirectory -Force -Recurse } + New-Item -Path $TempOutputDirectory -Force -ItemType "Directory" | Out-Null Add-Type -assembly "system.io.compression.filesystem" - [io.compression.zipfile]::ExtractToDirectory("$ZipPath", "$OutputDirectory") + [io.compression.zipfile]::ExtractToDirectory("$ZipPath", "$TempOutputDirectory") if ($? -Eq $False) { Write-Error "Unable to extract '$ZipPath'" return $False } + + Move-Item -Path $TempOutputDirectory -Destination $OutputDirectory } catch { Write-Host $_ diff --git a/eng/common/performance/perfhelixpublish.proj b/eng/common/performance/perfhelixpublish.proj index cf5941e1..1db5e8a8 100644 --- a/eng/common/performance/perfhelixpublish.proj +++ b/eng/common/performance/perfhelixpublish.proj @@ -6,6 +6,7 @@ py -3 %HELIX_CORRELATION_PAYLOAD%\Core_Root\CoreRun.exe %HELIX_CORRELATION_PAYLOAD%\Baseline_Core_Root\CoreRun.exe + $(HelixPreCommands);call %HELIX_CORRELATION_PAYLOAD%\performance\tools\machine-setup.cmd;set PYTHONPATH=%HELIX_WORKITEM_PAYLOAD%\scripts%3B%HELIX_WORKITEM_PAYLOAD% %HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts %HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts_Baseline @@ -40,6 +41,13 @@ $HELIX_WORKITEM_ROOT/testResults.xml + + --corerun %HELIX_CORRELATION_PAYLOAD%\dotnet-mono\shared\Microsoft.NETCore.App\5.0.0\corerun.exe + + + --corerun $(BaseDirectory)/dotnet-mono/shared/Microsoft.NETCore.App/5.0.0/corerun + + --corerun $(CoreRun) diff --git a/eng/common/performance/performance-setup.ps1 b/eng/common/performance/performance-setup.ps1 index f5e6ca68..31a99e49 100644 --- a/eng/common/performance/performance-setup.ps1 +++ b/eng/common/performance/performance-setup.ps1 @@ -12,8 +12,12 @@ Param( [string] $RunCategories="Libraries Runtime", [string] $Csproj="src\benchmarks\micro\MicroBenchmarks.csproj", [string] $Kind="micro", + [switch] $LLVM, + [switch] $MonoInterpreter, + [switch] $MonoAOT, [switch] $Internal, [switch] $Compare, + [string] $MonoDotnet="", [string] $Configurations="CompilationMode=$CompilationMode RunKind=$Kind" ) @@ -50,6 +54,21 @@ if ($Internal) { $HelixSourcePrefix = "official" } +if($MonoDotnet -ne "") +{ + $Configurations += " LLVM=$LLVM MonoInterpreter=$MonoInterpreter MonoAOT=$MonoAOT" + if($ExtraBenchmarkDotNetArguments -eq "") + { + #FIX ME: We need to block these tests as they don't run on mono for now + $ExtraBenchmarkDotNetArguments = "--exclusion-filter *Perf_Image* *Perf_NamedPipeStream*" + } + else + { + #FIX ME: We need to block these tests as they don't run on mono for now + $ExtraBenchmarkDotNetArguments += " --exclusion-filter *Perf_Image* *Perf_NamedPipeStream*" + } +} + # FIX ME: This is a workaround until we get this from the actual pipeline $CommonSetupArguments="--channel master --queue $Queue --build-number $BuildNumber --build-configs $Configurations --architecture $Architecture" $SetupArguments = "--repository https://github.com/$Repository --branch $Branch --get-perf-hash --commit-sha $CommitSha $CommonSetupArguments" @@ -70,6 +89,13 @@ else { git clone --branch master --depth 1 --quiet https://github.com/dotnet/performance $PerformanceDirectory } +if($MonoDotnet -ne "") +{ + $UsingMono = "true" + $MonoDotnetPath = (Join-Path $PayloadDirectory "dotnet-mono") + Move-Item -Path $MonoDotnet -Destination $MonoDotnetPath +} + if ($UseCoreRun) { $NewCoreRoot = (Join-Path $PayloadDirectory "Core_Root") Move-Item -Path $CoreRootDirectory -Destination $NewCoreRoot @@ -105,6 +131,7 @@ Write-PipelineSetVariable -Name 'UseCoreRun' -Value "$UseCoreRun" -IsMultiJobVar Write-PipelineSetVariable -Name 'UseBaselineCoreRun' -Value "$UseBaselineCoreRun" -IsMultiJobVariable $false Write-PipelineSetVariable -Name 'RunFromPerfRepo' -Value "$RunFromPerformanceRepo" -IsMultiJobVariable $false Write-PipelineSetVariable -Name 'Compare' -Value "$Compare" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'MonoDotnet' -Value "$UsingMono" -IsMultiJobVariable $false # Helix Arguments Write-PipelineSetVariable -Name 'Creator' -Value "$Creator" -IsMultiJobVariable $false diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index 34eed8b6..9409e4d8 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -12,13 +12,18 @@ commit_sha=$BUILD_SOURCEVERSION build_number=$BUILD_BUILDNUMBER internal=false compare=false +mono_dotnet= kind="micro" +llvm=false +monointerpreter=false +monoaot=false run_categories="Libraries Runtime" csproj="src\benchmarks\micro\MicroBenchmarks.csproj" configurations="CompliationMode=$compilation_mode RunKind=$kind" run_from_perf_repo=false use_core_run=true use_baseline_core_run=true +using_mono=false while (($# > 0)); do lowerI="$(echo $1 | awk '{print tolower($0)}')" @@ -65,6 +70,7 @@ while (($# > 0)); do ;; --kind) kind=$2 + configurations="CompliationMode=$compilation_mode RunKind=$kind" shift 2 ;; --runcategories) @@ -79,6 +85,22 @@ while (($# > 0)); do internal=true shift 1 ;; + --llvm) + llvm=true + shift 1 + ;; + --monointerpreter) + monointerpreter=true + shift 1 + ;; + --monoaot) + monoaot=true + shift 1 + ;; + --monodotnet) + mono_dotnet=$2 + shift 2 + ;; --compare) compare=true shift 1 @@ -107,6 +129,7 @@ while (($# > 0)); do echo " --kind Related to csproj. The kind of benchmarks that should be run. Defaults to micro" echo " --runcategories Related to csproj. Categories of benchmarks to run. Defaults to \"coreclr corefx\"" echo " --internal If the benchmarks are running as an official job." + echo " --monodotnet Pass the path to the mono dotnet for mono performance testing." echo "" exit 0 ;; @@ -164,6 +187,10 @@ if [[ "$internal" == true ]]; then fi fi +if [[ "$mono_dotnet" != "" ]]; then + configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot" +fi + common_setup_arguments="--channel master --queue $queue --build-number $build_number --build-configs $configurations --architecture $architecture" setup_arguments="--repository https://github.com/$repository --branch $branch --get-perf-hash --commit-sha $commit_sha $common_setup_arguments" @@ -186,6 +213,12 @@ else mv $docs_directory $workitem_directory fi +if [[ "$mono_dotnet" != "" ]]; then + using_mono=true + mono_dotnet_path=$payload_directory/dotnet-mono + mv $mono_dotnet $mono_dotnet_path +fi + if [[ "$use_core_run" = true ]]; then new_core_root=$payload_directory/Core_Root mv $core_root_directory $new_core_root @@ -221,3 +254,4 @@ Write-PipelineSetVariable -name "HelixSourcePrefix" -value "$helix_source_prefix Write-PipelineSetVariable -name "Kind" -value "$kind" -is_multi_job_variable false Write-PipelineSetVariable -name "_BuildConfig" -value "$architecture.$kind.$framework" -is_multi_job_variable false Write-PipelineSetVariable -name "Compare" -value "$compare" -is_multi_job_variable false +Write-PipelineSetVariable -name "MonoDotnet" -value "$using_mono" -is_multi_job_variable false diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 index 22eeb265..8e952711 100644 --- a/eng/common/post-build/symbols-validation.ps1 +++ b/eng/common/post-build/symbols-validation.ps1 @@ -2,7 +2,8 @@ param( [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation [Parameter(Mandatory=$true)][string] $DotnetSymbolVersion, # Version of dotnet symbol to use - [Parameter(Mandatory=$false)][switch] $ContinueOnError # If we should keep checking symbols after an error + [Parameter(Mandatory=$false)][switch] $ContinueOnError, # If we should keep checking symbols after an error + [Parameter(Mandatory=$false)][switch] $Clean # Clean extracted symbols directory after checking symbols ) function FirstMatchingSymbolDescriptionOrDefault { @@ -81,7 +82,14 @@ function CountMissingSymbols { $ExtractPath = Join-Path -Path $ExtractPath -ChildPath $PackageGuid $SymbolsPath = Join-Path -Path $ExtractPath -ChildPath 'Symbols' - [System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath) + try { + [System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath) + } + catch { + Write-Host "Something went wrong extracting $PackagePath" + Write-Host $_ + return -1 + } Get-ChildItem -Recurse $ExtractPath | Where-Object {$RelevantExtensions -contains $_.Extension} | @@ -116,6 +124,10 @@ function CountMissingSymbols { } } + if ($Clean) { + Remove-Item $ExtractPath -Recurse -Force + } + Pop-Location return $MissingSymbols @@ -151,7 +163,7 @@ function CheckSymbolsAvailable { if ($Status -ne 0) { Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $Status modules in the package $FileName" - + if ($ContinueOnError) { $TotalFailures++ } diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 47be0bed..b51bc537 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -40,6 +40,8 @@ parameters: Net5Preview3ChannelId: 739 Net5Preview4ChannelId: 856 Net5Preview5ChannelId: 857 + NetCoreSDK313xxChannelId: 759 + NetCoreSDK313xxInternalChannelId: 760 NetCoreSDK314xxChannelId: 921 NetCoreSDK314xxInternalChannelId: 922 @@ -66,7 +68,7 @@ stages: inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 arguments: -PromoteToChannels "$(TargetChannels)" - -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview3ChannelId}},${{parameters.Net5Preview4ChannelId}},${{parameters.Net5Preview5ChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}} + -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview3ChannelId}},${{parameters.Net5Preview4ChannelId}},${{parameters.Net5Preview5ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}} - job: displayName: NuGet Validation @@ -408,3 +410,29 @@ stages: transportFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-symbols/nuget/v3/index.json' + +- template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NETCore_SDK_313xx_Publishing' + channelName: '.NET Core SDK 3.1.3xx' + channelId: ${{ parameters.NetCoreSDK313xxChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-symbols/nuget/v3/index.json' + +- template: \eng\common\templates\post-build\channels\generic-internal-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NETCore_SDK_313xx_Internal_Publishing' + channelName: '.NET Core SDK 3.1.3xx Internal' + channelId: ${{ parameters.NetCoreSDK313xxInternalChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-symbols/nuget/v3/index.json' diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index f31377a6..d8dfc5e0 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -7,9 +7,11 @@ # Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names. [string]$configuration = if (Test-Path variable:configuration) { $configuration } else { 'Debug' } +# Set to true to opt out of outputting binary log while running in CI +[bool]$excludeCIBinarylog = if (Test-Path variable:excludeCIBinarylog) { $excludeCIBinarylog } else { $false } + # Set to true to output binary log from msbuild. Note that emitting binary log slows down the build. -# Binary log must be enabled on CI. -[bool]$binaryLog = if (Test-Path variable:binaryLog) { $binaryLog } else { $ci } +[bool]$binaryLog = if (Test-Path variable:binaryLog) { $binaryLog } else { $ci -and !$excludeCIBinarylog } # Set to true to use the pipelines logger which will enable Azure logging output. # https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md @@ -55,10 +57,8 @@ set-strictmode -version 2.0 $ErrorActionPreference = 'Stop' [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 -function Create-Directory([string[]] $path) { - if (!(Test-Path $path)) { - New-Item -path $path -force -itemType 'Directory' | Out-Null - } +function Create-Directory ([string[]] $path) { + New-Item -Path $path -Force -ItemType 'Directory' | Out-Null } function Unzip([string]$zipfile, [string]$outpath) { @@ -605,8 +605,8 @@ function MSBuild() { # function MSBuild-Core() { if ($ci) { - if (!$binaryLog) { - Write-PipelineTelemetryError -Category 'Build' -Message 'Binary log must be enabled in CI build.' + if (!$binaryLog -and !$excludeCIBinarylog) { + Write-PipelineTelemetryError -Category 'Build' -Message 'Binary log must be enabled in CI build, or explicitly opted-out from with the -excludeCIBinarylog switch.' ExitWithExitCode 1 } diff --git a/eng/common/tools.sh b/eng/common/tools.sh index a9dff440..e94fce22 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -18,9 +18,17 @@ fi # Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names. configuration=${configuration:-'Debug'} +# Set to true to opt out of outputting binary log while running in CI +exclude_ci_binary_log=${exclude_ci_binary_log:-false} + +if [[ "$ci" == true && "$exclude_ci_binary_log" == false ]]; then + binary_log_default=true +else + binary_log_default=false +fi + # Set to true to output binary log from msbuild. Note that emitting binary log slows down the build. -# Binary log must be enabled on CI. -binary_log=${binary_log:-$ci} +binary_log=${binary_log:-$binary_log_default} # Turns on machine preparation/clean up code that changes the machine state (e.g. kills build processes). prepare_machine=${prepare_machine:-false} @@ -404,8 +412,8 @@ function MSBuild { function MSBuild-Core { if [[ "$ci" == true ]]; then - if [[ "$binary_log" != true ]]; then - Write-PipelineTelemetryError -category 'Build' "Binary log must be enabled in CI build." + if [[ "$binary_log" != true && "$exclude_ci_binary_log" != true ]]; then + Write-PipelineTelemetryError -category 'Build' "Binary log must be enabled in CI build, or explicitly opted-out from with the -noBinaryLog switch." ExitWithExitCode 1 fi diff --git a/global.json b/global.json index d6fc19c0..0b775d37 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "5.0.100-preview.5.20251.2" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20256.5" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20261.9" } } From b86047791d87d7ae51286266a9fde2af7cdc529c Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 1 Jun 2020 12:27:01 +0000 Subject: [PATCH 081/161] Update dependencies from https://github.com/dotnet/arcade build 20200528.4 (#618) Microsoft.DotNet.Arcade.Sdk From Version 5.0.0-beta.20261.9 -> To Version 5.0.0-beta.20278.4 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 4 +- eng/common/pipeline-logging-functions.sh | 21 +- eng/common/post-build/publish-using-darc.ps1 | 71 ++ eng/common/post-build/symbols-validation.ps1 | 180 ++-- eng/common/sdl/execute-all-sdl-tools.ps1 | 8 +- eng/common/templates/job/job.yml | 4 +- .../templates/post-build/post-build.yml | 805 +++++++++--------- eng/common/tools.ps1 | 28 +- global.json | 4 +- 9 files changed, 652 insertions(+), 473 deletions(-) create mode 100644 eng/common/post-build/publish-using-darc.ps1 diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 1f5c04cc..71ffc1f8 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 898e51ed5fdcc4871087ac5754ca9056e58e575d + 96c65ad967d2abbbd8e770ba752a6b64255ec669 diff --git a/eng/common/pipeline-logging-functions.sh b/eng/common/pipeline-logging-functions.sh index 33c3f0d8..6cd0a340 100755 --- a/eng/common/pipeline-logging-functions.sh +++ b/eng/common/pipeline-logging-functions.sh @@ -31,26 +31,21 @@ function Write-PipelineTelemetryError { return fi - message="(NETCORE_ENGINEERING_TELEMETRY=$telemetry_category) $message" - function_args+=("$message") if [[ $force == true ]]; then function_args+=("-force") fi - - Write-PipelineTaskError $function_args + message="(NETCORE_ENGINEERING_TELEMETRY=$telemetry_category) $message" + function_args+=("$message") + Write-PipelineTaskError ${function_args[@]} } function Write-PipelineTaskError { - if [[ $force != true ]] && [[ "$ci" != true ]]; then - echo "$@" >&2 - return - fi - local message_type="error" local sourcepath='' local linenumber='' local columnnumber='' local error_code='' + local force=false while [[ $# -gt 0 ]]; do opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" @@ -75,6 +70,9 @@ function Write-PipelineTaskError { error_code=$2 shift ;; + -force|-f) + force=true + ;; *) break ;; @@ -83,6 +81,11 @@ function Write-PipelineTaskError { shift done + if [[ $force != true ]] && [[ "$ci" != true ]]; then + echo "$@" >&2 + return + fi + local message="##vso[task.logissue" message="$message type=$message_type" diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1 new file mode 100644 index 00000000..d22eb439 --- /dev/null +++ b/eng/common/post-build/publish-using-darc.ps1 @@ -0,0 +1,71 @@ +param( + [Parameter(Mandatory=$true)][int] $BuildId, + [Parameter(Mandatory=$true)][string] $AzdoToken, + [Parameter(Mandatory=$true)][string] $MaestroToken, + [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com', + [Parameter(Mandatory=$true)][string] $WaitPublishingFinish, + [Parameter(Mandatory=$true)][string] $EnableSourceLinkValidation, + [Parameter(Mandatory=$true)][string] $EnableSigningValidation, + [Parameter(Mandatory=$true)][string] $EnableNugetValidation, + [Parameter(Mandatory=$true)][string] $PublishInstallersAndChecksums, + [Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters, + [Parameter(Mandatory=$false)][string] $SigningValidationAdditionalParameters +) + +try { + . $PSScriptRoot\post-build-utils.ps1 + . $PSScriptRoot\..\darc-init.ps1 + + $optionalParams = [System.Collections.ArrayList]::new() + + if ("" -ne $ArtifactsPublishingAdditionalParameters) { + $optionalParams.Add("artifact-publishing-parameters") | Out-Null + $optionalParams.Add($ArtifactsPublishingAdditionalParameters) | Out-Null + } + + if ("false" -eq $WaitPublishingFinish) { + $optionalParams.Add("--no-wait") | Out-Null + } + + if ("true" -eq $PublishInstallersAndChecksums) { + $optionalParams.Add("--publish-installers-and-checksums") | Out-Null + } + + if ("true" -eq $EnableNugetValidation) { + $optionalParams.Add("--validate-nuget") | Out-Null + } + + if ("true" -eq $EnableSourceLinkValidation) { + $optionalParams.Add("--validate-sourcelinkchecksums") | Out-Null + } + + if ("true" -eq $EnableSigningValidation) { + $optionalParams.Add("--validate-signingchecksums") | Out-Null + + if ("" -ne $SigningValidationAdditionalParameters) { + $optionalParams.Add("--signing-validation-parameters") | Out-Null + $optionalParams.Add($SigningValidationAdditionalParameters) | Out-Null + } + } + + & darc add-build-to-channel ` + --id $buildId ` + --default-channels ` + --source-branch master ` + --azdev-pat $AzdoToken ` + --bar-uri $MaestroApiEndPoint ` + --password $MaestroToken ` + @optionalParams + + if ($LastExitCode -ne 0) { + Write-Host "Problems using Darc to promote build ${buildId} to default channels. Stopping execution..." + exit 1 + } + + Write-Host 'done.' +} +catch { + Write-Host $_ + Write-PipelineTelemetryError -Category 'PromoteBuild' -Message "There was an error while trying to publish build '$BuildId' to default channels." + ExitWithExitCode 1 +} diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 index 8e952711..e108bf6c 100644 --- a/eng/common/post-build/symbols-validation.ps1 +++ b/eng/common/post-build/symbols-validation.ps1 @@ -6,69 +6,25 @@ param( [Parameter(Mandatory=$false)][switch] $Clean # Clean extracted symbols directory after checking symbols ) -function FirstMatchingSymbolDescriptionOrDefault { - param( - [string] $FullPath, # Full path to the module that has to be checked - [string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols - [string] $SymbolsPath - ) - - $FileName = [System.IO.Path]::GetFileName($FullPath) - $Extension = [System.IO.Path]::GetExtension($FullPath) - - # Those below are potential symbol files that the `dotnet symbol` might - # return. Which one will be returned depend on the type of file we are - # checking and which type of file was uploaded. - - # The file itself is returned - $SymbolPath = $SymbolsPath + '\' + $FileName - - # PDB file for the module - $PdbPath = $SymbolPath.Replace($Extension, '.pdb') - - # PDB file for R2R module (created by crossgen) - $NGenPdb = $SymbolPath.Replace($Extension, '.ni.pdb') +# Maximum number of jobs to run in parallel +$MaxParallelJobs = 6 - # DBG file for a .so library - $SODbg = $SymbolPath.Replace($Extension, '.so.dbg') +# Wait time between check for system load +$SecondsBetweenLoadChecks = 10 - # DWARF file for a .dylib - $DylibDwarf = $SymbolPath.Replace($Extension, '.dylib.dwarf') - - $dotnetSymbolExe = "$env:USERPROFILE\.dotnet\tools" - $dotnetSymbolExe = Resolve-Path "$dotnetSymbolExe\dotnet-symbol.exe" - - & $dotnetSymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null - - if (Test-Path $PdbPath) { - return 'PDB' - } - elseif (Test-Path $NGenPdb) { - return 'NGen PDB' - } - elseif (Test-Path $SODbg) { - return 'DBG for SO' - } - elseif (Test-Path $DylibDwarf) { - return 'Dwarf for Dylib' - } - elseif (Test-Path $SymbolPath) { - return 'Module' - } - else { - return $null - } -} - -function CountMissingSymbols { +$CountMissingSymbols = { param( [string] $PackagePath # Path to a NuGet package ) + . $using:PSScriptRoot\..\tools.ps1 + + Add-Type -AssemblyName System.IO.Compression.FileSystem + # Ensure input file exist if (!(Test-Path $PackagePath)) { Write-PipelineTaskError "Input file does not exist: $PackagePath" - ExitWithExitCode 1 + return 1 } # Extensions for which we'll look for symbols @@ -79,7 +35,7 @@ function CountMissingSymbols { $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) $PackageGuid = New-Guid - $ExtractPath = Join-Path -Path $ExtractPath -ChildPath $PackageGuid + $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageGuid $SymbolsPath = Join-Path -Path $ExtractPath -ChildPath 'Symbols' try { @@ -94,15 +50,70 @@ function CountMissingSymbols { Get-ChildItem -Recurse $ExtractPath | Where-Object {$RelevantExtensions -contains $_.Extension} | ForEach-Object { - if ($_.FullName -Match '\\ref\\') { - Write-Host "`t Ignoring reference assembly file " $_.FullName + $FileName = $_.FullName + if ($FileName -Match '\\ref\\') { + Write-Host "`t Ignoring reference assembly file " $FileName return } - $SymbolsOnMSDL = FirstMatchingSymbolDescriptionOrDefault $_.FullName '--microsoft-symbol-server' $SymbolsPath - $SymbolsOnSymWeb = FirstMatchingSymbolDescriptionOrDefault $_.FullName '--internal-server' $SymbolsPath + $FirstMatchingSymbolDescriptionOrDefault = { + param( + [string] $FullPath, # Full path to the module that has to be checked + [string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols + [string] $SymbolsPath + ) + + $FileName = [System.IO.Path]::GetFileName($FullPath) + $Extension = [System.IO.Path]::GetExtension($FullPath) + + # Those below are potential symbol files that the `dotnet symbol` might + # return. Which one will be returned depend on the type of file we are + # checking and which type of file was uploaded. + + # The file itself is returned + $SymbolPath = $SymbolsPath + '\' + $FileName + + # PDB file for the module + $PdbPath = $SymbolPath.Replace($Extension, '.pdb') + + # PDB file for R2R module (created by crossgen) + $NGenPdb = $SymbolPath.Replace($Extension, '.ni.pdb') + + # DBG file for a .so library + $SODbg = $SymbolPath.Replace($Extension, '.so.dbg') + + # DWARF file for a .dylib + $DylibDwarf = $SymbolPath.Replace($Extension, '.dylib.dwarf') + + $dotnetSymbolExe = "$env:USERPROFILE\.dotnet\tools" + $dotnetSymbolExe = Resolve-Path "$dotnetSymbolExe\dotnet-symbol.exe" + + & $dotnetSymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null + + if (Test-Path $PdbPath) { + return 'PDB' + } + elseif (Test-Path $NGenPdb) { + return 'NGen PDB' + } + elseif (Test-Path $SODbg) { + return 'DBG for SO' + } + elseif (Test-Path $DylibDwarf) { + return 'Dwarf for Dylib' + } + elseif (Test-Path $SymbolPath) { + return 'Module' + } + else { + return $null + } + } + + $SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault $FileName '--microsoft-symbol-server' $SymbolsPath + $SymbolsOnSymWeb = & $FirstMatchingSymbolDescriptionOrDefault $FileName '--internal-server' $SymbolsPath - Write-Host -NoNewLine "`t Checking file " $_.FullName "... " + Write-Host -NoNewLine "`t Checking file " $FileName "... " if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) { Write-Host "Symbols found on MSDL ($SymbolsOnMSDL) and SymWeb ($SymbolsOnSymWeb)" @@ -124,9 +135,14 @@ function CountMissingSymbols { } } - if ($Clean) { + if ($using:Clean) { Remove-Item $ExtractPath -Recurse -Force } + + if ($MissingSymbols -ne 0) + { + Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $MissingSymbols modules in the package $FileName" + } Pop-Location @@ -143,6 +159,7 @@ function CheckSymbolsAvailable { Get-ChildItem "$InputPath\*.nupkg" | ForEach-Object { $FileName = $_.Name + $FullName = $_.FullName # These packages from Arcade-Services include some native libraries that # our current symbol uploader can't handle. Below is a workaround until @@ -159,26 +176,39 @@ function CheckSymbolsAvailable { } Write-Host "Validating $FileName " - $Status = CountMissingSymbols "$InputPath\$FileName" - if ($Status -ne 0) { - Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $Status modules in the package $FileName" - - if ($ContinueOnError) { - $TotalFailures++ - } - else { - ExitWithExitCode 1 - } + Start-Job -ScriptBlock $CountMissingSymbols -ArgumentList $FullName | Out-Null + + $NumJobs = @(Get-Job -State 'Running').Count + Write-Host $NumJobs + + while ($NumJobs -ge $MaxParallelJobs) { + Write-Host "There are $NumJobs validation jobs running right now. Waiting $SecondsBetweenLoadChecks seconds to check again." + sleep $SecondsBetweenLoadChecks + $NumJobs = @(Get-Job -State 'Running').Count } + foreach ($Job in @(Get-Job -State 'Completed')) { + Receive-Job -Id $Job.Id + } Write-Host } - if ($TotalFailures -ne 0) { + foreach ($Job in @(Get-Job)) { + $jobResult = Wait-Job -Id $Job.Id | Receive-Job + + if ($jobResult -ne '0') { + $TotalFailures++ + } + } + + if ($TotalFailures -gt 0) { Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Symbols missing for $TotalFailures packages" ExitWithExitCode 1 } + else { + Write-Host "All symbols validated!" + } } function InstallDotnetSymbol { @@ -200,11 +230,13 @@ function InstallDotnetSymbol { try { . $PSScriptRoot\post-build-utils.ps1 - - Add-Type -AssemblyName System.IO.Compression.FileSystem InstallDotnetSymbol + foreach ($Job in @(Get-Job)) { + Remove-Job -Id $Job.Id + } + CheckSymbolsAvailable } catch { diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1 index b7f61f9a..b681d797 100644 --- a/eng/common/sdl/execute-all-sdl-tools.ps1 +++ b/eng/common/sdl/execute-all-sdl-tools.ps1 @@ -24,7 +24,8 @@ Param( [string] $TsaIterationPath, # Optional: only needed if TsaOnboard is true; the iteration path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. [string] $GuardianLoggerLevel='Standard', # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error [string[]] $CrScanAdditionalRunConfigParams, # Optional: Additional Params to custom build a CredScan run config in the format @("xyz:abc","sdf:1") - [string[]] $PoliCheckAdditionalRunConfigParams # Optional: Additional Params to custom build a Policheck run config in the format @("xyz:abc","sdf:1") + [string[]] $PoliCheckAdditionalRunConfigParams, # Optional: Additional Params to custom build a Policheck run config in the format @("xyz:abc","sdf:1") + [bool] $BreakOnFailure=$False # Optional: Fail the build if there were errors during the run ) try { @@ -106,6 +107,11 @@ try { ExitWithExitCode 1 } } + + if ($BreakOnFailure) { + Write-Host "Failing the build in case of breaking results..." + & $guardianCliLocation break + } } catch { Write-Host $_.ScriptStackTrace diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index fc39647f..c08bb416 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -26,7 +26,7 @@ parameters: enablePublishUsingPipelines: false useBuildManifest: false mergeTestResults: false - testRunTitle: $(AgentOsName)-$(BuildConfiguration)-xunit + testRunTitle: '' name: '' preSteps: [] runAsPublic: false @@ -197,7 +197,7 @@ jobs: testResultsFormat: 'xUnit' testResultsFiles: '*.xml' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' - testRunTitle: ${{ parameters.testRunTitle }} + testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit mergeTestResults: ${{ parameters.mergeTestResults }} continueOnError: true condition: always() diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index b51bc537..75890b37 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -1,4 +1,13 @@ parameters: + # When set to true the publishing templates from the repo will be used + # otherwise Darc add-build-to-channel will be used to trigger the promotion pipeline + inline: true + + # Only used if inline==false. When set to true will stall the current build until + # the Promotion Pipeline build finishes. Otherwise, the current build continue + # execution concurrently with the promotion build. + waitPublishingFinish: true + enableSourceLinkValidation: false enableSigningValidation: true enableSymbolValidation: false @@ -37,402 +46,436 @@ parameters: NETCoreExperimentalChannelId: 562 NetEngServicesIntChannelId: 678 NetEngServicesProdChannelId: 679 - Net5Preview3ChannelId: 739 - Net5Preview4ChannelId: 856 Net5Preview5ChannelId: 857 + Net5Preview6ChannelId: 1013 + Net5Preview7ChannelId: 1014 NetCoreSDK313xxChannelId: 759 NetCoreSDK313xxInternalChannelId: 760 NetCoreSDK314xxChannelId: 921 NetCoreSDK314xxInternalChannelId: 922 stages: -- stage: Validate - dependsOn: ${{ parameters.validateDependsOn }} - displayName: Validate - variables: - - template: common-variables.yml - jobs: - - template: setup-maestro-vars.yml - - - job: - displayName: Post-build Checks - dependsOn: setupMaestroVars - variables: - - name: TargetChannels - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ] - pool: - vmImage: 'windows-2019' - steps: - - task: PowerShell@2 - displayName: Maestro Channels Consistency - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 - arguments: -PromoteToChannels "$(TargetChannels)" - -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview3ChannelId}},${{parameters.Net5Preview4ChannelId}},${{parameters.Net5Preview5ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}} - - - job: - displayName: NuGet Validation - dependsOn: setupMaestroVars - condition: eq( ${{ parameters.enableNugetValidation }}, 'true') - pool: - vmImage: 'windows-2019' +- ${{ if ne(parameters.inline, 'true') }}: + - stage: publish_using_darc + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Publish using Darc variables: - - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 - arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ - -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ - - - job: - displayName: Signing Validation - dependsOn: setupMaestroVars - condition: eq( ${{ parameters.enableSigningValidation }}, 'true') + - template: common-variables.yml + jobs: + - template: setup-maestro-vars.yml + + - job: + displayName: Publish Using Darc + dependsOn: setupMaestroVars + variables: + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + pool: + vmImage: 'windows-2019' + steps: + - task: PowerShell@2 + displayName: Publish Using Darc + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) + -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -MaestroToken '$(MaestroApiAccessToken)' + -WaitPublishingFinish ${{ parameters.waitPublishingFinish }} + -EnableSourceLinkValidation ${{ parameters.enableSourceLinkValidation }} + -EnableSigningValidation ${{ parameters.enableSourceLinkValidation }} + -EnableNugetValidation ${{ parameters.enableSourceLinkValidation }} + -PublishInstallersAndChecksums ${{ parameters.publishInstallersAndChecksums }} + -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' + -SigningValidationAdditionalParameters '${{ parameters.signingValidationAdditionalParameters }}' + +- ${{ if eq(parameters.inline, 'true') }}: + - stage: Validate + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Validate Build Assets variables: - template: common-variables.yml - - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] - pool: - vmImage: 'windows-2019' - steps: - - ${{ if eq(parameters.useBuildManifest, true) }}: + jobs: + - template: setup-maestro-vars.yml + + - job: + displayName: Post-build Checks + dependsOn: setupMaestroVars + variables: + - name: TargetChannels + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ] + pool: + vmImage: 'windows-2019' + steps: + - task: PowerShell@2 + displayName: Maestro Channels Consistency + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 + arguments: -PromoteToChannels "$(TargetChannels)" + -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview5ChannelId}},${{parameters.Net5Preview6ChannelId}},${{parameters.Net5Preview7ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}} + + - job: + displayName: NuGet Validation + dependsOn: setupMaestroVars + condition: eq( ${{ parameters.enableNugetValidation }}, 'true') + pool: + vmImage: 'windows-2019' + variables: + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + steps: - task: DownloadBuildArtifacts@0 - displayName: Download build manifest + displayName: Download Package Artifacts inputs: buildType: specific buildVersionToDownload: specific project: $(AzDOProjectName) pipeline: $(AzDOPipelineId) buildId: $(AzDOBuildId) - artifactName: BuildManifests - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - - # This is necessary whenever we want to publish/restore to an AzDO private feed - # Since sdk-task.ps1 tries to restore packages we need to do this authentication here - # otherwise it'll complain about accessing a private feed. - - task: NuGetAuthenticate@0 - displayName: 'Authenticate to AzDO Feeds' - - - task: PowerShell@2 - displayName: Enable cross-org publishing - inputs: - filePath: eng\common\enable-cross-org-publishing.ps1 - arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) - - # Signing validation will optionally work with the buildmanifest file which is downloaded from - # Azure DevOps above. - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task SigningValidation -restore -msbuildEngine vs - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' - /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' - ${{ parameters.signingValidationAdditionalParameters }} - - - template: ../steps/publish-logs.yml - parameters: - StageLabel: 'Validation' - JobLabel: 'Signing' - - - job: - displayName: SourceLink Validation - dependsOn: setupMaestroVars - condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') - variables: - - template: common-variables.yml - - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: BlobArtifacts - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -ExtractPath $(Agent.BuildDirectory)/Extract/ - -GHRepoName $(Build.Repository.Name) - -GHCommit $(Build.SourceVersion) - -SourcelinkCliVersion $(SourceLinkCLIVersion) - continueOnError: true - - - template: /eng/common/templates/job/execute-sdl.yml - parameters: - enable: ${{ parameters.SDLValidationParameters.enable }} + artifactName: PackageArtifacts + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ + -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ + + - job: + displayName: Signing Validation dependsOn: setupMaestroVars - additionalParameters: ${{ parameters.SDLValidationParameters.params }} - continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} - artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }} - downloadArtifacts: ${{ parameters.SDLValidationParameters.downloadArtifacts }} - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'NetCore_Dev5_Publish' - channelName: '.NET 5 Dev' - akaMSChannelName: 'net5/dev' - channelId: ${{ parameters.NetDev5ChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net5_Preview3_Publish' - channelName: '.NET 5 Preview 3' - akaMSChannelName: 'net5/preview3' - channelId: ${{ parameters.Net5Preview3ChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net5_Preview4_Publish' - channelName: '.NET 5 Preview 4' - akaMSChannelName: 'net5/preview4' - channelId: ${{ parameters.Net5Preview4ChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net5_Preview5_Publish' - channelName: '.NET 5 Preview 5' - akaMSChannelName: 'net5/preview5' - channelId: ${{ parameters.Net5Preview5ChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net_Eng_Latest_Publish' - channelName: '.NET Eng - Latest' - akaMSChannelName: 'eng/daily' - channelId: ${{ parameters.NetEngLatestChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net_Eng_Validation_Publish' - channelName: '.NET Eng - Validation' - akaMSChannelName: 'eng/validation' - channelId: ${{ parameters.NetEngValidationChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'General_Testing_Publish' - channelName: 'General Testing' - akaMSChannelName: 'generaltesting' - channelId: ${{ parameters.GeneralTestingChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'NETCore_Tooling_Dev_Publishing' - channelName: '.NET Core Tooling Dev' - channelId: ${{ parameters.NETCoreToolingDevChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'NETCore_Tooling_Release_Publishing' - channelName: '.NET Core Tooling Release' - channelId: ${{ parameters.NETCoreToolingReleaseChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-internal-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'NET_Internal_Tooling_Publishing' - channelName: '.NET Internal Tooling' - channelId: ${{ parameters.NETInternalToolingChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'NETCore_Experimental_Publishing' - channelName: '.NET Core Experimental' - channelId: ${{ parameters.NETCoreExperimentalChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net_Eng_Services_Int_Publish' - channelName: '.NET Eng Services - Int' - channelId: ${{ parameters.NetEngServicesIntChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net_Eng_Services_Prod_Publish' - channelName: '.NET Eng Services - Prod' - channelId: ${{ parameters.NetEngServicesProdChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'NETCore_SDK_314xx_Publishing' - channelName: '.NET Core SDK 3.1.4xx' - channelId: ${{ parameters.NetCoreSDK314xxChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-internal-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'NETCore_SDK_314xx_Internal_Publishing' - channelName: '.NET Core SDK 3.1.4xx Internal' - channelId: ${{ parameters.NetCoreSDK314xxInternalChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'NETCore_SDK_313xx_Publishing' - channelName: '.NET Core SDK 3.1.3xx' - channelId: ${{ parameters.NetCoreSDK313xxChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-internal-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'NETCore_SDK_313xx_Internal_Publishing' - channelName: '.NET Core SDK 3.1.3xx Internal' - channelId: ${{ parameters.NetCoreSDK313xxInternalChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-symbols/nuget/v3/index.json' + condition: eq( ${{ parameters.enableSigningValidation }}, 'true') + variables: + - template: common-variables.yml + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + pool: + vmImage: 'windows-2019' + steps: + - ${{ if eq(parameters.useBuildManifest, true) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download build manifest + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BuildManifests + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + + # This is necessary whenever we want to publish/restore to an AzDO private feed + # Since sdk-task.ps1 tries to restore packages we need to do this authentication here + # otherwise it'll complain about accessing a private feed. + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + + - task: PowerShell@2 + displayName: Enable cross-org publishing + inputs: + filePath: eng\common\enable-cross-org-publishing.ps1 + arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) + + # Signing validation will optionally work with the buildmanifest file which is downloaded from + # Azure DevOps above. + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task SigningValidation -restore -msbuildEngine vs + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' + /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' + ${{ parameters.signingValidationAdditionalParameters }} + + - template: ../steps/publish-logs.yml + parameters: + StageLabel: 'Validation' + JobLabel: 'Signing' + + - job: + displayName: SourceLink Validation + dependsOn: setupMaestroVars + condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') + variables: + - template: common-variables.yml + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BlobArtifacts + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) + -GHCommit $(Build.SourceVersion) + -SourcelinkCliVersion $(SourceLinkCLIVersion) + continueOnError: true + + - template: /eng/common/templates/job/execute-sdl.yml + parameters: + enable: ${{ parameters.SDLValidationParameters.enable }} + dependsOn: setupMaestroVars + additionalParameters: ${{ parameters.SDLValidationParameters.params }} + continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} + artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }} + downloadArtifacts: ${{ parameters.SDLValidationParameters.downloadArtifacts }} + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NetCore_Dev5_Publish' + channelName: '.NET 5 Dev' + akaMSChannelName: 'net5/dev' + channelId: ${{ parameters.NetDev5ChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'Net5_Preview5_Publish' + channelName: '.NET 5 Preview 5' + akaMSChannelName: 'net5/preview5' + channelId: ${{ parameters.Net5Preview5ChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'Net5_Preview6_Publish' + channelName: '.NET 5 Preview 6' + akaMSChannelName: 'net5/preview6' + channelId: ${{ parameters.Net5Preview6ChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'Net5_Preview7_Publish' + channelName: '.NET 5 Preview 7' + akaMSChannelName: 'net5/preview7' + channelId: ${{ parameters.Net5Preview7ChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'Net_Eng_Latest_Publish' + channelName: '.NET Eng - Latest' + akaMSChannelName: 'eng/daily' + channelId: ${{ parameters.NetEngLatestChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'Net_Eng_Validation_Publish' + channelName: '.NET Eng - Validation' + akaMSChannelName: 'eng/validation' + channelId: ${{ parameters.NetEngValidationChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'General_Testing_Publish' + channelName: 'General Testing' + akaMSChannelName: 'generaltesting' + channelId: ${{ parameters.GeneralTestingChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NETCore_Tooling_Dev_Publishing' + channelName: '.NET Core Tooling Dev' + channelId: ${{ parameters.NETCoreToolingDevChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NETCore_Tooling_Release_Publishing' + channelName: '.NET Core Tooling Release' + channelId: ${{ parameters.NETCoreToolingReleaseChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NET_Internal_Tooling_Publishing' + channelName: '.NET Internal Tooling' + channelId: ${{ parameters.NETInternalToolingChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NETCore_Experimental_Publishing' + channelName: '.NET Core Experimental' + channelId: ${{ parameters.NETCoreExperimentalChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'Net_Eng_Services_Int_Publish' + channelName: '.NET Eng Services - Int' + channelId: ${{ parameters.NetEngServicesIntChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'Net_Eng_Services_Prod_Publish' + channelName: '.NET Eng Services - Prod' + channelId: ${{ parameters.NetEngServicesProdChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NETCore_SDK_314xx_Publishing' + channelName: '.NET Core SDK 3.1.4xx' + channelId: ${{ parameters.NetCoreSDK314xxChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NETCore_SDK_314xx_Internal_Publishing' + channelName: '.NET Core SDK 3.1.4xx Internal' + channelId: ${{ parameters.NetCoreSDK314xxInternalChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NETCore_SDK_313xx_Publishing' + channelName: '.NET Core SDK 3.1.3xx' + channelId: ${{ parameters.NetCoreSDK313xxChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NETCore_SDK_313xx_Internal_Publishing' + channelName: '.NET Core SDK 3.1.3xx Internal' + channelId: ${{ parameters.NetCoreSDK313xxInternalChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-symbols/nuget/v3/index.json' diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index d8dfc5e0..9f6e75f7 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -124,7 +124,9 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { # Find the first path on %PATH% that contains the dotnet.exe if ($useInstalledDotNetCli -and (-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -eq $null)) { - $dotnetCmd = Get-Command 'dotnet.exe' -ErrorAction SilentlyContinue + $dotnetExecutable = GetExecutableFileName 'dotnet' + $dotnetCmd = Get-Command $dotnetExecutable -ErrorAction SilentlyContinue + if ($dotnetCmd -ne $null) { $env:DOTNET_INSTALL_DIR = Split-Path $dotnetCmd.Path -Parent } @@ -283,6 +285,10 @@ function InstallDotNet([string] $dotnetRoot, # Throws on failure. # function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = $null) { + if (-not (IsWindowsPlatform)) { + throw "Cannot initialize Visual Studio on non-Windows" + } + if (Test-Path variable:global:_MSBuildExe) { return $global:_MSBuildExe } @@ -387,6 +393,10 @@ function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { # or $null if no instance meeting the requirements is found on the machine. # function LocateVisualStudio([object]$vsRequirements = $null){ + if (-not (IsWindowsPlatform)) { + throw "Cannot run vswhere on non-Windows platforms." + } + if (Get-Member -InputObject $GlobalJson.tools -Name 'vswhere') { $vswhereVersion = $GlobalJson.tools.vswhere } else { @@ -452,7 +462,8 @@ function InitializeBuildTool() { Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "/global.json must specify 'tools.dotnet'." ExitWithExitCode 1 } - $buildTool = @{ Path = Join-Path $dotnetRoot 'dotnet.exe'; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'netcoreapp2.1' } + $dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet') + $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'netcoreapp2.1' } } elseif ($msbuildEngine -eq "vs") { try { $msbuildPath = InitializeVisualStudioMSBuild -install:$restore @@ -666,6 +677,19 @@ function GetMSBuildBinaryLogCommandLineArgument($arguments) { return $null } +function GetExecutableFileName($baseName) { + if (IsWindowsPlatform) { + return "$baseName.exe" + } + else { + return $baseName + } +} + +function IsWindowsPlatform() { + return [environment]::OSVersion.Platform -eq [PlatformID]::Win32NT +} + . $PSScriptRoot\pipeline-logging-functions.ps1 $RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..') diff --git a/global.json b/global.json index 0b775d37..c58886cf 100644 --- a/global.json +++ b/global.json @@ -1,8 +1,8 @@ { "tools": { - "dotnet": "5.0.100-preview.5.20251.2" + "dotnet": "5.0.100-preview.6.20266.3" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20261.9" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20278.4" } } From 1996e4976760e12a2ba4461bc405a0b86d7f7d04 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 8 Jun 2020 12:26:28 +0000 Subject: [PATCH 082/161] Update dependencies from https://github.com/dotnet/arcade build 20200530.1 (#622) Microsoft.DotNet.Arcade.Sdk From Version 5.0.0-beta.20278.4 -> To Version 5.0.0-beta.20280.1 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 4 ++-- eng/common/sdk-task.ps1 | 7 ++++++- eng/common/tools.ps1 | 19 ++++++++++++++++--- global.json | 2 +- 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 71ffc1f8..fd22adfe 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 96c65ad967d2abbbd8e770ba752a6b64255ec669 + fef373440d604c428950236fbc2b99ce0df368a9 diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index f997be43..32ad5c7e 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -65,8 +65,13 @@ try { if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.5.0-alpha" -MemberType NoteProperty } + if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { + $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true + } + if ($xcopyMSBuildToolsFolder -eq $null) { + throw 'Unable to get xcopy downloadable version of msbuild' + } - $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true $global:_MSBuildExe = "$($xcopyMSBuildToolsFolder)\MSBuild\Current\Bin\MSBuild.exe" } diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 9f6e75f7..d68684e2 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -293,8 +293,11 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = return $global:_MSBuildExe } + $vsMinVersionReqdStr = '16.5' + $vsMinVersionReqd = [Version]::new($vsMinVersionReqdStr) + if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } - $vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { '15.9' } + $vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { $vsMinVersionReqdStr } $vsMinVersion = [Version]::new($vsMinVersionStr) # Try msbuild command available in the environment. @@ -327,8 +330,18 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = $xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild' $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0] } else { - $vsMajorVersion = $vsMinVersion.Major - $xcopyMSBuildVersion = "$vsMajorVersion.$($vsMinVersion.Minor).0-alpha" + #if vs version provided in global.json is incompatible then use the default version for xcopy msbuild download + if($vsMinVersion -lt $vsMinVersionReqd){ + Write-Host "Using xcopy-msbuild version of $vsMinVersionReqdStr.0-alpha since VS version $vsMinVersionStr provided in global.json is not compatible" + $vsMajorVersion = $vsMinVersionReqd.Major + $vsMinorVersion = $vsMinVersionReqd.Minor + } + else{ + $vsMajorVersion = $vsMinVersion.Major + $vsMinorVersion = $vsMinVersion.Minor + } + + $xcopyMSBuildVersion = "$vsMajorVersion.$vsMinorVersion.0-alpha" } $vsInstallDir = $null diff --git a/global.json b/global.json index c58886cf..9c5826d8 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "5.0.100-preview.6.20266.3" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20278.4" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20280.1" } } From 79fcb428780aabf5ca49d407618a2122a9dd2865 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 15 Jun 2020 12:11:27 +0000 Subject: [PATCH 083/161] Update dependencies from https://github.com/dotnet/arcade build 20200609.1 (#623) Microsoft.DotNet.Arcade.Sdk From Version 5.0.0-beta.20280.1 -> To Version 5.0.0-beta.20309.1 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 4 +- eng/common/cross/build-rootfs.sh | 67 ++++++++++++++++++- eng/common/cross/toolchain.cmake | 45 ++++++++++++- eng/common/performance/perfhelixpublish.proj | 38 ++++++++++- .../post-build/sourcelink-validation.ps1 | 9 ++- eng/common/post-build/symbols-validation.ps1 | 6 +- global.json | 2 +- 7 files changed, 158 insertions(+), 13 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index fd22adfe..d48bc511 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - fef373440d604c428950236fbc2b99ce0df368a9 + 54d096ee50a9bcf7ac6efd16a07d51af651d20a1 diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index d780fefb..d6ec94b7 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -8,8 +8,10 @@ usage() echo "BuildArch can be: arm(default), armel, arm64, x86" echo "CodeName - optional, Code name for Linux, can be: trusty, xenial(default), zesty, bionic, alpine. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." echo " for FreeBSD can be: freebsd11 or freebsd12." + echo " for illumos can be: illumos." echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FReeBSD" echo "--skipunmount - optional, will skip the unmount of rootfs folder." + echo "--use-mirror - optional, use mirror URL to fetch resources, when available." exit 1 } @@ -67,6 +69,13 @@ __FreeBSDPackages+=" libinotify" __FreeBSDPackages+=" lttng-ust" __FreeBSDPackages+=" krb5" +__IllumosPackages="icu-64.2nb2" +__IllumosPackages+=" mit-krb5-1.16.2nb4" +__IllumosPackages+=" openssl-1.1.1e" +__IllumosPackages+=" zlib-1.2.11" + +__UseMirror=0 + __UnprocessedBuildArgs= while :; do if [ $# -le 0 ]; then @@ -179,6 +188,11 @@ while :; do __BuildArch=x64 __SkipUnmount=1 ;; + illumos) + __CodeName=illumos + __BuildArch=x64 + __SkipUnmount=1 + ;; --skipunmount) __SkipUnmount=1 ;; @@ -186,6 +200,9 @@ while :; do shift __RootfsDir=$1 ;; + --use-mirror) + __UseMirror=1 + ;; *) __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1" ;; @@ -214,6 +231,9 @@ if [ -d "$__RootfsDir" ]; then rm -rf $__RootfsDir fi +mkdir -p $__RootfsDir +__RootfsDir="$( cd "$__RootfsDir" && pwd )" + if [[ "$__CodeName" == "alpine" ]]; then __ApkToolsVersion=2.9.1 __AlpineVersion=3.9 @@ -257,6 +277,51 @@ elif [[ "$__CodeName" == "freebsd" ]]; then # install packages we need. INSTALL_AS_USER=$(whoami) $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf update INSTALL_AS_USER=$(whoami) $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf install --yes $__FreeBSDPackages +elif [[ "$__CodeName" == "illumos" ]]; then + mkdir "$__RootfsDir/tmp" + pushd "$__RootfsDir/tmp" + JOBS="$(getconf _NPROCESSORS_ONLN)" + echo "Downloading sysroot." + wget -O - https://github.com/illumos/sysroot/releases/download/20181213-de6af22ae73b-v1/illumos-sysroot-i386-20181213-de6af22ae73b-v1.tar.gz | tar -C "$__RootfsDir" -xzf - + echo "Building binutils. Please wait.." + wget -O - https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.bz2 | tar -xjf - + mkdir build-binutils && cd build-binutils + ../binutils-2.33.1/configure --prefix="$__RootfsDir" --target="x86_64-sun-solaris2.10" --program-prefix="x86_64-illumos-" --with-sysroot="$__RootfsDir" + make -j "$JOBS" && make install && cd .. + echo "Building gcc. Please wait.." + wget -O - https://ftp.gnu.org/gnu/gcc/gcc-8.4.0/gcc-8.4.0.tar.xz | tar -xJf - + CFLAGS="-fPIC" + CXXFLAGS="-fPIC" + CXXFLAGS_FOR_TARGET="-fPIC" + CFLAGS_FOR_TARGET="-fPIC" + export CFLAGS CXXFLAGS CXXFLAGS_FOR_TARGET CFLAGS_FOR_TARGET + mkdir build-gcc && cd build-gcc + ../gcc-8.4.0/configure --prefix="$__RootfsDir" --target="x86_64-sun-solaris2.10" --program-prefix="x86_64-illumos-" --with-sysroot="$__RootfsDir" --with-gnu-as \ + --with-gnu-ld --disable-nls --disable-libgomp --disable-libquadmath --disable-libssp --disable-libvtv --disable-libcilkrts --disable-libada --disable-libsanitizer \ + --disable-libquadmath-support --disable-shared --enable-tls + make -j "$JOBS" && make install && cd .. + BaseUrl=https://pkgsrc.joyent.com + if [[ "$__UseMirror" == 1 ]]; then + BaseUrl=http://pkgsrc.smartos.skylime.net + fi + BaseUrl="$BaseUrl"/packages/SmartOS/2020Q1/x86_64/All + echo "Downloading dependencies." + read -ra array <<<"$__IllumosPackages" + for package in "${array[@]}"; do + echo "Installing $package..." + wget "$BaseUrl"/"$package".tgz + ar -x "$package".tgz + tar --skip-old-files -xzf "$package".tmp.tgz -C "$__RootfsDir" 2>/dev/null + done + echo "Cleaning up temporary files." + popd + rm -rf "$__RootfsDir"/{tmp,+*} + mkdir -p "$__RootfsDir"/usr/include/net + mkdir -p "$__RootfsDir"/usr/include/netpacket + wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/bpf.h + wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/dlt.h + wget -P "$__RootfsDir"/usr/include/netpacket https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/inet/sockmods/netpacket/packet.h + wget -P "$__RootfsDir"/usr/include/sys https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/sys/sdt.h elif [[ -n $__CodeName ]]; then qemu-debootstrap --arch $__UbuntuArch $__CodeName $__RootfsDir $__UbuntuRepo cp $__CrossDir/$__BuildArch/sources.list.$__CodeName $__RootfsDir/etc/apt/sources.list @@ -275,7 +340,7 @@ elif [[ -n $__CodeName ]]; then patch -p1 < $__CrossDir/$__BuildArch/trusty-lttng-2.4.patch popd fi -elif [ "$__Tizen" == "tizen" ]; then +elif [[ "$__Tizen" == "tizen" ]]; then ROOTFS_DIR=$__RootfsDir $__CrossDir/$__BuildArch/tizen-build-rootfs.sh else echo "Unsupported target platform." diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index 534f1d19..ec512d01 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -3,6 +3,9 @@ set(CROSS_ROOTFS $ENV{ROOTFS_DIR}) set(TARGET_ARCH_NAME $ENV{TARGET_BUILD_ARCH}) if(EXISTS ${CROSS_ROOTFS}/bin/freebsd-version) set(CMAKE_SYSTEM_NAME FreeBSD) +elseif(EXISTS ${CROSS_ROOTFS}/usr/platform/i86pc) + set(CMAKE_SYSTEM_NAME SunOS) + set(ILLUMOS 1) else() set(CMAKE_SYSTEM_NAME Linux) endif() @@ -34,6 +37,9 @@ elseif(TARGET_ARCH_NAME STREQUAL "x86") elseif (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") set(CMAKE_SYSTEM_PROCESSOR "x86_64") set(triple "x86_64-unknown-freebsd11") +elseif (ILLUMOS) + set(CMAKE_SYSTEM_PROCESSOR "x86_64") + set(TOOLCHAIN "x86_64-illumos") else() message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only armel, arm, arm64 and x86 are supported!") endif() @@ -67,12 +73,43 @@ if("$ENV{__DistroRid}" MATCHES "android.*") # include official NDK toolchain script include(${CROSS_ROOTFS}/../build/cmake/android.toolchain.cmake) -elseif (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") +elseif(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") # we cross-compile by instructing clang set(CMAKE_C_COMPILER_TARGET ${triple}) set(CMAKE_CXX_COMPILER_TARGET ${triple}) set(CMAKE_ASM_COMPILER_TARGET ${triple}) set(CMAKE_SYSROOT "${CROSS_ROOTFS}") +elseif(ILLUMOS) + set(CMAKE_SYSROOT "${CROSS_ROOTFS}") + + include_directories(SYSTEM ${CROSS_ROOTFS}/include) + + set(TOOLSET_PREFIX ${TOOLCHAIN}-) + function(locate_toolchain_exec exec var) + string(TOUPPER ${exec} EXEC_UPPERCASE) + if(NOT "$ENV{CLR_${EXEC_UPPERCASE}}" STREQUAL "") + set(${var} "$ENV{CLR_${EXEC_UPPERCASE}}" PARENT_SCOPE) + return() + endif() + + find_program(EXEC_LOCATION_${exec} + NAMES + "${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}" + "${TOOLSET_PREFIX}${exec}") + + if (EXEC_LOCATION_${exec} STREQUAL "EXEC_LOCATION_${exec}-NOTFOUND") + message(FATAL_ERROR "Unable to find toolchain executable. Name: ${exec}, Prefix: ${TOOLSET_PREFIX}.") + endif() + set(${var} ${EXEC_LOCATION_${exec}} PARENT_SCOPE) + endfunction() + + set(CMAKE_SYSTEM_PREFIX_PATH "${CROSS_ROOTFS}") + + locate_toolchain_exec(gcc CMAKE_C_COMPILER) + locate_toolchain_exec(g++ CMAKE_CXX_COMPILER) + + set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp") + set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp") else() set(CMAKE_SYSROOT "${CROSS_ROOTFS}") @@ -92,11 +129,14 @@ if(TARGET_ARCH_NAME STREQUAL "armel") endif() elseif(TARGET_ARCH_NAME STREQUAL "x86") add_link_options(-m32) +elseif(ILLUMOS) + add_link_options("-L${CROSS_ROOTFS}/lib/amd64") + add_link_options("-L${CROSS_ROOTFS}/usr/amd64/lib") endif() # Specify compile options -if(TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64)$" AND NOT "$ENV{__DistroRid}" MATCHES "android.*") +if((TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64)$" AND NOT "$ENV{__DistroRid}" MATCHES "android.*") OR ILLUMOS) set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN}) @@ -155,7 +195,6 @@ if(TARGET_ARCH_NAME MATCHES "^(arm|armel|x86)$") endif() endif() - set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) diff --git a/eng/common/performance/perfhelixpublish.proj b/eng/common/performance/perfhelixpublish.proj index 1db5e8a8..3f7315b7 100644 --- a/eng/common/performance/perfhelixpublish.proj +++ b/eng/common/performance/perfhelixpublish.proj @@ -113,17 +113,49 @@ $(WorkItemDirectory)\ScenarioCorrelation $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name System.Private.Xml.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root - + $(WorkItemDirectory)\ScenarioCorrelation $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name System.Linq.Expressions.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root - + $(WorkItemDirectory)\ScenarioCorrelation $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name Microsoft.CodeAnalysis.VisualBasic.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root - + $(WorkItemDirectory)\ScenarioCorrelation $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name Microsoft.CodeAnalysis.CSharp.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name System.Private.CoreLib.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single System.Private.Xml.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single System.Linq.Expressions.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single Microsoft.CodeAnalysis.VisualBasic.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single Microsoft.CodeAnalysis.CSharp.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single System.Private.CoreLib.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --composite %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\framework-r2r.dll.rsp --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + \ No newline at end of file diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1 index cc9d059d..cdd1bba4 100644 --- a/eng/common/post-build/sourcelink-validation.ps1 +++ b/eng/common/post-build/sourcelink-validation.ps1 @@ -196,6 +196,8 @@ function ValidateSourceLinkLinks { Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue } + $ValidationFailures = 0 + # Process each NuGet package in parallel Get-ChildItem "$InputPath\*.symbols.nupkg" | ForEach-Object { @@ -209,17 +211,20 @@ function ValidateSourceLinkLinks { } foreach ($Job in @(Get-Job -State 'Completed')) { - Receive-Job -Id $Job.Id + $jobResult = Receive-Job -Id $Job.Id + if ($jobResult -ne '0') { + $ValidationFailures++ + } Remove-Job -Id $Job.Id } } - $ValidationFailures = 0 foreach ($Job in @(Get-Job)) { $jobResult = Wait-Job -Id $Job.Id | Receive-Job if ($jobResult -ne '0') { $ValidationFailures++ } + Remove-Job -Id $Job.Id } if ($ValidationFailures -gt 0) { Write-PipelineTelemetryError -Category 'SourceLink' -Message "$ValidationFailures package(s) failed validation." diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 index e108bf6c..bf7c15e7 100644 --- a/eng/common/post-build/symbols-validation.ps1 +++ b/eng/common/post-build/symbols-validation.ps1 @@ -189,7 +189,11 @@ function CheckSymbolsAvailable { } foreach ($Job in @(Get-Job -State 'Completed')) { - Receive-Job -Id $Job.Id + $jobResult = Wait-Job -Id $Job.Id | Receive-Job + if ($jobResult -ne '0') { + $TotalFailures++ + } + Remove-Job -Id $Job.Id } Write-Host } diff --git a/global.json b/global.json index 9c5826d8..b1fead38 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "5.0.100-preview.6.20266.3" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20280.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20309.1" } } From 03f1e08a99ab80620d2194331a5e48afaac3af2c Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 22 Jun 2020 12:11:21 +0000 Subject: [PATCH 084/161] Update dependencies from https://github.com/dotnet/arcade build 20200616.1 (#624) Microsoft.DotNet.Arcade.Sdk From Version 5.0.0-beta.20309.1 -> To Version 5.0.0-beta.20316.1 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 4 +- eng/common/cross/armel/tizen-build-rootfs.sh | 7 --- eng/common/performance/perfhelixpublish.proj | 4 -- .../templates/post-build/post-build.yml | 46 ++++++++++++++++++- global.json | 4 +- 5 files changed, 48 insertions(+), 17 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index d48bc511..66ebefdb 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 54d096ee50a9bcf7ac6efd16a07d51af651d20a1 + 71b580038fb704df63e03c6b7ae7d2c6a4fdd71d diff --git a/eng/common/cross/armel/tizen-build-rootfs.sh b/eng/common/cross/armel/tizen-build-rootfs.sh index 87c48e78..25a0efa0 100755 --- a/eng/common/cross/armel/tizen-build-rootfs.sh +++ b/eng/common/cross/armel/tizen-build-rootfs.sh @@ -9,13 +9,6 @@ if [[ -z "$ROOTFS_DIR" ]]; then exit 1; fi -# Clean-up (TODO-Cleanup: We may already delete $ROOTFS_DIR at ./cross/build-rootfs.sh.) -# hk0110 -if [ -d "$ROOTFS_DIR" ]; then - umount $ROOTFS_DIR/* - rm -rf $ROOTFS_DIR -fi - TIZEN_TMP_DIR=$ROOTFS_DIR/tizen_tmp mkdir -p $TIZEN_TMP_DIR diff --git a/eng/common/performance/perfhelixpublish.proj b/eng/common/performance/perfhelixpublish.proj index 3f7315b7..47d3ba00 100644 --- a/eng/common/performance/perfhelixpublish.proj +++ b/eng/common/performance/perfhelixpublish.proj @@ -152,10 +152,6 @@ $(WorkItemDirectory)\ScenarioCorrelation $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single System.Private.CoreLib.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root - - $(WorkItemDirectory)\ScenarioCorrelation - $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --composite %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\framework-r2r.dll.rsp --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root - \ No newline at end of file diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 75890b37..514bfaa5 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -48,11 +48,14 @@ parameters: NetEngServicesProdChannelId: 679 Net5Preview5ChannelId: 857 Net5Preview6ChannelId: 1013 - Net5Preview7ChannelId: 1014 + Net5Preview7ChannelId: 1065 NetCoreSDK313xxChannelId: 759 NetCoreSDK313xxInternalChannelId: 760 NetCoreSDK314xxChannelId: 921 NetCoreSDK314xxInternalChannelId: 922 + VS166ChannelId: 1010 + VS167ChannelId: 1011 + VSMasterChannelId: 1012 stages: - ${{ if ne(parameters.inline, 'true') }}: @@ -111,7 +114,7 @@ stages: inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 arguments: -PromoteToChannels "$(TargetChannels)" - -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview5ChannelId}},${{parameters.Net5Preview6ChannelId}},${{parameters.Net5Preview7ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}} + -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview5ChannelId}},${{parameters.Net5Preview6ChannelId}},${{parameters.Net5Preview7ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}}${{parameters.VS167ChannelId}},${{parameters.VSMasterChannelId}} - job: displayName: NuGet Validation @@ -479,3 +482,42 @@ stages: transportFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'VS16_6_Publishing' + channelName: 'VS 16.6' + channelId: ${{ parameters.VS166ChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'VS16_7_Publishing' + channelName: 'VS 16.7' + channelId: ${{ parameters.VS167ChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'VS_Master_Publishing' + channelName: 'VS Master' + channelId: ${{ parameters.VSMasterChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' \ No newline at end of file diff --git a/global.json b/global.json index b1fead38..74627d3d 100644 --- a/global.json +++ b/global.json @@ -1,8 +1,8 @@ { "tools": { - "dotnet": "5.0.100-preview.6.20266.3" + "dotnet": "5.0.100-preview.6.20310.4" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20309.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20316.1" } } From f225c7a58fb63c232505622642ad69c5b32523e2 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2020 12:11:27 +0000 Subject: [PATCH 085/161] Update dependencies from https://github.com/dotnet/arcade build 20200630.3 (#627) Microsoft.DotNet.Arcade.Sdk From Version 5.0.0-beta.20316.1 -> To Version 5.0.0-beta.20330.3 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 4 +- eng/common/cross/arm64/tizen-build-rootfs.sh | 35 ++++ eng/common/cross/arm64/tizen-fetch.sh | 170 ++++++++++++++++++ eng/common/cross/arm64/tizen/tizen.patch | 9 + eng/common/cross/build-rootfs.sh | 4 +- eng/common/cross/toolchain.cmake | 33 +++- eng/common/internal-feed-operations.ps1 | 1 + eng/common/internal-feed-operations.sh | 1 + eng/common/performance/perfhelixpublish.proj | 39 +++- .../post-build/check-channel-consistency.ps1 | 10 ++ eng/common/post-build/symbols-validation.ps1 | 45 +++-- .../templates/post-build/post-build.yml | 4 +- global.json | 2 +- 13 files changed, 331 insertions(+), 26 deletions(-) create mode 100644 eng/common/cross/arm64/tizen-build-rootfs.sh create mode 100644 eng/common/cross/arm64/tizen-fetch.sh create mode 100644 eng/common/cross/arm64/tizen/tizen.patch diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 66ebefdb..7a3cc16e 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 71b580038fb704df63e03c6b7ae7d2c6a4fdd71d + 243cc92161ad44c2a07464425892daee19121c99 diff --git a/eng/common/cross/arm64/tizen-build-rootfs.sh b/eng/common/cross/arm64/tizen-build-rootfs.sh new file mode 100644 index 00000000..13bfddb5 --- /dev/null +++ b/eng/common/cross/arm64/tizen-build-rootfs.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -e + +__CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +__TIZEN_CROSSDIR="$__CrossDir/tizen" + +if [[ -z "$ROOTFS_DIR" ]]; then + echo "ROOTFS_DIR is not defined." + exit 1; +fi + +TIZEN_TMP_DIR=$ROOTFS_DIR/tizen_tmp +mkdir -p $TIZEN_TMP_DIR + +# Download files +echo ">>Start downloading files" +VERBOSE=1 $__CrossDir/tizen-fetch.sh $TIZEN_TMP_DIR +echo "<>Start constructing Tizen rootfs" +TIZEN_RPM_FILES=`ls $TIZEN_TMP_DIR/*.rpm` +cd $ROOTFS_DIR +for f in $TIZEN_RPM_FILES; do + rpm2cpio $f | cpio -idm --quiet +done +echo "<>Start configuring Tizen rootfs" +ln -sfn asm-arm64 ./usr/include/asm +patch -p1 < $__TIZEN_CROSSDIR/tizen.patch +echo "</dev/null; then + VERBOSE=0 +fi + +Log() +{ + if [ $VERBOSE -ge $1 ]; then + echo ${@:2} + fi +} + +Inform() +{ + Log 1 -e "\x1B[0;34m$@\x1B[m" +} + +Debug() +{ + Log 2 -e "\x1B[0;32m$@\x1B[m" +} + +Error() +{ + >&2 Log 0 -e "\x1B[0;31m$@\x1B[m" +} + +Fetch() +{ + URL=$1 + FILE=$2 + PROGRESS=$3 + if [ $VERBOSE -ge 1 ] && [ $PROGRESS ]; then + CURL_OPT="--progress-bar" + else + CURL_OPT="--silent" + fi + curl $CURL_OPT $URL > $FILE +} + +hash curl 2> /dev/null || { Error "Require 'curl' Aborting."; exit 1; } +hash xmllint 2> /dev/null || { Error "Require 'xmllint' Aborting."; exit 1; } +hash sha256sum 2> /dev/null || { Error "Require 'sha256sum' Aborting."; exit 1; } + +TMPDIR=$1 +if [ ! -d $TMPDIR ]; then + TMPDIR=./tizen_tmp + Debug "Create temporary directory : $TMPDIR" + mkdir -p $TMPDIR +fi + +TIZEN_URL=http://download.tizen.org/snapshots/tizen/ +BUILD_XML=build.xml +REPOMD_XML=repomd.xml +PRIMARY_XML=primary.xml +TARGET_URL="http://__not_initialized" + +Xpath_get() +{ + XPATH_RESULT='' + XPATH=$1 + XML_FILE=$2 + RESULT=$(xmllint --xpath $XPATH $XML_FILE) + if [[ -z ${RESULT// } ]]; then + Error "Can not find target from $XML_FILE" + Debug "Xpath = $XPATH" + exit 1 + fi + XPATH_RESULT=$RESULT +} + +fetch_tizen_pkgs_init() +{ + TARGET=$1 + PROFILE=$2 + Debug "Initialize TARGET=$TARGET, PROFILE=$PROFILE" + + TMP_PKG_DIR=$TMPDIR/tizen_${PROFILE}_pkgs + if [ -d $TMP_PKG_DIR ]; then rm -rf $TMP_PKG_DIR; fi + mkdir -p $TMP_PKG_DIR + + PKG_URL=$TIZEN_URL/$PROFILE/latest + + BUILD_XML_URL=$PKG_URL/$BUILD_XML + TMP_BUILD=$TMP_PKG_DIR/$BUILD_XML + TMP_REPOMD=$TMP_PKG_DIR/$REPOMD_XML + TMP_PRIMARY=$TMP_PKG_DIR/$PRIMARY_XML + TMP_PRIMARYGZ=${TMP_PRIMARY}.gz + + Fetch $BUILD_XML_URL $TMP_BUILD + + Debug "fetch $BUILD_XML_URL to $TMP_BUILD" + + TARGET_XPATH="//build/buildtargets/buildtarget[@name=\"$TARGET\"]/repo[@type=\"binary\"]/text()" + Xpath_get $TARGET_XPATH $TMP_BUILD + TARGET_PATH=$XPATH_RESULT + TARGET_URL=$PKG_URL/$TARGET_PATH + + REPOMD_URL=$TARGET_URL/repodata/repomd.xml + PRIMARY_XPATH='string(//*[local-name()="data"][@type="primary"]/*[local-name()="location"]/@href)' + + Fetch $REPOMD_URL $TMP_REPOMD + + Debug "fetch $REPOMD_URL to $TMP_REPOMD" + + Xpath_get $PRIMARY_XPATH $TMP_REPOMD + PRIMARY_XML_PATH=$XPATH_RESULT + PRIMARY_URL=$TARGET_URL/$PRIMARY_XML_PATH + + Fetch $PRIMARY_URL $TMP_PRIMARYGZ + + Debug "fetch $PRIMARY_URL to $TMP_PRIMARYGZ" + + gunzip $TMP_PRIMARYGZ + + Debug "unzip $TMP_PRIMARYGZ to $TMP_PRIMARY" +} + +fetch_tizen_pkgs() +{ + ARCH=$1 + PACKAGE_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="location"]/@href)' + + PACKAGE_CHECKSUM_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="checksum"]/text())' + + for pkg in ${@:2} + do + Inform "Fetching... $pkg" + XPATH=${PACKAGE_XPATH_TPL/_PKG_/$pkg} + XPATH=${XPATH/_ARCH_/$ARCH} + Xpath_get $XPATH $TMP_PRIMARY + PKG_PATH=$XPATH_RESULT + + XPATH=${PACKAGE_CHECKSUM_XPATH_TPL/_PKG_/$pkg} + XPATH=${XPATH/_ARCH_/$ARCH} + Xpath_get $XPATH $TMP_PRIMARY + CHECKSUM=$XPATH_RESULT + + PKG_URL=$TARGET_URL/$PKG_PATH + PKG_FILE=$(basename $PKG_PATH) + PKG_PATH=$TMPDIR/$PKG_FILE + + Debug "Download $PKG_URL to $PKG_PATH" + Fetch $PKG_URL $PKG_PATH true + + echo "$CHECKSUM $PKG_PATH" | sha256sum -c - > /dev/null + if [ $? -ne 0 ]; then + Error "Fail to fetch $PKG_URL to $PKG_PATH" + Debug "Checksum = $CHECKSUM" + exit 1 + fi + done +} + +Inform "Initialize arm base" +fetch_tizen_pkgs_init standard base +Inform "fetch common packages" +fetch_tizen_pkgs aarch64 gcc glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel +Inform "fetch coreclr packages" +fetch_tizen_pkgs aarch64 lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu +Inform "fetch corefx packages" +fetch_tizen_pkgs aarch64 libcom_err libcom_err-devel zlib zlib-devel libopenssl libopenssl1.1-devel krb5 krb5-devel + +Inform "Initialize standard unified" +fetch_tizen_pkgs_init standard unified +Inform "fetch corefx packages" +fetch_tizen_pkgs aarch64 gssdp gssdp-devel tizen-release + diff --git a/eng/common/cross/arm64/tizen/tizen.patch b/eng/common/cross/arm64/tizen/tizen.patch new file mode 100644 index 00000000..af7c8be0 --- /dev/null +++ b/eng/common/cross/arm64/tizen/tizen.patch @@ -0,0 +1,9 @@ +diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so +--- a/usr/lib64/libc.so 2016-12-30 23:00:08.284951863 +0900 ++++ b/usr/lib64/libc.so 2016-12-30 23:00:32.140951815 +0900 +@@ -2,4 +2,4 @@ + Use the shared library, but some functions are only in + the static library, so try that secondarily. */ + OUTPUT_FORMAT(elf64-littleaarch64) +-GROUP ( /lib64/libc.so.6 /usr/lib64/libc_nonshared.a AS_NEEDED ( /lib/ld-linux-aarch64.so.1 ) ) ++GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux-aarch64.so.1 ) ) diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index d6ec94b7..ffdff385 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -167,8 +167,8 @@ while :; do __LLDB_Package="liblldb-6.0-dev" ;; tizen) - if [ "$__BuildArch" != "armel" ]; then - echo "Tizen is available only for armel." + if [ "$__BuildArch" != "armel" ] && [ "$__BuildArch" != "arm64" ]; then + echo "Tizen is available only for armel and arm64." usage; exit 1; fi diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index ec512d01..88a758af 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -31,6 +31,9 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm64") else() set(TOOLCHAIN "aarch64-linux-gnu") endif() + if("$ENV{__DistroRid}" MATCHES "tizen.*") + set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu/9.2.0") + endif() elseif(TARGET_ARCH_NAME STREQUAL "x86") set(CMAKE_SYSTEM_PROCESSOR i686) set(TOOLCHAIN "i686-linux-gnu") @@ -49,11 +52,15 @@ if(DEFINED ENV{TOOLCHAIN}) endif() # Specify include paths -if(TARGET_ARCH_NAME STREQUAL "armel") - if(DEFINED TIZEN_TOOLCHAIN) +if(DEFINED TIZEN_TOOLCHAIN) + if(TARGET_ARCH_NAME STREQUAL "armel") include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/) include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/armv7l-tizen-linux-gnueabi) endif() + if(TARGET_ARCH_NAME STREQUAL "arm64") + include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/) + include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/aarch64-tizen-linux-gnu) + endif() endif() if("$ENV{__DistroRid}" MATCHES "android.*") @@ -127,6 +134,17 @@ if(TARGET_ARCH_NAME STREQUAL "armel") add_link_options("-L${CROSS_ROOTFS}/usr/lib") add_link_options("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") endif() +elseif(TARGET_ARCH_NAME STREQUAL "arm64") + if(DEFINED TIZEN_TOOLCHAIN) # For Tizen only + add_link_options("-B${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") + add_link_options("-L${CROSS_ROOTFS}/lib64") + add_link_options("-L${CROSS_ROOTFS}/usr/lib64") + add_link_options("-L${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") + + add_link_options("-Wl,--rpath-link=${CROSS_ROOTFS}/lib64") + add_link_options("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64") + add_link_options("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") + endif() elseif(TARGET_ARCH_NAME STREQUAL "x86") add_link_options(-m32) elseif(ILLUMOS) @@ -157,16 +175,19 @@ if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$") if(TARGET_ARCH_NAME STREQUAL "armel") add_compile_options(-mfloat-abi=softfp) - if(DEFINED TIZEN_TOOLCHAIN) - add_compile_options(-Wno-deprecated-declarations) # compile-time option - add_compile_options(-D__extern_always_inline=inline) # compile-time option - endif() endif() elseif(TARGET_ARCH_NAME STREQUAL "x86") add_compile_options(-m32) add_compile_options(-Wno-error=unused-command-line-argument) endif() +if(DEFINED TIZEN_TOOLCHAIN) + if(TARGET_ARCH_NAME MATCHES "^(armel|arm64)$") + add_compile_options(-Wno-deprecated-declarations) # compile-time option + add_compile_options(-D__extern_always_inline=inline) # compile-time option + endif() +endif() + # Set LLDB include and library paths for builds that need lldb. if(TARGET_ARCH_NAME MATCHES "^(arm|armel|x86)$") if(TARGET_ARCH_NAME STREQUAL "x86") diff --git a/eng/common/internal-feed-operations.ps1 b/eng/common/internal-feed-operations.ps1 index db0baac9..b8f6529f 100644 --- a/eng/common/internal-feed-operations.ps1 +++ b/eng/common/internal-feed-operations.ps1 @@ -63,6 +63,7 @@ function SetupCredProvider { } if (($endpoints | Measure-Object).Count -gt 0) { + # [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Endpoint code example with no real credentials.")] # Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}' $endpointCredentials = @{endpointCredentials=$endpoints} | ConvertTo-Json -Compress diff --git a/eng/common/internal-feed-operations.sh b/eng/common/internal-feed-operations.sh index 5941ea28..9ed225e7 100755 --- a/eng/common/internal-feed-operations.sh +++ b/eng/common/internal-feed-operations.sh @@ -62,6 +62,7 @@ function SetupCredProvider { endpoints+=']' if [ ${#endpoints} -gt 2 ]; then + # [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Endpoint code example with no real credentials.")] # Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}' local endpointCredentials="{\"endpointCredentials\": "$endpoints"}" diff --git a/eng/common/performance/perfhelixpublish.proj b/eng/common/performance/perfhelixpublish.proj index 47d3ba00..272366da 100644 --- a/eng/common/performance/perfhelixpublish.proj +++ b/eng/common/performance/perfhelixpublish.proj @@ -63,6 +63,11 @@ $(WorkItemCommand) $(CliArguments) + + + 2:30 + 0:15 + @@ -71,7 +76,7 @@ - 5 + 30 @@ -79,6 +84,31 @@ + + + + + + + + + + + + + + + + + + + + + + + + + @@ -94,7 +124,7 @@ $(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" $(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" $(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults);$(FinalCommand) - 4:00 + $(WorkItemTimeout) @@ -152,6 +182,11 @@ $(WorkItemDirectory)\ScenarioCorrelation $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single System.Private.CoreLib.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --composite %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\framework-r2r.dll.rsp --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + 1:00 + \ No newline at end of file diff --git a/eng/common/post-build/check-channel-consistency.ps1 b/eng/common/post-build/check-channel-consistency.ps1 index 38abc539..63f3464c 100644 --- a/eng/common/post-build/check-channel-consistency.ps1 +++ b/eng/common/post-build/check-channel-consistency.ps1 @@ -15,12 +15,22 @@ try { # is available in YAML $PromoteToChannelsIds = $PromoteToChannels -split "\D" | Where-Object { $_ } + $hasErrors = $false + foreach ($id in $PromoteToChannelsIds) { if (($id -ne 0) -and ($id -notin $AvailableChannelIds)) { Write-PipelineTaskError -Message "Channel $id is not present in the post-build YAML configuration! This is an error scenario. Please contact @dnceng." + $hasErrors = $true } } + # The `Write-PipelineTaskError` doesn't error the script and we might report several errors + # in the previous lines. The check below makes sure that we return an error state from the + # script if we reported any validation error + if ($hasErrors) { + ExitWithExitCode 1 + } + Write-Host 'done.' } catch { diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 index bf7c15e7..495428ea 100644 --- a/eng/common/post-build/symbols-validation.ps1 +++ b/eng/common/post-build/symbols-validation.ps1 @@ -24,7 +24,7 @@ $CountMissingSymbols = { # Ensure input file exist if (!(Test-Path $PackagePath)) { Write-PipelineTaskError "Input file does not exist: $PackagePath" - return 1 + return -2 } # Extensions for which we'll look for symbols @@ -44,7 +44,10 @@ $CountMissingSymbols = { catch { Write-Host "Something went wrong extracting $PackagePath" Write-Host $_ - return -1 + return [pscustomobject]@{ + result = -1 + packagePath = $PackagePath + } } Get-ChildItem -Recurse $ExtractPath | @@ -146,7 +149,24 @@ $CountMissingSymbols = { Pop-Location - return $MissingSymbols + return [pscustomobject]@{ + result = $MissingSymbols + packagePath = $PackagePath + } +} + +function CheckJobResult( + $result, + $packagePath, + [ref]$DupedSymbols, + [ref]$TotalFailures) { + if ($result -eq '-1') { + Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$packagePath has duplicated symbol files" + $DupedSymbols.Value++ + } + elseif ($jobResult.result -ne '0') { + $TotalFailures.Value++ + } } function CheckSymbolsAvailable { @@ -155,6 +175,7 @@ function CheckSymbolsAvailable { } $TotalFailures = 0 + $DupedSymbols = 0 Get-ChildItem "$InputPath\*.nupkg" | ForEach-Object { @@ -190,9 +211,7 @@ function CheckSymbolsAvailable { foreach ($Job in @(Get-Job -State 'Completed')) { $jobResult = Wait-Job -Id $Job.Id | Receive-Job - if ($jobResult -ne '0') { - $TotalFailures++ - } + CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$DupedSymbols) ([ref]$TotalFailures) Remove-Job -Id $Job.Id } Write-Host @@ -200,14 +219,18 @@ function CheckSymbolsAvailable { foreach ($Job in @(Get-Job)) { $jobResult = Wait-Job -Id $Job.Id | Receive-Job + CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$DupedSymbols) ([ref]$TotalFailures) + } - if ($jobResult -ne '0') { - $TotalFailures++ + if ($TotalFailures -gt 0 -or $DupedSymbols -gt 0) { + if ($TotalFailures -gt 0) { + Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Symbols missing for $TotalFailures packages" } - } - if ($TotalFailures -gt 0) { - Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Symbols missing for $TotalFailures packages" + if ($DupedSymbols -gt 0) { + Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$DupedSymbols packages had duplicated symbol files" + } + ExitWithExitCode 1 } else { diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 514bfaa5..07fc2e98 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -114,7 +114,7 @@ stages: inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 arguments: -PromoteToChannels "$(TargetChannels)" - -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview5ChannelId}},${{parameters.Net5Preview6ChannelId}},${{parameters.Net5Preview7ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}}${{parameters.VS167ChannelId}},${{parameters.VSMasterChannelId}} + -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview5ChannelId}},${{parameters.Net5Preview6ChannelId}},${{parameters.Net5Preview7ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VSMasterChannelId}} - job: displayName: NuGet Validation @@ -520,4 +520,4 @@ stages: channelId: ${{ parameters.VSMasterChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' \ No newline at end of file + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' diff --git a/global.json b/global.json index 74627d3d..c9bb4b4b 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "5.0.100-preview.6.20310.4" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20316.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20330.3" } } From 201097984ee6cf228af0f78d84e9c7ff04377ec9 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 16 Jul 2020 00:00:58 +0000 Subject: [PATCH 086/161] Update dependencies from https://github.com/dotnet/arcade build 20200714.3 (#630) Microsoft.DotNet.Arcade.Sdk From Version 5.0.0-beta.20330.3 -> To Version 5.0.0-beta.20364.3 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 4 +- eng/common/SetupNugetSources.ps1 | 12 ++-- eng/common/SetupNugetSources.sh | 41 +++++++------ eng/common/performance/performance-setup.ps1 | 5 ++ eng/common/performance/performance-setup.sh | 4 ++ eng/common/post-build/symbols-validation.ps1 | 2 +- eng/common/templates/job/job.yml | 12 ++++ .../channels/generic-internal-channel.yml | 4 ++ .../templates/post-build/post-build.yml | 60 ++++++++++++------- eng/common/tools.ps1 | 2 +- global.json | 2 +- 11 files changed, 93 insertions(+), 55 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 7a3cc16e..2e53d45f 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 243cc92161ad44c2a07464425892daee19121c99 + ff5d4b6c8dbdaeacb6e6159d3f8185118dffd915 diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index c3c473eb..dc221781 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -128,16 +128,16 @@ $userName = "dn-bot" # Insert credential nodes for Maestro's private feeds InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -Password $Password -$dotnet3Source = $sources.SelectSingleNode("add[@key='dotnet3']") -if ($dotnet3Source -ne $null) { - AddPackageSource -Sources $sources -SourceName "dotnet3-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password - AddPackageSource -Sources $sources -SourceName "dotnet3-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password -} - $dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']") if ($dotnet31Source -ne $null) { AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password } +$dotnet5Source = $sources.SelectSingleNode("add[@key='dotnet5']") +if ($dotnet5Source -ne $null) { + AddPackageSource -Sources $sources -SourceName "dotnet5-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet5-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password +} + $doc.Save($filename) diff --git a/eng/common/SetupNugetSources.sh b/eng/common/SetupNugetSources.sh index 7d6fef27..f33e3771 100644 --- a/eng/common/SetupNugetSources.sh +++ b/eng/common/SetupNugetSources.sh @@ -79,53 +79,52 @@ fi PackageSources=() -# Ensure dotnet3-internal and dotnet3-internal-transport are in the packageSources if the public dotnet3 feeds are present -grep -i "" $ConfigFile + grep -i "" + PackageSourceTemplate="${TB}" sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile fi - PackageSources+=('dotnet3-internal') + PackageSources+=('dotnet3.1-internal') - grep -i "" $ConfigFile if [ "$?" != "0" ]; then - echo "Adding dotnet3-internal-transport to the packageSources." + echo "Adding dotnet3.1-internal-transport to the packageSources." PackageSourcesNodeFooter="" - PackageSourceTemplate="${TB}" + PackageSourceTemplate="${TB}" sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile fi - PackageSources+=('dotnet3-internal-transport') + PackageSources+=('dotnet3.1-internal-transport') fi -# Ensure dotnet3.1-internal and dotnet3.1-internal-transport are in the packageSources if the public dotnet3.1 feeds are present -grep -i "" + PackageSourceTemplate="${TB}" sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile fi - PackageSources+=('dotnet3.1-internal') + PackageSources+=('dotnet5-internal') - grep -i "" $ConfigFile + grep -i "" $ConfigFile if [ "$?" != "0" ]; then - echo "Adding dotnet3.1-internal-transport to the packageSources." + echo "Adding dotnet5-internal-transport to the packageSources." PackageSourcesNodeFooter="" - PackageSourceTemplate="${TB}" + PackageSourceTemplate="${TB}" sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile fi - PackageSources+=('dotnet3.1-internal-transport') + PackageSources+=('dotnet5-internal-transport') fi # I want things split line by line diff --git a/eng/common/performance/performance-setup.ps1 b/eng/common/performance/performance-setup.ps1 index 31a99e49..656c0bd9 100644 --- a/eng/common/performance/performance-setup.ps1 +++ b/eng/common/performance/performance-setup.ps1 @@ -54,6 +54,11 @@ if ($Internal) { $HelixSourcePrefix = "official" } +if($MonoInterpreter) +{ + $ExtraBenchmarkDotNetArguments = "--category-exclusion-filter NoInterpreter" +} + if($MonoDotnet -ne "") { $Configurations += " LLVM=$LLVM MonoInterpreter=$MonoInterpreter MonoAOT=$MonoAOT" diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index 9409e4d8..c87cbf0f 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -191,6 +191,10 @@ if [[ "$mono_dotnet" != "" ]]; then configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot" fi +if [[ "$monointerpreter" == "true" ]]; then + extra_benchmark_dotnet_arguments="--category-exclusion-filter NoInterpreter" +fi + common_setup_arguments="--channel master --queue $queue --build-number $build_number --build-configs $configurations --architecture $architecture" setup_arguments="--repository https://github.com/$repository --branch $branch --get-perf-hash --commit-sha $commit_sha $common_setup_arguments" diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 index 495428ea..a36fa6f2 100644 --- a/eng/common/post-build/symbols-validation.ps1 +++ b/eng/common/post-build/symbols-validation.ps1 @@ -144,7 +144,7 @@ $CountMissingSymbols = { if ($MissingSymbols -ne 0) { - Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $MissingSymbols modules in the package $FileName" + Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $MissingSymbols modules in the package $PackagePath" } Pop-Location diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index c08bb416..e78ed9a1 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -65,6 +65,9 @@ jobs: - ${{ if ne(parameters.enableTelemetry, 'false') }}: - name: DOTNET_CLI_TELEMETRY_PROFILE value: '$(Build.Repository.Uri)' + - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}: + - name: EnableRichCodeNavigation + value: 'true' - ${{ each variable in parameters.variables }}: # handle name-value variable syntax # example: @@ -124,6 +127,15 @@ jobs: - ${{ each step in parameters.steps }}: - ${{ step }} + - ${{ if eq(parameters.enableRichCodeNavigation, true) }}: + - task: RichCodeNavIndexer@0 + displayName: RichCodeNav Upload + inputs: + languages: 'csharp' + environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'prod') }} + richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin + continueOnError: true + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - task: MicroBuildCleanup@1 diff --git a/eng/common/templates/post-build/channels/generic-internal-channel.yml b/eng/common/templates/post-build/channels/generic-internal-channel.yml index 258ba4b7..12db36eb 100644 --- a/eng/common/templates/post-build/channels/generic-internal-channel.yml +++ b/eng/common/templates/post-build/channels/generic-internal-channel.yml @@ -35,6 +35,10 @@ stages: pool: vmImage: 'windows-2019' steps: + # This is necessary whenever we want to publish/restore to an AzDO private feed + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + - task: DownloadBuildArtifacts@0 displayName: Download Build Assets continueOnError: true diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 07fc2e98..0e79a546 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -46,15 +46,16 @@ parameters: NETCoreExperimentalChannelId: 562 NetEngServicesIntChannelId: 678 NetEngServicesProdChannelId: 679 - Net5Preview5ChannelId: 857 - Net5Preview6ChannelId: 1013 Net5Preview7ChannelId: 1065 + Net5Preview8ChannelId: 1155 + Net5RC1ChannelId: 1157 NetCoreSDK313xxChannelId: 759 NetCoreSDK313xxInternalChannelId: 760 NetCoreSDK314xxChannelId: 921 NetCoreSDK314xxInternalChannelId: 922 VS166ChannelId: 1010 VS167ChannelId: 1011 + VS168ChannelId: 1154 VSMasterChannelId: 1012 stages: @@ -114,7 +115,7 @@ stages: inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 arguments: -PromoteToChannels "$(TargetChannels)" - -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview5ChannelId}},${{parameters.Net5Preview6ChannelId}},${{parameters.Net5Preview7ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VSMasterChannelId}} + -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview7ChannelId}},${{parameters.Net5Preview8ChannelId}},${{parameters.Net5RC1ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}} - job: displayName: NuGet Validation @@ -275,41 +276,41 @@ stages: dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net5_Preview5_Publish' - channelName: '.NET 5 Preview 5' - akaMSChannelName: 'net5/preview5' - channelId: ${{ parameters.Net5Preview5ChannelId }} + stageName: 'Net5_Preview7_Publish' + channelName: '.NET 5 Preview 7' + akaMSChannelName: 'net5/preview7' + channelId: ${{ parameters.Net5Preview7ChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' - - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml parameters: artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net5_Preview6_Publish' - channelName: '.NET 5 Preview 6' - akaMSChannelName: 'net5/preview6' - channelId: ${{ parameters.Net5Preview6ChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' + stageName: 'Net5_Preview8_Publish' + channelName: '.NET 5 Preview 8' + akaMSChannelName: 'net5/preview8' + channelId: ${{ parameters.Net5Preview8ChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-symbols/nuget/v3/index.json' - - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml parameters: artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net5_Preview7_Publish' - channelName: '.NET 5 Preview 7' - akaMSChannelName: 'net5/preview7' - channelId: ${{ parameters.Net5Preview7ChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' + stageName: 'Net5_RC1_Publish' + channelName: '.NET 5 RC 1' + akaMSChannelName: 'net5/rc1' + channelId: ${{ parameters.Net5RC1ChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-symbols/nuget/v3/index.json' - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: @@ -509,6 +510,19 @@ stages: shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'VS16_8_Publishing' + channelName: 'VS 16.8' + channelId: ${{ parameters.VS168ChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index d68684e2..6f8bc411 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -385,7 +385,7 @@ function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { Create-Directory $packageDir Write-Host "Downloading $packageName $packageVersion" $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit - Invoke-WebRequest "https://dotnet.myget.org/F/roslyn-tools/api/v2/package/$packageName/$packageVersion/" -OutFile $packagePath + Invoke-WebRequest "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/flat2/$packageName/$packageVersion/$packageName.$packageVersion.nupkg" -OutFile $packagePath Unzip $packagePath $packageDir } diff --git a/global.json b/global.json index c9bb4b4b..ea83bbe2 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "5.0.100-preview.6.20310.4" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20330.3" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20364.3" } } From 0815575f4f226973dac0d3b3f9d0229dca8ac4f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Matou=C5=A1ek?= Date: Sun, 19 Jul 2020 11:19:19 -0700 Subject: [PATCH 087/161] Target netcoreapp3.1 (#631) * Update .NET Core target to netcoreapp3.1 * Fix nullability warnings --- src/Common/GetSourceLinkUrlGitTask.cs | 2 +- src/Common/NullableAttributes.cs | 6 +++++- src/Common/SequenceComparer.cs | 9 +++++++-- src/Common/UriUtilities.cs | 4 ++-- .../Microsoft.Build.StandardCI.csproj | 2 +- .../GitConfigTests.cs | 4 ++-- .../Microsoft.Build.Tasks.Git.UnitTests.csproj | 2 +- .../GitDataReader/GitConfig.Reader.cs | 4 ++-- .../GitDataReader/GitIgnore.cs | 4 ++-- .../GitDataReader/GitRepository.cs | 11 ++++++----- .../GitDataReader/GitVariableName.cs | 2 +- .../Microsoft.Build.Tasks.Git.csproj | 2 +- .../build/Microsoft.Build.Tasks.Git.props | 2 +- ...Link.AzureDevOpsServer.Git.UnitTests.csproj | 2 +- ...oft.SourceLink.AzureDevOpsServer.Git.csproj | 2 +- ...ft.SourceLink.AzureDevOpsServer.Git.targets | 2 +- ....SourceLink.AzureRepos.Git.UnitTests.csproj | 2 +- .../Microsoft.SourceLink.AzureRepos.Git.csproj | 2 +- ...Microsoft.SourceLink.AzureRepos.Git.targets | 2 +- ...icrosoft.SourceLink.AzureRepos.Tfvc.targets | 2 +- ...t.SourceLink.Bitbucket.Git.UnitTests.csproj | 2 +- .../Microsoft.SourceLink.Bitbucket.Git.csproj | 2 +- .../Microsoft.SourceLink.Bitbucket.Git.targets | 2 +- ...icrosoft.SourceLink.Common.UnitTests.csproj | 2 +- .../Microsoft.SourceLink.Common.csproj | 2 +- .../build/Microsoft.SourceLink.Common.props | 2 +- ...soft.SourceLink.Git.IntegrationTests.csproj | 2 +- ...icrosoft.SourceLink.GitHub.UnitTests.csproj | 2 +- .../Microsoft.SourceLink.GitHub.csproj | 2 +- .../build/Microsoft.SourceLink.GitHub.targets | 2 +- ...icrosoft.SourceLink.GitLab.UnitTests.csproj | 2 +- .../Microsoft.SourceLink.GitLab.csproj | 2 +- .../build/Microsoft.SourceLink.GitLab.targets | 2 +- ...icrosoft.SourceLink.GitWeb.UnitTests.csproj | 2 +- .../Microsoft.SourceLink.GitWeb.csproj | 2 +- .../build/Microsoft.SourceLink.GitWeb.targets | 2 +- src/TestUtilities/AssertEx.cs | 11 ++++++----- src/TestUtilities/ConditionalFactAttribute.cs | 4 ++-- .../DotNetSdk/DotNetSdkTestBase.cs | 18 +++++++++--------- src/TestUtilities/TestUtilities.csproj | 2 +- 40 files changed, 73 insertions(+), 62 deletions(-) diff --git a/src/Common/GetSourceLinkUrlGitTask.cs b/src/Common/GetSourceLinkUrlGitTask.cs index b49cac57..2962db3c 100644 --- a/src/Common/GetSourceLinkUrlGitTask.cs +++ b/src/Common/GetSourceLinkUrlGitTask.cs @@ -164,7 +164,7 @@ static bool isValidContentUri(Uri uri) continue; } - Uri contentUri; + Uri? contentUri; string contentUrl = item.GetMetadata(ContentUrlMetadataName); bool hasDefaultContentUri = string.IsNullOrEmpty(contentUrl); if (hasDefaultContentUri) diff --git a/src/Common/NullableAttributes.cs b/src/Common/NullableAttributes.cs index 8db4ab49..8bce5d69 100644 --- a/src/Common/NullableAttributes.cs +++ b/src/Common/NullableAttributes.cs @@ -3,6 +3,8 @@ using System.Diagnostics; using System.Diagnostics.CodeAnalysis; +#if NETFRAMEWORK + namespace System.Diagnostics.CodeAnalysis { /// Specifies that null is allowed as an input even if the corresponding type disallows it. @@ -83,6 +85,8 @@ internal sealed class DoesNotReturnIfAttribute : Attribute } } +#endif + namespace System { internal static class NullableString @@ -107,4 +111,4 @@ public static void Assert([DoesNotReturnIf(false)]bool b, string message) => Debug.Assert(b, message); } -} \ No newline at end of file +} diff --git a/src/Common/SequenceComparer.cs b/src/Common/SequenceComparer.cs index da909ad1..c280573c 100644 --- a/src/Common/SequenceComparer.cs +++ b/src/Common/SequenceComparer.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; namespace Microsoft.Build.Tasks.SourceControl { @@ -33,7 +34,11 @@ public int Compare(T[] left, int leftLength, T[] right, int rightLength) return leftLength.CompareTo(rightLength); } - public int Compare(T[] left, T[] right) - => Compare(left, left.Length, right, right.Length); + public int Compare([AllowNull] T[] left, [AllowNull] T[] right) + { + left ??= Array.Empty(); + right ??= Array.Empty(); + return Compare(left, left.Length, right, right.Length); + } } } diff --git a/src/Common/UriUtilities.cs b/src/Common/UriUtilities.cs index 6c6d6be0..b490faf7 100644 --- a/src/Common/UriUtilities.cs +++ b/src/Common/UriUtilities.cs @@ -8,7 +8,7 @@ namespace Microsoft.Build.Tasks.SourceControl { internal static class UriUtilities { - public static bool TryParseAuthority(string value, out Uri uri) + public static bool TryParseAuthority(string value, [NotNullWhen(true)] out Uri? uri) => Uri.TryCreate("unknown://" + value, UriKind.Absolute, out uri) && uri.Scheme == "unknown" && uri.Host != "" && uri.UserInfo == "" && uri.PathAndQuery == "/"; @@ -36,7 +36,7 @@ public static bool UrlStartsWith(string url, string prefix) return url.StartsWith(prefix, StringComparison.Ordinal); } - public static bool TrySplitRelativeUrl(string relativeUrl, [NotNullWhen(true)]out string[]? parts) + public static bool TrySplitRelativeUrl(string relativeUrl, [NotNullWhen(true)] out string[]? parts) { // required leading slash: if (relativeUrl.Length == 0 || relativeUrl == "/") diff --git a/src/Microsoft.Build.StandardCI/Microsoft.Build.StandardCI.csproj b/src/Microsoft.Build.StandardCI/Microsoft.Build.StandardCI.csproj index e04370c0..79031b5b 100644 --- a/src/Microsoft.Build.StandardCI/Microsoft.Build.StandardCI.csproj +++ b/src/Microsoft.Build.StandardCI/Microsoft.Build.StandardCI.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp2.0 + net461;netcoreapp3.1 true true true diff --git a/src/Microsoft.Build.Tasks.Git.UnitTests/GitConfigTests.cs b/src/Microsoft.Build.Tasks.Git.UnitTests/GitConfigTests.cs index 6959d605..b47b2abb 100644 --- a/src/Microsoft.Build.Tasks.Git.UnitTests/GitConfigTests.cs +++ b/src/Microsoft.Build.Tasks.Git.UnitTests/GitConfigTests.cs @@ -61,7 +61,7 @@ public void ConditionalInclude() TextReader openFile(string path) { - Assert.Equal(gitDirectory, PathUtils.EnsureTrailingSlash(PathUtils.ToPosixPath(Path.GetDirectoryName(path)))); + Assert.Equal(gitDirectory, PathUtils.EnsureTrailingSlash(PathUtils.ToPosixPath(Path.GetDirectoryName(path)!))); return new StringReader(Path.GetFileName(path) switch { @@ -182,7 +182,7 @@ public void IncludeRecursion() TextReader openFile(string path) { - Assert.Equal(gitDirectory, PathUtils.EnsureTrailingSlash(PathUtils.ToPosixPath(Path.GetDirectoryName(path)))); + Assert.Equal(gitDirectory, PathUtils.EnsureTrailingSlash(PathUtils.ToPosixPath(Path.GetDirectoryName(path)!))); return new StringReader(Path.GetFileName(path) switch { diff --git a/src/Microsoft.Build.Tasks.Git.UnitTests/Microsoft.Build.Tasks.Git.UnitTests.csproj b/src/Microsoft.Build.Tasks.Git.UnitTests/Microsoft.Build.Tasks.Git.UnitTests.csproj index 5cd629ab..038b1028 100644 --- a/src/Microsoft.Build.Tasks.Git.UnitTests/Microsoft.Build.Tasks.Git.UnitTests.csproj +++ b/src/Microsoft.Build.Tasks.Git.UnitTests/Microsoft.Build.Tasks.Git.UnitTests.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp2.0 + net461;netcoreapp3.1 diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitConfig.Reader.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitConfig.Reader.cs index 34c28916..e01f1954 100644 --- a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitConfig.Reader.cs +++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitConfig.Reader.cs @@ -243,7 +243,7 @@ private string NormalizeRelativePath(string relativePath, string basePath, GitVa } else { - root = Path.GetDirectoryName(basePath); + root = Path.GetDirectoryName(basePath) ?? ""; } try @@ -294,7 +294,7 @@ private bool IsIncludePath(GitVariableName key, string configFilePath) if (pattern.Length >= 2 && pattern[0] == '.' && pattern[1] == '/') { // leading './' is substituted with the path to the directory containing the current config file. - pattern = PathUtils.CombinePosixPaths(PathUtils.ToPosixPath(Path.GetDirectoryName(configFilePath)), pattern.Substring(2)); + pattern = PathUtils.CombinePosixPaths(PathUtils.ToPosixPath(Path.GetDirectoryName(configFilePath)!), pattern.Substring(2)); } else if (pattern.Length >= 2 && pattern[0] == '~' && pattern[1] == '/') { diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitIgnore.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitIgnore.cs index 19ab8688..c4e10052 100644 --- a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitIgnore.cs +++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitIgnore.cs @@ -117,14 +117,14 @@ public Matcher CreateMatcher() var reusableBuffer = new StringBuilder(); - var directory = PathUtils.ToPosixDirectoryPath(Path.GetFullPath(Path.GetDirectoryName(path))); + var directory = PathUtils.ToPosixDirectoryPath(Path.GetFullPath(Path.GetDirectoryName(path)!)); var patterns = ImmutableArray.CreateBuilder(); using (reader) { while (true) { - string line = reader.ReadLine(); + var line = reader.ReadLine(); if (line == null) { break; diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs index c2fbfeb5..9f1648f8 100644 --- a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs +++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs @@ -376,9 +376,10 @@ public static bool TryGetRepositoryLocation(string directory, out GitRepositoryL /// False if no git repository can be found that contains the specified path. public static bool TryFindRepository(string directory, out GitRepositoryLocation location) { + var dir = directory; try { - directory = Path.GetFullPath(directory); + dir = Path.GetFullPath(dir); } catch { @@ -386,16 +387,16 @@ public static bool TryFindRepository(string directory, out GitRepositoryLocation return false; } - while (directory != null) + while (dir != null) { - if (TryGetRepositoryLocationImpl(directory, out location)) + if (TryGetRepositoryLocationImpl(dir, out location)) { return true; } // TODO: https://github.com/dotnet/sourcelink/issues/302 // stop on device boundary - directory = Path.GetDirectoryName(directory); + dir = Path.GetDirectoryName(dir); } location = default; @@ -463,7 +464,7 @@ private static string ReadDotGitFile(string path) try { // link is relative to the directory containing the file: - return Path.GetFullPath(Path.Combine(Path.GetDirectoryName(path), link)); + return Path.GetFullPath(Path.Combine(Path.GetDirectoryName(path)!, link)); } catch { diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitVariableName.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitVariableName.cs index f4df8a7a..b4eefb68 100644 --- a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitVariableName.cs +++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitVariableName.cs @@ -40,7 +40,7 @@ public bool Equals(GitVariableName other) SubsectionNameEquals(other.SubsectionName) && VariableNameEquals(other.VariableName); - public override bool Equals(object obj) + public override bool Equals(object? obj) => obj is GitVariableName other && Equals(other); public override int GetHashCode() diff --git a/src/Microsoft.Build.Tasks.Git/Microsoft.Build.Tasks.Git.csproj b/src/Microsoft.Build.Tasks.Git/Microsoft.Build.Tasks.Git.csproj index e79a425e..c8cc26b7 100644 --- a/src/Microsoft.Build.Tasks.Git/Microsoft.Build.Tasks.Git.csproj +++ b/src/Microsoft.Build.Tasks.Git/Microsoft.Build.Tasks.Git.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp2.0 + net461;netcoreapp3.1 true diff --git a/src/Microsoft.Build.Tasks.Git/build/Microsoft.Build.Tasks.Git.props b/src/Microsoft.Build.Tasks.Git/build/Microsoft.Build.Tasks.Git.props index 2078031d..8204380b 100644 --- a/src/Microsoft.Build.Tasks.Git/build/Microsoft.Build.Tasks.Git.props +++ b/src/Microsoft.Build.Tasks.Git/build/Microsoft.Build.Tasks.Git.props @@ -2,6 +2,6 @@ $(MSBuildThisFileDirectory)..\tools\net461\Microsoft.Build.Tasks.Git.dll - $(MSBuildThisFileDirectory)..\tools\netcoreapp2.0\Microsoft.Build.Tasks.Git.dll + $(MSBuildThisFileDirectory)..\tools\netcoreapp3.1\Microsoft.Build.Tasks.Git.dll diff --git a/src/SourceLink.AzureDevOpsServer.Git.UnitTests/Microsoft.SourceLink.AzureDevOpsServer.Git.UnitTests.csproj b/src/SourceLink.AzureDevOpsServer.Git.UnitTests/Microsoft.SourceLink.AzureDevOpsServer.Git.UnitTests.csproj index 3757afe2..8c8d3235 100644 --- a/src/SourceLink.AzureDevOpsServer.Git.UnitTests/Microsoft.SourceLink.AzureDevOpsServer.Git.UnitTests.csproj +++ b/src/SourceLink.AzureDevOpsServer.Git.UnitTests/Microsoft.SourceLink.AzureDevOpsServer.Git.UnitTests.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp2.0 + net461;netcoreapp3.1 diff --git a/src/SourceLink.AzureDevOpsServer.Git/Microsoft.SourceLink.AzureDevOpsServer.Git.csproj b/src/SourceLink.AzureDevOpsServer.Git/Microsoft.SourceLink.AzureDevOpsServer.Git.csproj index 40abab53..0d4851b3 100644 --- a/src/SourceLink.AzureDevOpsServer.Git/Microsoft.SourceLink.AzureDevOpsServer.Git.csproj +++ b/src/SourceLink.AzureDevOpsServer.Git/Microsoft.SourceLink.AzureDevOpsServer.Git.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp2.0 + net461;netcoreapp3.1 true diff --git a/src/SourceLink.AzureDevOpsServer.Git/build/Microsoft.SourceLink.AzureDevOpsServer.Git.targets b/src/SourceLink.AzureDevOpsServer.Git/build/Microsoft.SourceLink.AzureDevOpsServer.Git.targets index dca4fab9..94a6ca8b 100644 --- a/src/SourceLink.AzureDevOpsServer.Git/build/Microsoft.SourceLink.AzureDevOpsServer.Git.targets +++ b/src/SourceLink.AzureDevOpsServer.Git/build/Microsoft.SourceLink.AzureDevOpsServer.Git.targets @@ -2,7 +2,7 @@ <_SourceLinkAzureDevOpsServerGitAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net461\Microsoft.SourceLink.AzureDevOpsServer.Git.dll - <_SourceLinkAzureDevOpsServerGitAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\netcoreapp2.0\Microsoft.SourceLink.AzureDevOpsServer.Git.dll + <_SourceLinkAzureDevOpsServerGitAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\netcoreapp3.1\Microsoft.SourceLink.AzureDevOpsServer.Git.dll diff --git a/src/SourceLink.AzureRepos.Git.UnitTests/Microsoft.SourceLink.AzureRepos.Git.UnitTests.csproj b/src/SourceLink.AzureRepos.Git.UnitTests/Microsoft.SourceLink.AzureRepos.Git.UnitTests.csproj index 5c567068..504f0e01 100644 --- a/src/SourceLink.AzureRepos.Git.UnitTests/Microsoft.SourceLink.AzureRepos.Git.UnitTests.csproj +++ b/src/SourceLink.AzureRepos.Git.UnitTests/Microsoft.SourceLink.AzureRepos.Git.UnitTests.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp2.0 + net461;netcoreapp3.1 diff --git a/src/SourceLink.AzureRepos.Git/Microsoft.SourceLink.AzureRepos.Git.csproj b/src/SourceLink.AzureRepos.Git/Microsoft.SourceLink.AzureRepos.Git.csproj index 5ea7d293..816dcfc9 100644 --- a/src/SourceLink.AzureRepos.Git/Microsoft.SourceLink.AzureRepos.Git.csproj +++ b/src/SourceLink.AzureRepos.Git/Microsoft.SourceLink.AzureRepos.Git.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp2.0 + net461;netcoreapp3.1 true diff --git a/src/SourceLink.AzureRepos.Git/build/Microsoft.SourceLink.AzureRepos.Git.targets b/src/SourceLink.AzureRepos.Git/build/Microsoft.SourceLink.AzureRepos.Git.targets index e30145ec..e955685e 100644 --- a/src/SourceLink.AzureRepos.Git/build/Microsoft.SourceLink.AzureRepos.Git.targets +++ b/src/SourceLink.AzureRepos.Git/build/Microsoft.SourceLink.AzureRepos.Git.targets @@ -2,7 +2,7 @@ <_SourceLinkAzureReposGitAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net461\Microsoft.SourceLink.AzureRepos.Git.dll - <_SourceLinkAzureReposGitAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\netcoreapp2.0\Microsoft.SourceLink.AzureRepos.Git.dll + <_SourceLinkAzureReposGitAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\netcoreapp3.1\Microsoft.SourceLink.AzureRepos.Git.dll diff --git a/src/SourceLink.AzureRepos.Tfvc/build/Microsoft.SourceLink.AzureRepos.Tfvc.targets b/src/SourceLink.AzureRepos.Tfvc/build/Microsoft.SourceLink.AzureRepos.Tfvc.targets index ede39e4e..3a26e1a5 100644 --- a/src/SourceLink.AzureRepos.Tfvc/build/Microsoft.SourceLink.AzureRepos.Tfvc.targets +++ b/src/SourceLink.AzureRepos.Tfvc/build/Microsoft.SourceLink.AzureRepos.Tfvc.targets @@ -2,7 +2,7 @@ <_SourceLinkAzureReposTfvcAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net461\Microsoft.SourceLink.AzureRepos.Tfvc.dll - <_SourceLinkAzureReposTfvcAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\netcoreapp2.0\Microsoft.SourceLink.AzureRepos.Tfvc.dll + <_SourceLinkAzureReposTfvcAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\netcoreapp3.1\Microsoft.SourceLink.AzureRepos.Tfvc.dll diff --git a/src/SourceLink.Bitbucket.Git.UnitTests/Microsoft.SourceLink.Bitbucket.Git.UnitTests.csproj b/src/SourceLink.Bitbucket.Git.UnitTests/Microsoft.SourceLink.Bitbucket.Git.UnitTests.csproj index caa3a43c..bbe4ecba 100644 --- a/src/SourceLink.Bitbucket.Git.UnitTests/Microsoft.SourceLink.Bitbucket.Git.UnitTests.csproj +++ b/src/SourceLink.Bitbucket.Git.UnitTests/Microsoft.SourceLink.Bitbucket.Git.UnitTests.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp2.0 + net461;netcoreapp3.1 diff --git a/src/SourceLink.Bitbucket.Git/Microsoft.SourceLink.Bitbucket.Git.csproj b/src/SourceLink.Bitbucket.Git/Microsoft.SourceLink.Bitbucket.Git.csproj index 621598ef..2f00828c 100644 --- a/src/SourceLink.Bitbucket.Git/Microsoft.SourceLink.Bitbucket.Git.csproj +++ b/src/SourceLink.Bitbucket.Git/Microsoft.SourceLink.Bitbucket.Git.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp2.0 + net461;netcoreapp3.1 true diff --git a/src/SourceLink.Bitbucket.Git/build/Microsoft.SourceLink.Bitbucket.Git.targets b/src/SourceLink.Bitbucket.Git/build/Microsoft.SourceLink.Bitbucket.Git.targets index 16576b73..12fc0b8d 100644 --- a/src/SourceLink.Bitbucket.Git/build/Microsoft.SourceLink.Bitbucket.Git.targets +++ b/src/SourceLink.Bitbucket.Git/build/Microsoft.SourceLink.Bitbucket.Git.targets @@ -2,7 +2,7 @@ <_SourceLinkBitbucketAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net461\Microsoft.SourceLink.Bitbucket.Git.dll - <_SourceLinkBitbucketAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\netcoreapp2.0\Microsoft.SourceLink.Bitbucket.Git.dll + <_SourceLinkBitbucketAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\netcoreapp3.1\Microsoft.SourceLink.Bitbucket.Git.dll diff --git a/src/SourceLink.Common.UnitTests/Microsoft.SourceLink.Common.UnitTests.csproj b/src/SourceLink.Common.UnitTests/Microsoft.SourceLink.Common.UnitTests.csproj index e94aa501..54a8a2fc 100644 --- a/src/SourceLink.Common.UnitTests/Microsoft.SourceLink.Common.UnitTests.csproj +++ b/src/SourceLink.Common.UnitTests/Microsoft.SourceLink.Common.UnitTests.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp2.0 + net461;netcoreapp3.1 diff --git a/src/SourceLink.Common/Microsoft.SourceLink.Common.csproj b/src/SourceLink.Common/Microsoft.SourceLink.Common.csproj index efc214ab..eeb641d3 100644 --- a/src/SourceLink.Common/Microsoft.SourceLink.Common.csproj +++ b/src/SourceLink.Common/Microsoft.SourceLink.Common.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp2.0 + net461;netcoreapp3.1 true diff --git a/src/SourceLink.Common/build/Microsoft.SourceLink.Common.props b/src/SourceLink.Common/build/Microsoft.SourceLink.Common.props index 1e2e1f21..4d3ac740 100644 --- a/src/SourceLink.Common/build/Microsoft.SourceLink.Common.props +++ b/src/SourceLink.Common/build/Microsoft.SourceLink.Common.props @@ -2,7 +2,7 @@ <_MicrosoftSourceLinkCommonAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net461\Microsoft.SourceLink.Common.dll - <_MicrosoftSourceLinkCommonAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\netcoreapp2.0\Microsoft.SourceLink.Common.dll + <_MicrosoftSourceLinkCommonAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\netcoreapp3.1\Microsoft.SourceLink.Common.dll diff --git a/src/SourceLink.Git.IntegrationTests/Microsoft.SourceLink.Git.IntegrationTests.csproj b/src/SourceLink.Git.IntegrationTests/Microsoft.SourceLink.Git.IntegrationTests.csproj index a3f1755e..60e8ea90 100644 --- a/src/SourceLink.Git.IntegrationTests/Microsoft.SourceLink.Git.IntegrationTests.csproj +++ b/src/SourceLink.Git.IntegrationTests/Microsoft.SourceLink.Git.IntegrationTests.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp2.0 + net461;netcoreapp3.1 diff --git a/src/SourceLink.GitHub.UnitTests/Microsoft.SourceLink.GitHub.UnitTests.csproj b/src/SourceLink.GitHub.UnitTests/Microsoft.SourceLink.GitHub.UnitTests.csproj index 78fc15de..5dfd5d1c 100644 --- a/src/SourceLink.GitHub.UnitTests/Microsoft.SourceLink.GitHub.UnitTests.csproj +++ b/src/SourceLink.GitHub.UnitTests/Microsoft.SourceLink.GitHub.UnitTests.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp2.0 + net461;netcoreapp3.1 diff --git a/src/SourceLink.GitHub/Microsoft.SourceLink.GitHub.csproj b/src/SourceLink.GitHub/Microsoft.SourceLink.GitHub.csproj index 59bdd335..56e25605 100644 --- a/src/SourceLink.GitHub/Microsoft.SourceLink.GitHub.csproj +++ b/src/SourceLink.GitHub/Microsoft.SourceLink.GitHub.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp2.0 + net461;netcoreapp3.1 true diff --git a/src/SourceLink.GitHub/build/Microsoft.SourceLink.GitHub.targets b/src/SourceLink.GitHub/build/Microsoft.SourceLink.GitHub.targets index f0a0c1d9..3300c412 100644 --- a/src/SourceLink.GitHub/build/Microsoft.SourceLink.GitHub.targets +++ b/src/SourceLink.GitHub/build/Microsoft.SourceLink.GitHub.targets @@ -2,7 +2,7 @@ <_SourceLinkGitHubAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net461\Microsoft.SourceLink.GitHub.dll - <_SourceLinkGitHubAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\netcoreapp2.0\Microsoft.SourceLink.GitHub.dll + <_SourceLinkGitHubAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\netcoreapp3.1\Microsoft.SourceLink.GitHub.dll diff --git a/src/SourceLink.GitLab.UnitTests/Microsoft.SourceLink.GitLab.UnitTests.csproj b/src/SourceLink.GitLab.UnitTests/Microsoft.SourceLink.GitLab.UnitTests.csproj index f0bbc702..c74ab311 100644 --- a/src/SourceLink.GitLab.UnitTests/Microsoft.SourceLink.GitLab.UnitTests.csproj +++ b/src/SourceLink.GitLab.UnitTests/Microsoft.SourceLink.GitLab.UnitTests.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp2.0 + net461;netcoreapp3.1 diff --git a/src/SourceLink.GitLab/Microsoft.SourceLink.GitLab.csproj b/src/SourceLink.GitLab/Microsoft.SourceLink.GitLab.csproj index dd54bbb7..440baba4 100644 --- a/src/SourceLink.GitLab/Microsoft.SourceLink.GitLab.csproj +++ b/src/SourceLink.GitLab/Microsoft.SourceLink.GitLab.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp2.0 + net461;netcoreapp3.1 true diff --git a/src/SourceLink.GitLab/build/Microsoft.SourceLink.GitLab.targets b/src/SourceLink.GitLab/build/Microsoft.SourceLink.GitLab.targets index ed8d16b3..608000b3 100644 --- a/src/SourceLink.GitLab/build/Microsoft.SourceLink.GitLab.targets +++ b/src/SourceLink.GitLab/build/Microsoft.SourceLink.GitLab.targets @@ -2,7 +2,7 @@ <_SourceLinkGitLabAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net461\Microsoft.SourceLink.GitLab.dll - <_SourceLinkGitLabAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\netcoreapp2.0\Microsoft.SourceLink.GitLab.dll + <_SourceLinkGitLabAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\netcoreapp3.1\Microsoft.SourceLink.GitLab.dll diff --git a/src/SourceLink.GitWeb.UnitTests/Microsoft.SourceLink.GitWeb.UnitTests.csproj b/src/SourceLink.GitWeb.UnitTests/Microsoft.SourceLink.GitWeb.UnitTests.csproj index 16608152..411ec39f 100644 --- a/src/SourceLink.GitWeb.UnitTests/Microsoft.SourceLink.GitWeb.UnitTests.csproj +++ b/src/SourceLink.GitWeb.UnitTests/Microsoft.SourceLink.GitWeb.UnitTests.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp2.0 + net461;netcoreapp3.1 diff --git a/src/SourceLink.GitWeb/Microsoft.SourceLink.GitWeb.csproj b/src/SourceLink.GitWeb/Microsoft.SourceLink.GitWeb.csproj index d9fb3080..3fd7502a 100644 --- a/src/SourceLink.GitWeb/Microsoft.SourceLink.GitWeb.csproj +++ b/src/SourceLink.GitWeb/Microsoft.SourceLink.GitWeb.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp2.0 + net461;netcoreapp3.1 true diff --git a/src/SourceLink.GitWeb/build/Microsoft.SourceLink.GitWeb.targets b/src/SourceLink.GitWeb/build/Microsoft.SourceLink.GitWeb.targets index d5019e7a..065585ee 100644 --- a/src/SourceLink.GitWeb/build/Microsoft.SourceLink.GitWeb.targets +++ b/src/SourceLink.GitWeb/build/Microsoft.SourceLink.GitWeb.targets @@ -2,7 +2,7 @@ <_SourceLinkGitWebAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net461\Microsoft.SourceLink.GitWeb.dll - <_SourceLinkGitWebAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\netcoreapp2.0\Microsoft.SourceLink.GitWeb.dll + <_SourceLinkGitWebAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\netcoreapp3.1\Microsoft.SourceLink.GitWeb.dll diff --git a/src/TestUtilities/AssertEx.cs b/src/TestUtilities/AssertEx.cs index 77c780a8..82cc2eb1 100644 --- a/src/TestUtilities/AssertEx.cs +++ b/src/TestUtilities/AssertEx.cs @@ -4,6 +4,7 @@ using System.Collections; using System.Collections.Generic; using System.Collections.Immutable; +using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Reflection; using System.Runtime.CompilerServices; @@ -45,7 +46,7 @@ public static bool Equals(T left, T right) return s_instance.Equals(left, right); } - bool IEqualityComparer.Equals(T x, T y) + bool IEqualityComparer.Equals([AllowNull] T x, [AllowNull] T y) { if (CanBeNull()) { @@ -282,12 +283,12 @@ public static void All(IEnumerable actual, Func predicate) } } - public static string ToString(object? o) + public static string? ToString(object? o) { return Convert.ToString(o); } - public static string ToString(IEnumerable list, string separator = ", ", Func? itemInspector = null) + public static string ToString(IEnumerable list, string separator = ", ", Func? itemInspector = null) { if (itemInspector == null) { @@ -410,7 +411,7 @@ public static string GetAssertMessage(string expected, string actual, bool escap public static string GetAssertMessage(IEnumerable expected, IEnumerable actual, bool escapeQuotes) { - Func? itemInspector = escapeQuotes ? new Func(t => t?.ToString().Replace("\"", "\"\"") ?? "null") : null; + Func? itemInspector = escapeQuotes ? new Func(t => t?.ToString()?.Replace("\"", "\"\"") ?? "null") : null; return GetAssertMessage(expected, actual, itemInspector: itemInspector, itemSeparator: "\r\n"); } @@ -429,7 +430,7 @@ public static string GetAssertMessage( } else { - itemInspector = new Func(obj => (obj != null) ? obj.ToString() : ""); + itemInspector = new Func(obj => obj?.ToString() ?? ""); } } diff --git a/src/TestUtilities/ConditionalFactAttribute.cs b/src/TestUtilities/ConditionalFactAttribute.cs index 0556f880..8a013c7b 100644 --- a/src/TestUtilities/ConditionalFactAttribute.cs +++ b/src/TestUtilities/ConditionalFactAttribute.cs @@ -12,7 +12,7 @@ public ConditionalFactAttribute(params Type[] skipConditions) { foreach (var skipCondition in skipConditions) { - ExecutionCondition condition = (ExecutionCondition)Activator.CreateInstance(skipCondition); + ExecutionCondition condition = (ExecutionCondition)Activator.CreateInstance(skipCondition)!; if (condition.ShouldSkip) { Skip = condition.SkipReason; @@ -28,7 +28,7 @@ public ConditionalTheoryAttribute(params Type[] skipConditions) { foreach (var skipCondition in skipConditions) { - ExecutionCondition condition = (ExecutionCondition)Activator.CreateInstance(skipCondition); + ExecutionCondition condition = (ExecutionCondition)Activator.CreateInstance(skipCondition)!; if (condition.ShouldSkip) { Skip = condition.SkipReason; diff --git a/src/TestUtilities/DotNetSdk/DotNetSdkTestBase.cs b/src/TestUtilities/DotNetSdk/DotNetSdkTestBase.cs index d66b208c..79d5bca4 100644 --- a/src/TestUtilities/DotNetSdk/DotNetSdkTestBase.cs +++ b/src/TestUtilities/DotNetSdk/DotNetSdkTestBase.cs @@ -21,7 +21,7 @@ public sealed class DotNetSdkAvailable : ExecutionCondition public readonly TempRoot Temp = new TempRoot(); - private static readonly string? s_dotnetExeName; + private static readonly string s_dotnetExeName; private static readonly string? s_dotnetInstallDir; private static readonly BuildInfoAttribute s_buildInfo; private static readonly string? s_dotnetSdkPath; @@ -80,15 +80,15 @@ private static string GetLocalNuGetConfigContent(string packagesDir) => static DotNetSdkTestBase() { s_dotnetExeName = "dotnet" + (Path.DirectorySeparatorChar == '/' ? "" : ".exe"); - s_buildInfo = typeof(DotNetSdkTestBase).Assembly.GetCustomAttribute(); + s_buildInfo = typeof(DotNetSdkTestBase).Assembly!.GetCustomAttribute()!; var minSdkVersion = SemanticVersion.Parse(s_buildInfo.SdkVersion); - static bool isDotNetInstallDirectory(string dir) + static bool isDotNetInstallDirectory(string? dir) => dir != null && File.Exists(Path.Combine(dir, s_dotnetExeName)); var dotnetInstallDir = - new[] { Environment.GetEnvironmentVariable("DOTNET_INSTALL_DIR") }.Concat(Environment.GetEnvironmentVariable("PATH").Split(Path.PathSeparator)). + new[] { Environment.GetEnvironmentVariable("DOTNET_INSTALL_DIR") }.Concat(Environment.GetEnvironmentVariable("PATH")!.Split(Path.PathSeparator)). FirstOrDefault(isDotNetInstallDirectory); if (dotnetInstallDir != null) @@ -157,9 +157,9 @@ public DotNetSdkTestBase(params string[] packages) { Assert.True(s_dotnetInstallDir != null, $"SDK not found. Use {nameof(ConditionalFactAttribute)}(typeof({nameof(DotNetSdkAvailable)})) to skip the test if the SDK is not found."); - DotNetPath = Path.Combine(s_dotnetInstallDir, s_dotnetExeName); + DotNetPath = Path.Combine(s_dotnetInstallDir!, s_dotnetExeName); var testBinDirectory = Path.GetDirectoryName(typeof(DotNetSdkTestBase).Assembly.Location); - var sdksDir = Path.Combine(s_dotnetSdkPath, "Sdks"); + var sdksDir = Path.Combine(s_dotnetSdkPath!, "Sdks"); ProjectName = "test"; ProjectFileName = ProjectName + ".csproj"; @@ -208,7 +208,7 @@ protected void VerifyValues( string[]? expectedWarnings = null, string? additionalCommandLineArgs = null, string buildVerbosity = "minimal", - Func? expectedBuildOutputFilter = null) + Predicate? expectedBuildOutputFilter = null) { NullableDebug.Assert(targets != null); NullableDebug.Assert(expressions != null); @@ -248,7 +248,7 @@ string[] getDiagnostics(string[] lines, bool error) where match.Success select match.Groups[1].Value).ToArray(); - bool diagnosticsEqual(string expected, string actual) + static bool diagnosticsEqual(string expected, string actual) { string ellipsis = "..."; int index = expected.IndexOf(ellipsis); @@ -280,7 +280,7 @@ bool diagnosticsEqual(string expected, string actual) if (expectedBuildOutputFilter != null) { - Assert.True(outputLines.Any(expectedBuildOutputFilter)); + Assert.Contains(outputLines, expectedBuildOutputFilter); } success = true; diff --git a/src/TestUtilities/TestUtilities.csproj b/src/TestUtilities/TestUtilities.csproj index 20953914..22b4d7e3 100644 --- a/src/TestUtilities/TestUtilities.csproj +++ b/src/TestUtilities/TestUtilities.csproj @@ -1,6 +1,6 @@  - net461;netstandard2.0 + net461;netcoreapp3.1 false true From dfea1e608dbb3bddb1c02779886c0e6c8a986466 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2020 12:16:40 +0000 Subject: [PATCH 088/161] Update dependencies from https://github.com/dotnet/arcade build 20200724.1 (#636) [master] Update dependencies from dotnet/arcade - Updates: - Microsoft.DotNet.Arcade.Sdk: from 5.0.0-beta.20364.3 to 5.0.0-beta.20374.1 --- eng/Version.Details.xml | 4 ++-- eng/common/cross/toolchain.cmake | 41 ++++++++++++++++++++------------ global.json | 2 +- 3 files changed, 29 insertions(+), 18 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 2e53d45f..0d23aab7 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - ff5d4b6c8dbdaeacb6e6159d3f8185118dffd915 + f6192d1e284a08ac05041d05fa6e60dec74b24f5 diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index 88a758af..b9fe796f 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -127,29 +127,40 @@ endif() # Specify link flags +function(add_toolchain_linker_flag Flag) + set(Config "${ARGV1}") + set(CONFIG_SUFFIX "") + if (NOT Config STREQUAL "") + set(CONFIG_SUFFIX "_${Config}") + endif() + set("CMAKE_EXE_LINKER_FLAGS${CONFIG_SUFFIX}" "${CMAKE_EXE_LINKER_FLAGS${CONFIG_SUFFIX}} ${Flag}" PARENT_SCOPE) + set("CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}" "${CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}} ${Flag}" PARENT_SCOPE) +endfunction() + + if(TARGET_ARCH_NAME STREQUAL "armel") if(DEFINED TIZEN_TOOLCHAIN) # For Tizen only - add_link_options("-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") - add_link_options("-L${CROSS_ROOTFS}/lib") - add_link_options("-L${CROSS_ROOTFS}/usr/lib") - add_link_options("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") + add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib") + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib") + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") endif() elseif(TARGET_ARCH_NAME STREQUAL "arm64") if(DEFINED TIZEN_TOOLCHAIN) # For Tizen only - add_link_options("-B${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") - add_link_options("-L${CROSS_ROOTFS}/lib64") - add_link_options("-L${CROSS_ROOTFS}/usr/lib64") - add_link_options("-L${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") - - add_link_options("-Wl,--rpath-link=${CROSS_ROOTFS}/lib64") - add_link_options("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64") - add_link_options("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") + add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib64") + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib64") + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") + + add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/lib64") + add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64") + add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") endif() elseif(TARGET_ARCH_NAME STREQUAL "x86") - add_link_options(-m32) + add_toolchain_linker_flag(-m32) elseif(ILLUMOS) - add_link_options("-L${CROSS_ROOTFS}/lib/amd64") - add_link_options("-L${CROSS_ROOTFS}/usr/amd64/lib") + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib/amd64") + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/amd64/lib") endif() # Specify compile options diff --git a/global.json b/global.json index ea83bbe2..b44139b5 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "5.0.100-preview.6.20310.4" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20364.3" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20374.1" } } From f82ec1a4a1c2574434f20ed9349601873428e098 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 10 Aug 2020 12:31:50 +0000 Subject: [PATCH 089/161] Update dependencies from https://github.com/dotnet/arcade build 20200803.5 (#638) [master] Update dependencies from dotnet/arcade - Updates: - Microsoft.DotNet.Arcade.Sdk: from 5.0.0-beta.20374.1 to 5.0.0-beta.20403.5 --- eng/Version.Details.xml | 4 +- eng/common/cross/arm64/tizen-fetch.sh | 2 +- eng/common/cross/armel/tizen-fetch.sh | 7 +- eng/common/cross/toolchain.cmake | 2 +- eng/common/performance/crossgen_perf.proj | 81 +++++++++++++++++++ ...helixpublish.proj => microbenchmarks.proj} | 56 +------------ eng/common/performance/performance-setup.sh | 27 ++++++- .../templates/steps/perf-send-to-helix.yml | 6 +- global.json | 2 +- 9 files changed, 120 insertions(+), 67 deletions(-) create mode 100644 eng/common/performance/crossgen_perf.proj rename eng/common/performance/{perfhelixpublish.proj => microbenchmarks.proj} (67%) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 0d23aab7..1f5d9cde 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - f6192d1e284a08ac05041d05fa6e60dec74b24f5 + 7385e2722b9fa517314aa5db1fa598a8d417b3c7 diff --git a/eng/common/cross/arm64/tizen-fetch.sh b/eng/common/cross/arm64/tizen-fetch.sh index 338d1c3b..a48a6f51 100644 --- a/eng/common/cross/arm64/tizen-fetch.sh +++ b/eng/common/cross/arm64/tizen-fetch.sh @@ -161,7 +161,7 @@ fetch_tizen_pkgs aarch64 gcc glibc glibc-devel libicu libicu-devel libatomic lin Inform "fetch coreclr packages" fetch_tizen_pkgs aarch64 lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu Inform "fetch corefx packages" -fetch_tizen_pkgs aarch64 libcom_err libcom_err-devel zlib zlib-devel libopenssl libopenssl1.1-devel krb5 krb5-devel +fetch_tizen_pkgs aarch64 libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel Inform "Initialize standard unified" fetch_tizen_pkgs_init standard unified diff --git a/eng/common/cross/armel/tizen-fetch.sh b/eng/common/cross/armel/tizen-fetch.sh index ed70e0a8..2776cbba 100755 --- a/eng/common/cross/armel/tizen-fetch.sh +++ b/eng/common/cross/armel/tizen-fetch.sh @@ -51,7 +51,7 @@ if [ ! -d $TMPDIR ]; then mkdir -p $TMPDIR fi -TIZEN_URL=http://download.tizen.org/releases/milestone/tizen +TIZEN_URL=http://download.tizen.org/snapshots/tizen BUILD_XML=build.xml REPOMD_XML=repomd.xml PRIMARY_XML=primary.xml @@ -157,12 +157,11 @@ fetch_tizen_pkgs() Inform "Initialize arm base" fetch_tizen_pkgs_init standard base Inform "fetch common packages" -fetch_tizen_pkgs armv7l gcc glibc glibc-devel libicu libicu-devel libatomic -fetch_tizen_pkgs noarch linux-glibc-devel +fetch_tizen_pkgs armv7l gcc glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel Inform "fetch coreclr packages" fetch_tizen_pkgs armv7l lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu Inform "fetch corefx packages" -fetch_tizen_pkgs armv7l libcom_err libcom_err-devel zlib zlib-devel libopenssl libopenssl-devel krb5 krb5-devel libcurl libcurl-devel +fetch_tizen_pkgs armv7l libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel Inform "Initialize standard unified" fetch_tizen_pkgs_init standard unified diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index b9fe796f..25667079 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -15,7 +15,7 @@ if(TARGET_ARCH_NAME STREQUAL "armel") set(CMAKE_SYSTEM_PROCESSOR armv7l) set(TOOLCHAIN "arm-linux-gnueabi") if("$ENV{__DistroRid}" MATCHES "tizen.*") - set(TIZEN_TOOLCHAIN "armv7l-tizen-linux-gnueabi/6.2.1") + set(TIZEN_TOOLCHAIN "armv7l-tizen-linux-gnueabi/9.2.0") endif() elseif(TARGET_ARCH_NAME STREQUAL "arm") set(CMAKE_SYSTEM_PROCESSOR armv7l) diff --git a/eng/common/performance/crossgen_perf.proj b/eng/common/performance/crossgen_perf.proj new file mode 100644 index 00000000..3c8c33d7 --- /dev/null +++ b/eng/common/performance/crossgen_perf.proj @@ -0,0 +1,81 @@ + + + + py -3 + $(HelixPreCommands);call %HELIX_CORRELATION_PAYLOAD%\performance\tools\machine-setup.cmd;set PYTHONPATH=%HELIX_WORKITEM_PAYLOAD%\scripts%3B%HELIX_WORKITEM_PAYLOAD% + %HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts + $HELIX_CORRELATION_PAYLOAD + $(BaseDirectory)/performance + + + + $(PerformanceDirectory)/scripts/benchmarks_ci.py --csproj $(PerformanceDirectory)/$(TargetCsproj) + --dotnet-versions $DOTNET_VERSION --cli-source-info args --cli-branch $PERFLAB_BRANCH --cli-commit-sha $PERFLAB_HASH --cli-repository https://github.com/$PERFLAB_REPO --cli-source-timestamp $PERFLAB_BUILDTIMESTAMP + python3 + $(BaseDirectory)/Core_Root/corerun + $(HelixPreCommands);chmod +x $(PerformanceDirectory)/tools/machine-setup.sh;. $(PerformanceDirectory)/tools/machine-setup.sh + $(BaseDirectory)/artifacts/BenchmarkDotNet.Artifacts + $(BaseDirectory)/artifacts/BenchmarkDotNet.Artifacts_Baseline + $(PerformanceDirectory)/src/tools/ResultsComparer/ResultsComparer.csproj + $(PerformanceDirectory)/tools/dotnet/$(Architecture)/dotnet + %25 + $HELIX_WORKITEM_ROOT/testResults.xml + + + + + %(Identity) + + + + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name System.Private.Xml.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name System.Linq.Expressions.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name Microsoft.CodeAnalysis.VisualBasic.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name Microsoft.CodeAnalysis.CSharp.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name System.Private.CoreLib.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single System.Private.Xml.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single System.Linq.Expressions.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single Microsoft.CodeAnalysis.VisualBasic.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single Microsoft.CodeAnalysis.CSharp.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single System.Private.CoreLib.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --composite %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\framework-r2r.dll.rsp --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + 1:00 + + + \ No newline at end of file diff --git a/eng/common/performance/perfhelixpublish.proj b/eng/common/performance/microbenchmarks.proj similarity index 67% rename from eng/common/performance/perfhelixpublish.proj rename to eng/common/performance/microbenchmarks.proj index 272366da..5c95ef45 100644 --- a/eng/common/performance/perfhelixpublish.proj +++ b/eng/common/performance/microbenchmarks.proj @@ -41,6 +41,10 @@ $HELIX_WORKITEM_ROOT/testResults.xml + + $(CliArguments) --wasm + + --corerun %HELIX_CORRELATION_PAYLOAD%\dotnet-mono\shared\Microsoft.NETCore.App\5.0.0\corerun.exe @@ -137,56 +141,4 @@ 4:00 - - - - $(WorkItemDirectory)\ScenarioCorrelation - $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name System.Private.Xml.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root - - - $(WorkItemDirectory)\ScenarioCorrelation - $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name System.Linq.Expressions.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root - - - $(WorkItemDirectory)\ScenarioCorrelation - $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name Microsoft.CodeAnalysis.VisualBasic.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root - - - $(WorkItemDirectory)\ScenarioCorrelation - $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name Microsoft.CodeAnalysis.CSharp.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root - - - $(WorkItemDirectory)\ScenarioCorrelation - $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name System.Private.CoreLib.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root - - - - - - $(WorkItemDirectory)\ScenarioCorrelation - $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single System.Private.Xml.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root - - - $(WorkItemDirectory)\ScenarioCorrelation - $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single System.Linq.Expressions.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root - - - $(WorkItemDirectory)\ScenarioCorrelation - $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single Microsoft.CodeAnalysis.VisualBasic.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root - - - $(WorkItemDirectory)\ScenarioCorrelation - $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single Microsoft.CodeAnalysis.CSharp.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root - - - $(WorkItemDirectory)\ScenarioCorrelation - $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single System.Private.CoreLib.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root - - - $(WorkItemDirectory)\ScenarioCorrelation - $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --composite %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\framework-r2r.dll.rsp --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root - 1:00 - - - \ No newline at end of file diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index c87cbf0f..fd26bdb1 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -24,6 +24,8 @@ run_from_perf_repo=false use_core_run=true use_baseline_core_run=true using_mono=false +wasm_runtime_loc= +using_wasm=false while (($# > 0)); do lowerI="$(echo $1 | awk '{print tolower($0)}')" @@ -70,7 +72,7 @@ while (($# > 0)); do ;; --kind) kind=$2 - configurations="CompliationMode=$compilation_mode RunKind=$kind" + configurations="CompilationMode=$compilation_mode RunKind=$kind" shift 2 ;; --runcategories) @@ -101,6 +103,10 @@ while (($# > 0)); do mono_dotnet=$2 shift 2 ;; + --wasm) + wasm_runtime_loc=$2 + shift 2 + ;; --compare) compare=true shift 1 @@ -130,6 +136,7 @@ while (($# > 0)); do echo " --runcategories Related to csproj. Categories of benchmarks to run. Defaults to \"coreclr corefx\"" echo " --internal If the benchmarks are running as an official job." echo " --monodotnet Pass the path to the mono dotnet for mono performance testing." + echo " --wasm Path to the unpacked wasm runtime pack." echo "" exit 0 ;; @@ -141,7 +148,7 @@ if [ "$repository" == "dotnet/performance" ] || [ "$repository" == "dotnet-perfo fi if [ -z "$configurations" ]; then - configurations="CompliationMode=$compilation_mode" + configurations="CompilationMode=$compilation_mode" fi if [ -z "$core_root_directory" ]; then @@ -191,11 +198,15 @@ if [[ "$mono_dotnet" != "" ]]; then configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot" fi +if [[ "$wasm_runtime_loc" != "" ]]; then + configurations="CompilationMode=wasm;RunKind=micro" +fi + if [[ "$monointerpreter" == "true" ]]; then - extra_benchmark_dotnet_arguments="--category-exclusion-filter NoInterpreter" + extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter" fi -common_setup_arguments="--channel master --queue $queue --build-number $build_number --build-configs $configurations --architecture $architecture" +common_setup_arguments="--channel master --queue $queue --build-number $build_number --build-configs \"$configurations\" --architecture $architecture" setup_arguments="--repository https://github.com/$repository --branch $branch --get-perf-hash --commit-sha $commit_sha $common_setup_arguments" @@ -217,6 +228,13 @@ else mv $docs_directory $workitem_directory fi +if [[ "$wasm_runtime_loc" != "" ]]; then + using_wasm=true + wasm_dotnet_path=$payload_directory/dotnet-wasm + mv $wasm_runtime_loc $wasm_dotnet_path + extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --wasmMainJS \$HELIX_CORRELATION_PAYLOAD/dotnet-wasm/runtime-test.js --wasmEngine /home/helixbot/.jsvu/v8 --customRuntimePack \$HELIX_CORRELATION_PAYLOAD/dotnet-wasm" +fi + if [[ "$mono_dotnet" != "" ]]; then using_mono=true mono_dotnet_path=$payload_directory/dotnet-mono @@ -259,3 +277,4 @@ Write-PipelineSetVariable -name "Kind" -value "$kind" -is_multi_job_variable fal Write-PipelineSetVariable -name "_BuildConfig" -value "$architecture.$kind.$framework" -is_multi_job_variable false Write-PipelineSetVariable -name "Compare" -value "$compare" -is_multi_job_variable false Write-PipelineSetVariable -name "MonoDotnet" -value "$using_mono" -is_multi_job_variable false +Write-PipelineSetVariable -name "WasmDotnet" -value "$using_wasm" -is_multi_job_variable false diff --git a/eng/common/templates/steps/perf-send-to-helix.yml b/eng/common/templates/steps/perf-send-to-helix.yml index b3ea9acf..e003fe2e 100644 --- a/eng/common/templates/steps/perf-send-to-helix.yml +++ b/eng/common/templates/steps/perf-send-to-helix.yml @@ -1,5 +1,6 @@ # Please remember to update the documentation if you make changes to these parameters! parameters: + ProjectFile: '' # required -- project file that specifies the helix workitems HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number @@ -18,9 +19,10 @@ parameters: DisplayNamePrefix: 'Send job to Helix' # optional -- rename the beginning of the displayName of the steps in AzDO condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false + steps: - - powershell: $(Build.SourcesDirectory)\eng\common\msbuild.ps1 $(Build.SourcesDirectory)\eng\common\performance\perfhelixpublish.proj /restore /t:Test /bl:$(Build.SourcesDirectory)\artifacts\log\$env:BuildConfig\SendToHelix.binlog + - powershell: $(Build.SourcesDirectory)\eng\common\msbuild.ps1 $(Build.SourcesDirectory)\eng\common\performance\${{ parameters.ProjectFile }} /restore /t:Test /bl:$(Build.SourcesDirectory)\artifacts\log\$env:BuildConfig\SendToHelix.binlog displayName: ${{ parameters.DisplayNamePrefix }} (Windows) env: BuildConfig: $(_BuildConfig) @@ -42,7 +44,7 @@ steps: SYSTEM_ACCESSTOKEN: $(System.AccessToken) condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) continueOnError: ${{ parameters.continueOnError }} - - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/performance/perfhelixpublish.proj /restore /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog + - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/performance/${{ parameters.ProjectFile }} /restore /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog displayName: ${{ parameters.DisplayNamePrefix }} (Unix) env: BuildConfig: $(_BuildConfig) diff --git a/global.json b/global.json index b44139b5..7bc48e1c 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "5.0.100-preview.6.20310.4" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20374.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20403.5" } } From d4214598cfdb7a9fe243a0e8d4a91b611e7ec391 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 17 Aug 2020 12:31:28 +0000 Subject: [PATCH 090/161] Update dependencies from https://github.com/dotnet/arcade build 20200811.8 (#641) [master] Update dependencies from dotnet/arcade - Updates: - Microsoft.DotNet.Arcade.Sdk: from 5.0.0-beta.20403.5 to 5.0.0-beta.20411.8 --- eng/Version.Details.xml | 4 ++-- eng/common/SetupNugetSources.ps1 | 17 +++++++++++++++++ eng/common/SetupNugetSources.sh | 23 +++++++++++++++++++++-- eng/common/sdk-task.ps1 | 1 + global.json | 2 +- 5 files changed, 42 insertions(+), 5 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 1f5d9cde..564548c2 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 7385e2722b9fa517314aa5db1fa598a8d417b3c7 + ecec08a0eebbd92bb9538e351d475582551d9092 diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index dc221781..bb361713 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -11,6 +11,8 @@ # See example YAML call for this script below. Note the use of the variable `$(dn-bot-dnceng-artifact-feeds-rw)` # from the AzureDevOps-Artifact-Feeds-Pats variable group. # +# Any disabledPackageSources entries which start with "darc-int" will be re-enabled as part of this script executing +# # - task: PowerShell@2 # displayName: Setup Private Feeds Credentials # condition: eq(variables['Agent.OS'], 'Windows_NT') @@ -94,6 +96,14 @@ function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $Passw } } +function EnablePrivatePackageSources($DisabledPackageSources) { + $maestroPrivateSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]") + ForEach ($DisabledPackageSource in $maestroPrivateSources) { + Write-Host "`tEnsuring private source '$($DisabledPackageSource.key)' is enabled" + $DisabledPackageSource.SetAttribute("value", "false") + } +} + if (!(Test-Path $ConfigFile -PathType Leaf)) { Write-PipelineTelemetryError -Category 'Build' -Message "Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. Couldn't find the NuGet config file: $ConfigFile" ExitWithExitCode 1 @@ -123,6 +133,13 @@ if ($creds -eq $null) { $doc.DocumentElement.AppendChild($creds) | Out-Null } +# Check for disabledPackageSources; we'll enable any darc-int ones we find there +$disabledSources = $doc.DocumentElement.SelectSingleNode("disabledPackageSources") +if ($disabledSources -ne $null) { + Write-Host "Checking for any darc-int disabled package sources in the disabledPackageSources node" + EnablePrivatePackageSources -DisabledPackageSources $disabledSources +} + $userName = "dn-bot" # Insert credential nodes for Maestro's private feeds diff --git a/eng/common/SetupNugetSources.sh b/eng/common/SetupNugetSources.sh index f33e3771..ef333829 100644 --- a/eng/common/SetupNugetSources.sh +++ b/eng/common/SetupNugetSources.sh @@ -13,6 +13,8 @@ # See example YAML call for this script below. Note the use of the variable `$(dn-bot-dnceng-artifact-feeds-rw)` # from the AzureDevOps-Artifact-Feeds-Pats variable group. # +# Any disabledPackageSources entries which start with "darc-int" will be re-enabled as part of this script executing. +# # - task: Bash@3 # displayName: Setup Private Feeds Credentials # inputs: @@ -63,7 +65,7 @@ if [ "$?" != "0" ]; then ConfigNodeHeader="" PackageSourcesTemplate="${TB}${NL}${TB}" - sed -i.bak "s|$ConfigNodeHeader|$ConfigNodeHeader${NL}$PackageSourcesTemplate|" NuGet.config + sed -i.bak "s|$ConfigNodeHeader|$ConfigNodeHeader${NL}$PackageSourcesTemplate|" $ConfigFile fi # Ensure there is a ... section. @@ -74,7 +76,7 @@ if [ "$?" != "0" ]; then PackageSourcesNodeFooter="" PackageSourceCredentialsTemplate="${TB}${NL}${TB}" - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourcesNodeFooter${NL}$PackageSourceCredentialsTemplate|" NuGet.config + sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourcesNodeFooter${NL}$PackageSourceCredentialsTemplate|" $ConfigFile fi PackageSources=() @@ -146,3 +148,20 @@ for FeedName in ${PackageSources[@]} ; do sed -i.bak "s|$PackageSourceCredentialsNodeFooter|$NewCredential${NL}$PackageSourceCredentialsNodeFooter|" $ConfigFile fi done + +# Re-enable any entries in disabledPackageSources where the feed name contains darc-int +grep -i "" $ConfigFile +if [ "$?" == "0" ]; then + DisabledDarcIntSources=() + echo "Re-enabling any disabled \"darc-int\" package sources in $ConfigFile" + DisabledDarcIntSources+=$(grep -oh '"darc-int-[^"]*" value="true"' $ConfigFile | tr -d '"') + for DisabledSourceName in ${DisabledDarcIntSources[@]} ; do + if [[ $DisabledSourceName == darc-int* ]] + then + OldDisableValue="add key=\"$DisabledSourceName\" value=\"true\"" + NewDisableValue="add key=\"$DisabledSourceName\" value=\"false\"" + sed -i.bak "s|$OldDisableValue|$NewDisableValue|" $ConfigFile + echo "Neutralized disablePackageSources entry for '$DisabledSourceName'" + fi + done +fi diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index 32ad5c7e..e159c6f1 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -42,6 +42,7 @@ function Build([string]$target) { /p:Configuration=$configuration ` /p:RepoRoot=$RepoRoot ` /p:BaseIntermediateOutputPath=$outputPath ` + /v:$verbosity ` @properties } diff --git a/global.json b/global.json index 7bc48e1c..527a25d1 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "5.0.100-preview.6.20310.4" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20403.5" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20411.8" } } From 785d853616ee734b223f2b000aa1171ad183310e Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Tue, 18 Aug 2020 16:33:10 +0000 Subject: [PATCH 091/161] Update dependencies from https://github.com/dotnet/arcade build 20200817.6 (#642) [master] Update dependencies from dotnet/arcade - Updates: - Microsoft.DotNet.Arcade.Sdk: from 5.0.0-beta.20411.8 to 5.0.0-beta.20417.6 --- eng/Version.Details.xml | 4 +-- eng/common/cross/toolchain.cmake | 4 ++- .../templates/post-build/post-build.yml | 26 +++++++++---------- global.json | 2 +- 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 564548c2..6d1127c7 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - ecec08a0eebbd92bb9538e351d475582551d9092 + 83fda4b3f6d93e713749fd1b27c4a6d40b118b13 diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index 25667079..137736c0 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -19,7 +19,9 @@ if(TARGET_ARCH_NAME STREQUAL "armel") endif() elseif(TARGET_ARCH_NAME STREQUAL "arm") set(CMAKE_SYSTEM_PROCESSOR armv7l) - if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/armv6-alpine-linux-musleabihf) + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/armv7-alpine-linux-musleabihf) + set(TOOLCHAIN "armv7-alpine-linux-musleabihf") + elseif(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/armv6-alpine-linux-musleabihf) set(TOOLCHAIN "armv6-alpine-linux-musleabihf") else() set(TOOLCHAIN "arm-linux-gnueabihf") diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 0e79a546..e814d5e0 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -39,6 +39,7 @@ parameters: NetEngLatestChannelId: 2 NetEngValidationChannelId: 9 NetDev5ChannelId: 131 + NetDev6ChannelId: 1296 GeneralTestingChannelId: 529 NETCoreToolingDevChannelId: 548 NETCoreToolingReleaseChannelId: 549 @@ -46,7 +47,6 @@ parameters: NETCoreExperimentalChannelId: 562 NetEngServicesIntChannelId: 678 NetEngServicesProdChannelId: 679 - Net5Preview7ChannelId: 1065 Net5Preview8ChannelId: 1155 Net5RC1ChannelId: 1157 NetCoreSDK313xxChannelId: 759 @@ -115,7 +115,7 @@ stages: inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 arguments: -PromoteToChannels "$(TargetChannels)" - -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview7ChannelId}},${{parameters.Net5Preview8ChannelId}},${{parameters.Net5RC1ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}} + -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview8ChannelId}},${{parameters.Net5RC1ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}} - job: displayName: NuGet Validation @@ -276,13 +276,13 @@ stages: dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net5_Preview7_Publish' - channelName: '.NET 5 Preview 7' - akaMSChannelName: 'net5/preview7' - channelId: ${{ parameters.Net5Preview7ChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' + stageName: 'NetCore_Dev6_Publish' + channelName: '.NET 6 Dev' + akaMSChannelName: 'net6/dev' + channelId: ${{ parameters.NetDev6ChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6-symbols/nuget/v3/index.json' - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml parameters: @@ -298,7 +298,7 @@ stages: shippingFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-symbols/nuget/v3/index.json' - - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} @@ -308,9 +308,9 @@ stages: channelName: '.NET 5 RC 1' akaMSChannelName: 'net5/rc1' channelId: ${{ parameters.Net5RC1ChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-symbols/nuget/v3/index.json' + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: diff --git a/global.json b/global.json index 527a25d1..9811afd4 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "5.0.100-preview.6.20310.4" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20411.8" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20417.6" } } From 08c79710cd8010ab3e5b20feb4245220c5e7e54c Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 24 Aug 2020 12:26:49 +0000 Subject: [PATCH 092/161] Update dependencies from https://github.com/dotnet/arcade build 20200819.21 (#645) [master] Update dependencies from dotnet/arcade - Updates: - Microsoft.DotNet.Arcade.Sdk: from 5.0.0-beta.20417.6 to 5.0.0-beta.20419.21 --- eng/Version.Details.xml | 4 +- eng/common/performance/performance-setup.sh | 5 +- eng/common/post-build/publish-using-darc.ps1 | 23 +- .../channels/generic-internal-channel.yml | 6 + .../channels/generic-public-channel.yml | 6 + .../templates/post-build/post-build.yml | 395 ++++++++++-------- .../post-build/setup-maestro-vars.yml | 28 +- global.json | 2 +- 8 files changed, 268 insertions(+), 201 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 6d1127c7..b8859d34 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 83fda4b3f6d93e713749fd1b27c4a6d40b118b13 + 56a95cc477558c1ccdf16d7abe962849ea970ba4 diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index fd26bdb1..65a9d8fc 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -199,14 +199,15 @@ if [[ "$mono_dotnet" != "" ]]; then fi if [[ "$wasm_runtime_loc" != "" ]]; then - configurations="CompilationMode=wasm;RunKind=micro" + configurations="CompilationMode=wasm RunKind=micro" + extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter NoWASM" fi if [[ "$monointerpreter" == "true" ]]; then extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter" fi -common_setup_arguments="--channel master --queue $queue --build-number $build_number --build-configs \"$configurations\" --architecture $architecture" +common_setup_arguments="--channel master --queue $queue --build-number $build_number --build-configs $configurations --architecture $architecture" setup_arguments="--repository https://github.com/$repository --branch $branch --get-perf-hash --commit-sha $commit_sha $common_setup_arguments" diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1 index d22eb439..f044afe8 100644 --- a/eng/common/post-build/publish-using-darc.ps1 +++ b/eng/common/post-build/publish-using-darc.ps1 @@ -1,12 +1,13 @@ param( [Parameter(Mandatory=$true)][int] $BuildId, + [Parameter(Mandatory=$true)][int] $PublishingInfraVersion, [Parameter(Mandatory=$true)][string] $AzdoToken, [Parameter(Mandatory=$true)][string] $MaestroToken, [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com', [Parameter(Mandatory=$true)][string] $WaitPublishingFinish, - [Parameter(Mandatory=$true)][string] $EnableSourceLinkValidation, - [Parameter(Mandatory=$true)][string] $EnableSigningValidation, - [Parameter(Mandatory=$true)][string] $EnableNugetValidation, + [Parameter(Mandatory=$false)][string] $EnableSourceLinkValidation, + [Parameter(Mandatory=$false)][string] $EnableSigningValidation, + [Parameter(Mandatory=$false)][string] $EnableNugetValidation, [Parameter(Mandatory=$true)][string] $PublishInstallersAndChecksums, [Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters, [Parameter(Mandatory=$false)][string] $SigningValidationAdditionalParameters @@ -14,7 +15,8 @@ param( try { . $PSScriptRoot\post-build-utils.ps1 - . $PSScriptRoot\..\darc-init.ps1 + # Hard coding darc version till the next arcade-services roll out, cos this version has required API changes for darc add-build-to-channel + . $PSScriptRoot\..\darc-init.ps1 -darcVersion "1.1.0-beta.20418.1" $optionalParams = [System.Collections.ArrayList]::new() @@ -49,12 +51,13 @@ try { } & darc add-build-to-channel ` - --id $buildId ` - --default-channels ` - --source-branch master ` - --azdev-pat $AzdoToken ` - --bar-uri $MaestroApiEndPoint ` - --password $MaestroToken ` + --id $buildId ` + --publishing-infra-version $PublishingInfraVersion ` + --default-channels ` + --source-branch master ` + --azdev-pat $AzdoToken ` + --bar-uri $MaestroApiEndPoint ` + --password $MaestroToken ` @optionalParams if ($LastExitCode -ne 0) { diff --git a/eng/common/templates/post-build/channels/generic-internal-channel.yml b/eng/common/templates/post-build/channels/generic-internal-channel.yml index 12db36eb..59eb93a4 100644 --- a/eng/common/templates/post-build/channels/generic-internal-channel.yml +++ b/eng/common/templates/post-build/channels/generic-internal-channel.yml @@ -1,4 +1,6 @@ parameters: + BARBuildId: '' + PromoteToChannelIds: '' artifactsPublishingAdditionalParameters: '' dependsOn: - Validate @@ -19,6 +21,9 @@ stages: displayName: ${{ parameters.channelName }} Publishing jobs: - template: ../setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - job: publish_symbols displayName: Symbol Publishing @@ -138,6 +143,7 @@ stages: inputs: filePath: eng\common\sdk-task.ps1 arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet + /p:PublishingInfraVersion=2 /p:IsStableBuild=$(IsStableBuild) /p:IsInternalBuild=$(IsInternalBuild) /p:RepositoryName=$(Build.Repository.Name) diff --git a/eng/common/templates/post-build/channels/generic-public-channel.yml b/eng/common/templates/post-build/channels/generic-public-channel.yml index bf98d990..7e80a621 100644 --- a/eng/common/templates/post-build/channels/generic-public-channel.yml +++ b/eng/common/templates/post-build/channels/generic-public-channel.yml @@ -1,4 +1,6 @@ parameters: + BARBuildId: '' + PromoteToChannelIds: '' artifactsPublishingAdditionalParameters: '' dependsOn: - Validate @@ -21,6 +23,9 @@ stages: displayName: ${{ parameters.channelName }} Publishing jobs: - template: ../setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - job: publish_symbols displayName: Symbol Publishing @@ -137,6 +142,7 @@ stages: inputs: filePath: eng\common\sdk-task.ps1 arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet + /p:PublishingInfraVersion=2 /p:ArtifactsCategory=$(ArtifactsCategory) /p:IsStableBuild=$(IsStableBuild) /p:IsInternalBuild=$(IsInternalBuild) diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index e814d5e0..03080863 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -1,13 +1,20 @@ parameters: + # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST. + # Publishing V2 accepts optionally outlining the publishing stages - default is inline. + # Publishing V3 DOES NOT accept inlining the publishing stages. + publishingInfraVersion: 2 # When set to true the publishing templates from the repo will be used # otherwise Darc add-build-to-channel will be used to trigger the promotion pipeline inline: true # Only used if inline==false. When set to true will stall the current build until - # the Promotion Pipeline build finishes. Otherwise, the current build continue + # the Promotion Pipeline build finishes. Otherwise, the current build will continue # execution concurrently with the promotion build. waitPublishingFinish: true + BARBuildId: '' + PromoteToChannelIds: '' + enableSourceLinkValidation: false enableSigningValidation: true enableSymbolValidation: false @@ -59,14 +66,183 @@ parameters: VSMasterChannelId: 1012 stages: -- ${{ if ne(parameters.inline, 'true') }}: +- stage: Validate + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Validate Build Assets + variables: + - template: common-variables.yml + jobs: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - job: + displayName: Post-build Checks + dependsOn: setupMaestroVars + variables: + - name: TargetChannels + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ] + pool: + vmImage: 'windows-2019' + steps: + - task: PowerShell@2 + displayName: Maestro Channels Consistency + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 + arguments: -PromoteToChannels "$(TargetChannels)" + -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview8ChannelId}},${{parameters.Net5RC1ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}} + + - job: + displayName: NuGet Validation + dependsOn: setupMaestroVars + condition: eq( ${{ parameters.enableNugetValidation }}, 'true') + pool: + vmImage: 'windows-2019' + variables: + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ + -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ + + - job: + displayName: Signing Validation + dependsOn: setupMaestroVars + condition: eq( ${{ parameters.enableSigningValidation }}, 'true') + variables: + - template: common-variables.yml + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + pool: + vmImage: 'windows-2019' + steps: + - ${{ if eq(parameters.useBuildManifest, true) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download build manifest + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BuildManifests + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + + # This is necessary whenever we want to publish/restore to an AzDO private feed + # Since sdk-task.ps1 tries to restore packages we need to do this authentication here + # otherwise it'll complain about accessing a private feed. + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + + - task: PowerShell@2 + displayName: Enable cross-org publishing + inputs: + filePath: eng\common\enable-cross-org-publishing.ps1 + arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) + + # Signing validation will optionally work with the buildmanifest file which is downloaded from + # Azure DevOps above. + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task SigningValidation -restore -msbuildEngine vs + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' + /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' + ${{ parameters.signingValidationAdditionalParameters }} + + - template: ../steps/publish-logs.yml + parameters: + StageLabel: 'Validation' + JobLabel: 'Signing' + + - job: + displayName: SourceLink Validation + dependsOn: setupMaestroVars + condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') + variables: + - template: common-variables.yml + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BlobArtifacts + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) + -GHCommit $(Build.SourceVersion) + -SourcelinkCliVersion $(SourceLinkCLIVersion) + continueOnError: true + + - template: /eng/common/templates/job/execute-sdl.yml + parameters: + enable: ${{ parameters.SDLValidationParameters.enable }} + dependsOn: setupMaestroVars + additionalParameters: ${{ parameters.SDLValidationParameters.params }} + continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} + artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }} + downloadArtifacts: ${{ parameters.SDLValidationParameters.downloadArtifacts }} + +- ${{ if or(ge(parameters.publishingInfraVersion, 3), eq(parameters.inline, 'false')) }}: - stage: publish_using_darc - dependsOn: ${{ parameters.validateDependsOn }} + dependsOn: Validate displayName: Publish using Darc variables: - template: common-variables.yml jobs: - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - job: displayName: Publish Using Darc @@ -82,182 +258,17 @@ stages: inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 arguments: -BuildId $(BARBuildId) + -PublishingInfraVersion ${{ parameters.PublishingInfraVersion }} -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' -MaestroToken '$(MaestroApiAccessToken)' -WaitPublishingFinish ${{ parameters.waitPublishingFinish }} - -EnableSourceLinkValidation ${{ parameters.enableSourceLinkValidation }} - -EnableSigningValidation ${{ parameters.enableSourceLinkValidation }} - -EnableNugetValidation ${{ parameters.enableSourceLinkValidation }} -PublishInstallersAndChecksums ${{ parameters.publishInstallersAndChecksums }} - -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' - -SigningValidationAdditionalParameters '${{ parameters.signingValidationAdditionalParameters }}' - -- ${{ if eq(parameters.inline, 'true') }}: - - stage: Validate - dependsOn: ${{ parameters.validateDependsOn }} - displayName: Validate Build Assets - variables: - - template: common-variables.yml - jobs: - - template: setup-maestro-vars.yml - - - job: - displayName: Post-build Checks - dependsOn: setupMaestroVars - variables: - - name: TargetChannels - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ] - pool: - vmImage: 'windows-2019' - steps: - - task: PowerShell@2 - displayName: Maestro Channels Consistency - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 - arguments: -PromoteToChannels "$(TargetChannels)" - -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview8ChannelId}},${{parameters.Net5RC1ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}} - - - job: - displayName: NuGet Validation - dependsOn: setupMaestroVars - condition: eq( ${{ parameters.enableNugetValidation }}, 'true') - pool: - vmImage: 'windows-2019' - variables: - - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 - arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ - -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ - - - job: - displayName: Signing Validation - dependsOn: setupMaestroVars - condition: eq( ${{ parameters.enableSigningValidation }}, 'true') - variables: - - template: common-variables.yml - - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] - pool: - vmImage: 'windows-2019' - steps: - - ${{ if eq(parameters.useBuildManifest, true) }}: - - task: DownloadBuildArtifacts@0 - displayName: Download build manifest - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: BuildManifests - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - - # This is necessary whenever we want to publish/restore to an AzDO private feed - # Since sdk-task.ps1 tries to restore packages we need to do this authentication here - # otherwise it'll complain about accessing a private feed. - - task: NuGetAuthenticate@0 - displayName: 'Authenticate to AzDO Feeds' - - - task: PowerShell@2 - displayName: Enable cross-org publishing - inputs: - filePath: eng\common\enable-cross-org-publishing.ps1 - arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) - - # Signing validation will optionally work with the buildmanifest file which is downloaded from - # Azure DevOps above. - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task SigningValidation -restore -msbuildEngine vs - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' - /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' - ${{ parameters.signingValidationAdditionalParameters }} - - - template: ../steps/publish-logs.yml - parameters: - StageLabel: 'Validation' - JobLabel: 'Signing' - - - job: - displayName: SourceLink Validation - dependsOn: setupMaestroVars - condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') - variables: - - template: common-variables.yml - - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: BlobArtifacts - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -ExtractPath $(Agent.BuildDirectory)/Extract/ - -GHRepoName $(Build.Repository.Name) - -GHCommit $(Build.SourceVersion) - -SourcelinkCliVersion $(SourceLinkCLIVersion) - continueOnError: true - - - template: /eng/common/templates/job/execute-sdl.yml - parameters: - enable: ${{ parameters.SDLValidationParameters.enable }} - dependsOn: setupMaestroVars - additionalParameters: ${{ parameters.SDLValidationParameters.params }} - continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} - artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }} - downloadArtifacts: ${{ parameters.SDLValidationParameters.downloadArtifacts }} +- ${{ if and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')) }}: - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -272,6 +283,8 @@ stages: - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -286,6 +299,8 @@ stages: - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -300,6 +315,8 @@ stages: - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -314,6 +331,8 @@ stages: - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -328,6 +347,8 @@ stages: - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -342,6 +363,8 @@ stages: - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -356,6 +379,8 @@ stages: - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -369,6 +394,8 @@ stages: - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -382,6 +409,8 @@ stages: - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -395,6 +424,8 @@ stages: - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -408,6 +439,8 @@ stages: - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -421,6 +454,8 @@ stages: - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -434,6 +469,8 @@ stages: - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -447,6 +484,8 @@ stages: - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -460,6 +499,8 @@ stages: - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -473,6 +514,8 @@ stages: - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -486,6 +529,8 @@ stages: - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -499,6 +544,8 @@ stages: - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -512,6 +559,8 @@ stages: - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} @@ -525,6 +574,8 @@ stages: - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml index b3d29d44..d0cbfb6c 100644 --- a/eng/common/templates/post-build/setup-maestro-vars.yml +++ b/eng/common/templates/post-build/setup-maestro-vars.yml @@ -1,25 +1,23 @@ +parameters: + BARBuildId: '' + PromoteToChannelIds: '' + jobs: - job: setupMaestroVars displayName: Setup Maestro Vars variables: - template: common-variables.yml - - name: BuildId - value: $[ coalesce(variables.BARBuildId, 0) ] - - name: PromoteToMaestroChannels - value: $[ coalesce(variables.PromoteToChannelIds, 0) ] - - name: PromoteToMaestroChannel - value: $[ coalesce(variables.PromoteToMaestroChannelId, 0) ] pool: vmImage: 'windows-2019' steps: - checkout: none - - task: DownloadBuildArtifacts@0 - displayName: Download Release Configs - condition: and(eq(variables.PromoteToMaestroChannels, 0), eq(variables.PromoteToMaestroChannel, 0)) - inputs: - buildType: current - artifactName: ReleaseConfigs + - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Release Configs + inputs: + buildType: current + artifactName: ReleaseConfigs - task: PowerShell@2 name: setReleaseVars @@ -28,7 +26,7 @@ jobs: targetType: inline script: | try { - if ($Env:PromoteToMaestroChannels -eq 0 -and $Env:PromoteToMaestroChannel -eq 0) { + if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt $BarId = $Content | Select -Index 0 @@ -51,7 +49,7 @@ jobs: $BarId = $Env:BARBuildId $Channels = $Env:PromoteToMaestroChannels -split "," $Channels = $Channels -join "][" - $Channels = "[$Channels][$Env:PromoteToMaestroChannel]" + $Channels = "[$Channels]" $IsStableBuild = $buildInfo.stable $AzureDevOpsProject = $buildInfo.azureDevOpsProject @@ -75,3 +73,5 @@ jobs: } env: MAESTRO_API_TOKEN: $(MaestroApiAccessToken) + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} diff --git a/global.json b/global.json index 9811afd4..c718371e 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "5.0.100-preview.6.20310.4" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20417.6" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20419.21" } } From 1eec7504d1a577ac99c92c3b6b7a966abcc13afe Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2020 12:16:38 +0000 Subject: [PATCH 093/161] Update dependencies from https://github.com/dotnet/arcade build 20200827.5 (#646) [master] Update dependencies from dotnet/arcade - Updates: - Microsoft.DotNet.Arcade.Sdk: from 5.0.0-beta.20419.21 to 5.0.0-beta.20427.5 --- eng/Version.Details.xml | 4 +- eng/common/build.ps1 | 2 + eng/common/build.sh | 11 +- eng/common/cross/armel/tizen-build-rootfs.sh | 2 - eng/common/cross/armel/tizen/tizen.patch | 9 -- eng/common/performance/blazor_perf.proj | 30 +++++ eng/common/performance/crossgen_perf.proj | 114 ++++++++---------- eng/common/performance/microbenchmarks.proj | 4 +- eng/common/performance/performance-setup.sh | 32 +++-- .../templates/post-build/post-build.yml | 19 ++- eng/common/tools.ps1 | 20 +-- eng/common/tools.sh | 70 +++++------ global.json | 2 +- 13 files changed, 181 insertions(+), 138 deletions(-) create mode 100644 eng/common/performance/blazor_perf.proj diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index b8859d34..db8de310 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 56a95cc477558c1ccdf16d7abe962849ea970ba4 + f2b7fe854a0b1f78c04dfc065164d6d61040f5b8 diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 67ee6d28..94a91c08 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -23,6 +23,8 @@ Param( [switch][Alias('nobl')]$excludeCIBinarylog, [switch] $ci, [switch] $prepareMachine, + [string] $runtimeSourceFeed = '', + [string] $runtimeSourceFeedKey = '', [switch] $help, [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties ) diff --git a/eng/common/build.sh b/eng/common/build.sh index 6d7c5a1f..252b6360 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -76,9 +76,10 @@ projects='' configuration='Debug' prepare_machine=false verbosity='minimal' +runtime_source_feed='' +runtime_source_feed_key='' properties='' - while [[ $# > 0 ]]; do opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" case "$opt" in @@ -151,6 +152,14 @@ while [[ $# > 0 ]]; do node_reuse=$2 shift ;; + -runtimesourcefeed) + runtime_source_feed=$2 + shift + ;; + -runtimesourcefeedkey) + runtime_source_feed_key=$2 + shift + ;; *) properties="$properties $1" ;; diff --git a/eng/common/cross/armel/tizen-build-rootfs.sh b/eng/common/cross/armel/tizen-build-rootfs.sh index 25a0efa0..9a4438af 100755 --- a/eng/common/cross/armel/tizen-build-rootfs.sh +++ b/eng/common/cross/armel/tizen-build-rootfs.sh @@ -30,8 +30,6 @@ rm -rf $TIZEN_TMP_DIR # Configure Tizen rootfs echo ">>Start configuring Tizen rootfs" -rm ./usr/lib/libunwind.so -ln -s libunwind.so.8 ./usr/lib/libunwind.so ln -sfn asm-arm ./usr/include/asm patch -p1 < $__TIZEN_CROSSDIR/tizen.patch echo "< + + python3 + $(HelixPreCommands);chmod +x $HELIX_WORKITEM_PAYLOAD/SOD/SizeOnDisk + + + + + %(Identity) + + + + + %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\ + $(ScenarioDirectory)blazor\ + + + $HELIX_CORRELATION_PAYLOAD/performance/src/scenarios/ + $(ScenarioDirectory)blazor/ + + + + + $(WorkItemDirectory) + cd $(BlazorDirectory);$(Python) pre.py publish --msbuild %27/p:_TrimmerDumpDependencies=true%27 --msbuild-static AdditionalMonoLinkerOptions=%27"%24(AdditionalMonoLinkerOptions) --dump-dependencies"%27 --binlog %27./traces/blazor_publish.binlog%27 + $(Python) test.py sod --scenario-name "%(Identity)" + $(Python) post.py + + + \ No newline at end of file diff --git a/eng/common/performance/crossgen_perf.proj b/eng/common/performance/crossgen_perf.proj index 3c8c33d7..42649203 100644 --- a/eng/common/performance/crossgen_perf.proj +++ b/eng/common/performance/crossgen_perf.proj @@ -1,80 +1,68 @@ + + + %(Identity) + + + + py -3 - $(HelixPreCommands);call %HELIX_CORRELATION_PAYLOAD%\performance\tools\machine-setup.cmd;set PYTHONPATH=%HELIX_WORKITEM_PAYLOAD%\scripts%3B%HELIX_WORKITEM_PAYLOAD% - %HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts - $HELIX_CORRELATION_PAYLOAD - $(BaseDirectory)/performance + $(HelixPreCommands) + %HELIX_CORRELATION_PAYLOAD%\Core_Root + %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\ + $(ScenarioDirectory)crossgen\ + $(ScenarioDirectory)crossgen2\ - - - $(PerformanceDirectory)/scripts/benchmarks_ci.py --csproj $(PerformanceDirectory)/$(TargetCsproj) - --dotnet-versions $DOTNET_VERSION --cli-source-info args --cli-branch $PERFLAB_BRANCH --cli-commit-sha $PERFLAB_HASH --cli-repository https://github.com/$PERFLAB_REPO --cli-source-timestamp $PERFLAB_BUILDTIMESTAMP + python3 - $(BaseDirectory)/Core_Root/corerun - $(HelixPreCommands);chmod +x $(PerformanceDirectory)/tools/machine-setup.sh;. $(PerformanceDirectory)/tools/machine-setup.sh - $(BaseDirectory)/artifacts/BenchmarkDotNet.Artifacts - $(BaseDirectory)/artifacts/BenchmarkDotNet.Artifacts_Baseline - $(PerformanceDirectory)/src/tools/ResultsComparer/ResultsComparer.csproj - $(PerformanceDirectory)/tools/dotnet/$(Architecture)/dotnet - %25 - $HELIX_WORKITEM_ROOT/testResults.xml - + $(HelixPreCommands);chmod +x $HELIX_WORKITEM_PAYLOAD/startup/Startup;chmod +x $HELIX_WORKITEM_PAYLOAD/startup/perfcollect;sudo apt update + $HELIX_CORRELATION_PAYLOAD/Core_Root + $HELIX_CORRELATION_PAYLOAD/performance/src/scenarios/ + $(ScenarioDirectory)crossgen/ + $(ScenarioDirectory)crossgen2/ + - - %(Identity) - + + + + + + + + - - - $(WorkItemDirectory)\ScenarioCorrelation - $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name System.Private.Xml.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root - - - $(WorkItemDirectory)\ScenarioCorrelation - $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name System.Linq.Expressions.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root - - - $(WorkItemDirectory)\ScenarioCorrelation - $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name Microsoft.CodeAnalysis.VisualBasic.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root - - - $(WorkItemDirectory)\ScenarioCorrelation - $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name Microsoft.CodeAnalysis.CSharp.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root - - - $(WorkItemDirectory)\ScenarioCorrelation - $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name System.Private.CoreLib.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root - + + + $(WorkItemDirectory) + $(Python) $(CrossgenDirectory)test.py crossgen --core-root $(CoreRoot) --test-name %(Identity) + - - - $(WorkItemDirectory)\ScenarioCorrelation - $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single System.Private.Xml.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root - - - $(WorkItemDirectory)\ScenarioCorrelation - $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single System.Linq.Expressions.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root - - - $(WorkItemDirectory)\ScenarioCorrelation - $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single Microsoft.CodeAnalysis.VisualBasic.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root - - - $(WorkItemDirectory)\ScenarioCorrelation - $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single Microsoft.CodeAnalysis.CSharp.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + $(WorkItemDirectory) + $(Python) $(Crossgen2Directory)test.py crossgen2 --core-root $(CoreRoot) --single %(Identity) + + + + + + + 4:00 - - $(WorkItemDirectory)\ScenarioCorrelation - $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single System.Private.CoreLib.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + 4:00 - - $(WorkItemDirectory)\ScenarioCorrelation - $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --composite %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\framework-r2r.dll.rsp --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + $(WorkItemDirectory) + $(Python) $(Crossgen2Directory)test.py crossgen2 --core-root $(CoreRoot) --composite $(Crossgen2Directory)framework-r2r.dll.rsp 1:00 diff --git a/eng/common/performance/microbenchmarks.proj b/eng/common/performance/microbenchmarks.proj index 5c95ef45..94b6efbc 100644 --- a/eng/common/performance/microbenchmarks.proj +++ b/eng/common/performance/microbenchmarks.proj @@ -46,10 +46,10 @@ - --corerun %HELIX_CORRELATION_PAYLOAD%\dotnet-mono\shared\Microsoft.NETCore.App\5.0.0\corerun.exe + --corerun %HELIX_CORRELATION_PAYLOAD%\dotnet-mono\shared\Microsoft.NETCore.App\6.0.0\corerun.exe - --corerun $(BaseDirectory)/dotnet-mono/shared/Microsoft.NETCore.App/5.0.0/corerun + --corerun $(BaseDirectory)/dotnet-mono/shared/Microsoft.NETCore.App/6.0.0/corerun diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index 65a9d8fc..806e56c6 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -26,6 +26,7 @@ use_baseline_core_run=true using_mono=false wasm_runtime_loc= using_wasm=false +use_latest_dotnet=false while (($# > 0)); do lowerI="$(echo $1 | awk '{print tolower($0)}')" @@ -115,7 +116,11 @@ while (($# > 0)); do configurations=$2 shift 2 ;; - --help) + --latestdotnet) + use_latest_dotnet=true + shift 1 + ;; + *) echo "Common settings:" echo " --corerootdirectory Directory where Core_Root exists, if running perf testing with --corerun" echo " --architecture Architecture of the testing being run" @@ -137,6 +142,7 @@ while (($# > 0)); do echo " --internal If the benchmarks are running as an official job." echo " --monodotnet Pass the path to the mono dotnet for mono performance testing." echo " --wasm Path to the unpacked wasm runtime pack." + echo " --latestdotnet --dotnet-versions will not be specified. --dotnet-versions defaults to LKG version in global.json " echo "" exit 0 ;; @@ -194,28 +200,30 @@ if [[ "$internal" == true ]]; then fi fi -if [[ "$mono_dotnet" != "" ]]; then +if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "false" ]]; then configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot" + extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoMono" fi if [[ "$wasm_runtime_loc" != "" ]]; then - configurations="CompilationMode=wasm RunKind=micro" - extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter NoWASM" + configurations="CompilationMode=wasm RunKind=$kind" + extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter NoWASM NoMono" fi -if [[ "$monointerpreter" == "true" ]]; then - extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter" +if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "true" ]]; then + extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter NoMono" fi common_setup_arguments="--channel master --queue $queue --build-number $build_number --build-configs $configurations --architecture $architecture" setup_arguments="--repository https://github.com/$repository --branch $branch --get-perf-hash --commit-sha $commit_sha $common_setup_arguments" -# Get the tools section from the global.json. -# This grabs the LKG version number of dotnet and passes it to our scripts -dotnet_version=`cat global.json | python3 -c 'import json,sys;obj=json.load(sys.stdin);print(obj["tools"]["dotnet"])'` -setup_arguments="--dotnet-versions $dotnet_version $setup_arguments" - +if [[ "$use_latest_dotnet" = false ]]; then + # Get the tools section from the global.json. + # This grabs the LKG version number of dotnet and passes it to our scripts + dotnet_version=`cat global.json | python3 -c 'import json,sys;obj=json.load(sys.stdin);print(obj["tools"]["dotnet"])'` + setup_arguments="--dotnet-versions $dotnet_version $setup_arguments" +fi if [[ "$run_from_perf_repo" = true ]]; then payload_directory= @@ -266,7 +274,7 @@ Write-PipelineSetVariable -name "PerformanceDirectory" -value "$performance_dire Write-PipelineSetVariable -name "WorkItemDirectory" -value "$workitem_directory" -is_multi_job_variable false Write-PipelineSetVariable -name "Queue" -value "$queue" -is_multi_job_variable false Write-PipelineSetVariable -name "SetupArguments" -value "$setup_arguments" -is_multi_job_variable false -Write-PipelineSetVariable -name "Python" -value "$python3" -is_multi_job_variable false +Write-PipelineSetVariable -name "Python" -value "python3" -is_multi_job_variable false Write-PipelineSetVariable -name "PerfLabArguments" -value "$perflab_arguments" -is_multi_job_variable false Write-PipelineSetVariable -name "ExtraBenchmarkDotNetArguments" -value "$extra_benchmark_dotnet_arguments" -is_multi_job_variable false Write-PipelineSetVariable -name "BDNCategories" -value "$run_categories" -is_multi_job_variable false diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 03080863..630a99d4 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -56,6 +56,7 @@ parameters: NetEngServicesProdChannelId: 679 Net5Preview8ChannelId: 1155 Net5RC1ChannelId: 1157 + Net5RC2ChannelId: 1329 NetCoreSDK313xxChannelId: 759 NetCoreSDK313xxInternalChannelId: 760 NetCoreSDK314xxChannelId: 921 @@ -91,7 +92,7 @@ stages: inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 arguments: -PromoteToChannels "$(TargetChannels)" - -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview8ChannelId}},${{parameters.Net5RC1ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}} + -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview8ChannelId}},${{parameters.Net5RC1ChannelId}},${{parameters.Net5RC2ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}} - job: displayName: NuGet Validation @@ -329,6 +330,22 @@ stages: shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'Net5_RC2_Publish' + channelName: '.NET 5 RC 2' + akaMSChannelName: 'net5/rc2' + channelId: ${{ parameters.Net5RC2ChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: BARBuildId: ${{ parameters.BARBuildId }} diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 6f8bc411..c2e575c9 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -57,6 +57,11 @@ set-strictmode -version 2.0 $ErrorActionPreference = 'Stop' [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +# If specifies, provides an alternate path for getting .NET Core SDKs and Runtimes. This script will still try public sources first. +[string]$runtimeSourceFeed = if (Test-Path variable:runtimeSourceFeed) { $runtimeSourceFeed } else { $null } +# Base-64 encoded SAS token that has permission to storage container described by $runtimeSourceFeed +[string]$runtimeSourceFeedKey = if (Test-Path variable:runtimeSourceFeedKey) { $runtimeSourceFeedKey } else { $null } + function Create-Directory ([string[]] $path) { New-Item -Path $path -Force -ItemType 'Directory' | Out-Null } @@ -223,7 +228,7 @@ function GetDotNetInstallScript([string] $dotnetRoot) { } function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $architecture = '') { - InstallDotNet $dotnetRoot $version $architecture + InstallDotNet $dotnetRoot $version $architecture '' $false $runtimeSourceFeed $runtimeSourceFeedKey } function InstallDotNet([string] $dotnetRoot, @@ -248,10 +253,8 @@ function InstallDotNet([string] $dotnetRoot, & $installScript @installParameters } catch { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet runtime '$runtime' from public location." - - # Only the runtime can be installed from a custom [private] location. - if ($runtime -and ($runtimeSourceFeed -or $runtimeSourceFeedKey)) { + if ($runtimeSourceFeed -or $runtimeSourceFeedKey) { + Write-Host "Failed to install dotnet from public location. Trying from '$runtimeSourceFeed'" if ($runtimeSourceFeed) { $installParameters.AzureFeed = $runtimeSourceFeed } if ($runtimeSourceFeedKey) { @@ -264,10 +267,11 @@ function InstallDotNet([string] $dotnetRoot, & $installScript @installParameters } catch { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet runtime '$runtime' from custom location '$runtimeSourceFeed'." + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from custom location '$runtimeSourceFeed'." ExitWithExitCode 1 } } else { + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from public location." ExitWithExitCode 1 } } @@ -602,11 +606,7 @@ function MSBuild() { if ($pipelinesLog) { $buildTool = InitializeBuildTool - # Work around issues with Azure Artifacts credential provider - # https://github.com/dotnet/arcade/issues/3932 if ($ci -and $buildTool.Tool -eq 'dotnet') { - dotnet nuget locals http-cache -c - $env:NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS = 20 $env:NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS = 20 Write-PipelineSetVariable -Name 'NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS' -Value '20' diff --git a/eng/common/tools.sh b/eng/common/tools.sh index e94fce22..2be5cba3 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -64,6 +64,10 @@ else use_global_nuget_cache=${use_global_nuget_cache:-true} fi +# Used when restoring .NET SDK from alternative feeds +runtime_source_feed=${runtime_source_feed:-''} +runtime_source_feed_key=${runtime_source_feed_key:-''} + # Resolve any symlinks in the given path. function ResolvePath { local path=$1 @@ -170,11 +174,11 @@ function InitializeDotNetCli { function InstallDotNetSdk { local root=$1 local version=$2 - local architecture="" - if [[ $# == 3 ]]; then + local architecture="unset" + if [[ $# -ge 3 ]]; then architecture=$3 fi - InstallDotNet "$root" "$version" $architecture + InstallDotNet "$root" "$version" $architecture 'sdk' 'false' $runtime_source_feed $runtime_source_feed_key } function InstallDotNet { @@ -185,50 +189,50 @@ function InstallDotNet { local install_script=$_GetDotNetInstallScript local archArg='' - if [[ -n "${3:-}" ]]; then + if [[ -n "${3:-}" ]] && [ "$3" != 'unset' ]; then archArg="--architecture $3" fi local runtimeArg='' - if [[ -n "${4:-}" ]]; then + if [[ -n "${4:-}" ]] && [ "$4" != 'sdk' ]; then runtimeArg="--runtime $4" fi - local skipNonVersionedFilesArg="" - if [[ "$#" -ge "5" ]]; then + if [[ "$#" -ge "5" ]] && [[ "$5" != 'false' ]]; then skipNonVersionedFilesArg="--skip-non-versioned-files" fi bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg || { local exit_code=$? - Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from public location (exit code '$exit_code')." + echo "Failed to install dotnet SDK from public location (exit code '$exit_code')." - if [[ -n "$runtimeArg" ]]; then - local runtimeSourceFeed='' - if [[ -n "${6:-}" ]]; then - runtimeSourceFeed="--azure-feed $6" - fi + local runtimeSourceFeed='' + if [[ -n "${6:-}" ]]; then + runtimeSourceFeed="--azure-feed $6" + fi - local runtimeSourceFeedKey='' - if [[ -n "${7:-}" ]]; then - # The 'base64' binary on alpine uses '-d' and doesn't support '--decode' - # '-d'. To work around this, do a simple detection and switch the parameter - # accordingly. - decodeArg="--decode" - if base64 --help 2>&1 | grep -q "BusyBox"; then - decodeArg="-d" - fi - decodedFeedKey=`echo $7 | base64 $decodeArg` - runtimeSourceFeedKey="--feed-credential $decodedFeedKey" + local runtimeSourceFeedKey='' + if [[ -n "${7:-}" ]]; then + # The 'base64' binary on alpine uses '-d' and doesn't support '--decode' + # '-d'. To work around this, do a simple detection and switch the parameter + # accordingly. + decodeArg="--decode" + if base64 --help 2>&1 | grep -q "BusyBox"; then + decodeArg="-d" fi + decodedFeedKey=`echo $7 | base64 $decodeArg` + runtimeSourceFeedKey="--feed-credential $decodedFeedKey" + fi - if [[ -n "$runtimeSourceFeed" || -n "$runtimeSourceFeedKey" ]]; then - bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg $runtimeSourceFeed $runtimeSourceFeedKey || { - local exit_code=$? - Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from custom location '$runtimeSourceFeed' (exit code '$exit_code')." - ExitWithExitCode $exit_code - } - else + if [[ -n "$runtimeSourceFeed" || -n "$runtimeSourceFeedKey" ]]; then + bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg $runtimeSourceFeed $runtimeSourceFeedKey || { + local exit_code=$? + Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from custom location '$runtimeSourceFeed' (exit code '$exit_code')." ExitWithExitCode $exit_code + } + else + if [[ $exit_code != 0 ]]; then + Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from public location (exit code '$exit_code')." fi + ExitWithExitCode $exit_code fi } } @@ -391,11 +395,7 @@ function MSBuild { InitializeBuildTool InitializeToolset - # Work around issues with Azure Artifacts credential provider - # https://github.com/dotnet/arcade/issues/3932 if [[ "$ci" == true ]]; then - "$_InitializeBuildTool" nuget locals http-cache -c - export NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS=20 export NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS=20 Write-PipelineSetVariable -name "NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS" -value "20" diff --git a/global.json b/global.json index c718371e..eb71a354 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "5.0.100-preview.6.20310.4" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20419.21" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20427.5" } } From e9d917c21e38e274a413a207d9e84eb80382b56f Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 7 Sep 2020 12:21:45 +0000 Subject: [PATCH 094/161] Update dependencies from https://github.com/dotnet/arcade build 20200903.7 (#648) [master] Update dependencies from dotnet/arcade - Updates: - Microsoft.DotNet.Arcade.Sdk: from 5.0.0-beta.20427.5 to 5.0.0-beta.20453.7 --- eng/Version.Details.xml | 4 +- eng/common/build.ps1 | 2 + eng/common/build.sh | 9 +- .../dotnet-install-scripts/dotnet-install.ps1 | 774 +++++++++++ .../dotnet-install-scripts/dotnet-install.sh | 1133 +++++++++++++++++ eng/common/internal/Directory.Build.props | 2 +- eng/common/post-build/publish-using-darc.ps1 | 8 +- .../templates/job/publish-build-assets.yml | 12 +- .../channels/generic-internal-channel.yml | 4 +- .../channels/generic-public-channel.yml | 2 +- .../templates/post-build/post-build.yml | 2 +- eng/common/templates/steps/publish-logs.yml | 2 +- eng/common/tools.ps1 | 89 +- eng/common/tools.sh | 37 +- global.json | 2 +- 15 files changed, 2015 insertions(+), 67 deletions(-) create mode 100644 eng/common/dotnet-install-scripts/dotnet-install.ps1 create mode 100644 eng/common/dotnet-install-scripts/dotnet-install.sh diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index db8de310..ee6e3fd3 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - f2b7fe854a0b1f78c04dfc065164d6d61040f5b8 + 16b71a2f216c3c5be5860977c4cb03a95ee2f0e3 diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 94a91c08..1fd7f686 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -7,6 +7,7 @@ Param( [string] $msbuildEngine = $null, [bool] $warnAsError = $true, [bool] $nodeReuse = $true, + [bool] $useDefaultDotnetInstall = $false, [switch][Alias('r')]$restore, [switch] $deployDeps, [switch][Alias('b')]$build, @@ -65,6 +66,7 @@ function Print-Usage() { Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build" Write-Host " -warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." + Write-Host " -useDefaultDotnetInstall Use dotnet-install.* scripts from public location as opposed to from eng common folder" Write-Host "" Write-Host "Command line arguments not listed above are passed thru to msbuild." diff --git a/eng/common/build.sh b/eng/common/build.sh index 252b6360..19849adb 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -36,6 +36,8 @@ usage() echo " --prepareMachine Prepare machine for CI run, clean up processes after build" echo " --nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')" echo " --warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" + echo " --useDefaultDotnetInstall Use dotnet-install.* scripts from public location as opposed to from eng common folder" + echo "" echo "Command line arguments not listed above are passed thru to msbuild." echo "Arguments can also be passed in with a single hyphen." @@ -78,6 +80,7 @@ prepare_machine=false verbosity='minimal' runtime_source_feed='' runtime_source_feed_key='' +use_default_dotnet_install=false properties='' while [[ $# > 0 ]]; do @@ -156,10 +159,14 @@ while [[ $# > 0 ]]; do runtime_source_feed=$2 shift ;; - -runtimesourcefeedkey) + -runtimesourcefeedkey) runtime_source_feed_key=$2 shift ;; + -usedefaultdotnetinstall) + use_default_dotnet_install=$2 + shift + ;; *) properties="$properties $1" ;; diff --git a/eng/common/dotnet-install-scripts/dotnet-install.ps1 b/eng/common/dotnet-install-scripts/dotnet-install.ps1 new file mode 100644 index 00000000..f63b533f --- /dev/null +++ b/eng/common/dotnet-install-scripts/dotnet-install.ps1 @@ -0,0 +1,774 @@ +# +# Copyright (c) .NET Foundation and contributors. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. +# + +# Copied from https://dot.net/v1/dotnet-install.ps1 on 8/26/2020 + +<# +.SYNOPSIS + Installs dotnet cli +.DESCRIPTION + Installs dotnet cli. If dotnet installation already exists in the given directory + it will update it only if the requested version differs from the one already installed. +.PARAMETER Channel + Default: LTS + Download from the Channel specified. Possible values: + - Current - most current release + - LTS - most current supported release + - 2-part version in a format A.B - represents a specific release + examples: 2.0, 1.0 + - Branch name + examples: release/2.0.0, Master + Note: The version parameter overrides the channel parameter. +.PARAMETER Version + Default: latest + Represents a build version on specific channel. Possible values: + - latest - most latest build on specific channel + - coherent - most latest coherent build on specific channel + coherent applies only to SDK downloads + - 3-part version in a format A.B.C - represents specific version of build + examples: 2.0.0-preview2-006120, 1.1.0 +.PARAMETER InstallDir + Default: %LocalAppData%\Microsoft\dotnet + Path to where to install dotnet. Note that binaries will be placed directly in a given directory. +.PARAMETER Architecture + Default: - this value represents currently running OS architecture + Architecture of dotnet binaries to be installed. + Possible values are: , amd64, x64, x86, arm64, arm +.PARAMETER SharedRuntime + This parameter is obsolete and may be removed in a future version of this script. + The recommended alternative is '-Runtime dotnet'. + Installs just the shared runtime bits, not the entire SDK. +.PARAMETER Runtime + Installs just a shared runtime, not the entire SDK. + Possible values: + - dotnet - the Microsoft.NETCore.App shared runtime + - aspnetcore - the Microsoft.AspNetCore.App shared runtime + - windowsdesktop - the Microsoft.WindowsDesktop.App shared runtime +.PARAMETER DryRun + If set it will not perform installation but instead display what command line to use to consistently install + currently requested version of dotnet cli. In example if you specify version 'latest' it will display a link + with specific version so that this command can be used deterministicly in a build script. + It also displays binaries location if you prefer to install or download it yourself. +.PARAMETER NoPath + By default this script will set environment variable PATH for the current process to the binaries folder inside installation folder. + If set it will display binaries location but not set any environment variable. +.PARAMETER Verbose + Displays diagnostics information. +.PARAMETER AzureFeed + Default: https://dotnetcli.azureedge.net/dotnet + This parameter typically is not changed by the user. + It allows changing the URL for the Azure feed used by this installer. +.PARAMETER UncachedFeed + This parameter typically is not changed by the user. + It allows changing the URL for the Uncached feed used by this installer. +.PARAMETER FeedCredential + Used as a query string to append to the Azure feed. + It allows changing the URL to use non-public blob storage accounts. +.PARAMETER ProxyAddress + If set, the installer will use the proxy when making web requests +.PARAMETER ProxyUseDefaultCredentials + Default: false + Use default credentials, when using proxy address. +.PARAMETER ProxyBypassList + If set with ProxyAddress, will provide the list of comma separated urls that will bypass the proxy +.PARAMETER SkipNonVersionedFiles + Default: false + Skips installing non-versioned files if they already exist, such as dotnet.exe. +.PARAMETER NoCdn + Disable downloading from the Azure CDN, and use the uncached feed directly. +.PARAMETER JSonFile + Determines the SDK version from a user specified global.json file + Note: global.json must have a value for 'SDK:Version' +#> +[cmdletbinding()] +param( + [string]$Channel="LTS", + [string]$Version="Latest", + [string]$JSonFile, + [string]$InstallDir="", + [string]$Architecture="", + [ValidateSet("dotnet", "aspnetcore", "windowsdesktop", IgnoreCase = $false)] + [string]$Runtime, + [Obsolete("This parameter may be removed in a future version of this script. The recommended alternative is '-Runtime dotnet'.")] + [switch]$SharedRuntime, + [switch]$DryRun, + [switch]$NoPath, + [string]$AzureFeed="https://dotnetcli.azureedge.net/dotnet", + [string]$UncachedFeed="https://dotnetcli.blob.core.windows.net/dotnet", + [string]$FeedCredential, + [string]$ProxyAddress, + [switch]$ProxyUseDefaultCredentials, + [string[]]$ProxyBypassList=@(), + [switch]$SkipNonVersionedFiles, + [switch]$NoCdn +) + +Set-StrictMode -Version Latest +$ErrorActionPreference="Stop" +$ProgressPreference="SilentlyContinue" + +if ($NoCdn) { + $AzureFeed = $UncachedFeed +} + +$BinFolderRelativePath="" + +if ($SharedRuntime -and (-not $Runtime)) { + $Runtime = "dotnet" +} + +# example path with regex: shared/1.0.0-beta-12345/somepath +$VersionRegEx="/\d+\.\d+[^/]+/" +$OverrideNonVersionedFiles = !$SkipNonVersionedFiles + +function Say($str) { + try + { + Write-Host "dotnet-install: $str" + } + catch + { + # Some platforms cannot utilize Write-Host (Azure Functions, for instance). Fall back to Write-Output + Write-Output "dotnet-install: $str" + } +} + +function Say-Verbose($str) { + try + { + Write-Verbose "dotnet-install: $str" + } + catch + { + # Some platforms cannot utilize Write-Verbose (Azure Functions, for instance). Fall back to Write-Output + Write-Output "dotnet-install: $str" + } +} + +function Say-Invocation($Invocation) { + $command = $Invocation.MyCommand; + $args = (($Invocation.BoundParameters.Keys | foreach { "-$_ `"$($Invocation.BoundParameters[$_])`"" }) -join " ") + Say-Verbose "$command $args" +} + +function Invoke-With-Retry([ScriptBlock]$ScriptBlock, [int]$MaxAttempts = 3, [int]$SecondsBetweenAttempts = 1) { + $Attempts = 0 + + while ($true) { + try { + return $ScriptBlock.Invoke() + } + catch { + $Attempts++ + if ($Attempts -lt $MaxAttempts) { + Start-Sleep $SecondsBetweenAttempts + } + else { + throw + } + } + } +} + +function Get-Machine-Architecture() { + Say-Invocation $MyInvocation + + # On PS x86, PROCESSOR_ARCHITECTURE reports x86 even on x64 systems. + # To get the correct architecture, we need to use PROCESSOR_ARCHITEW6432. + # PS x64 doesn't define this, so we fall back to PROCESSOR_ARCHITECTURE. + # Possible values: amd64, x64, x86, arm64, arm + + if( $ENV:PROCESSOR_ARCHITEW6432 -ne $null ) + { + return $ENV:PROCESSOR_ARCHITEW6432 + } + + return $ENV:PROCESSOR_ARCHITECTURE +} + +function Get-CLIArchitecture-From-Architecture([string]$Architecture) { + Say-Invocation $MyInvocation + + switch ($Architecture.ToLower()) { + { $_ -eq "" } { return Get-CLIArchitecture-From-Architecture $(Get-Machine-Architecture) } + { ($_ -eq "amd64") -or ($_ -eq "x64") } { return "x64" } + { $_ -eq "x86" } { return "x86" } + { $_ -eq "arm" } { return "arm" } + { $_ -eq "arm64" } { return "arm64" } + default { throw "Architecture not supported. If you think this is a bug, report it at https://github.com/dotnet/sdk/issues" } + } +} + +# The version text returned from the feeds is a 1-line or 2-line string: +# For the SDK and the dotnet runtime (2 lines): +# Line 1: # commit_hash +# Line 2: # 4-part version +# For the aspnetcore runtime (1 line): +# Line 1: # 4-part version +function Get-Version-Info-From-Version-Text([string]$VersionText) { + Say-Invocation $MyInvocation + + $Data = -split $VersionText + + $VersionInfo = @{ + CommitHash = $(if ($Data.Count -gt 1) { $Data[0] }) + Version = $Data[-1] # last line is always the version number. + } + return $VersionInfo +} + +function Load-Assembly([string] $Assembly) { + try { + Add-Type -Assembly $Assembly | Out-Null + } + catch { + # On Nano Server, Powershell Core Edition is used. Add-Type is unable to resolve base class assemblies because they are not GAC'd. + # Loading the base class assemblies is not unnecessary as the types will automatically get resolved. + } +} + +function GetHTTPResponse([Uri] $Uri) +{ + Invoke-With-Retry( + { + + $HttpClient = $null + + try { + # HttpClient is used vs Invoke-WebRequest in order to support Nano Server which doesn't support the Invoke-WebRequest cmdlet. + Load-Assembly -Assembly System.Net.Http + + if(-not $ProxyAddress) { + try { + # Despite no proxy being explicitly specified, we may still be behind a default proxy + $DefaultProxy = [System.Net.WebRequest]::DefaultWebProxy; + if($DefaultProxy -and (-not $DefaultProxy.IsBypassed($Uri))) { + $ProxyAddress = $DefaultProxy.GetProxy($Uri).OriginalString + $ProxyUseDefaultCredentials = $true + } + } catch { + # Eat the exception and move forward as the above code is an attempt + # at resolving the DefaultProxy that may not have been a problem. + $ProxyAddress = $null + Say-Verbose("Exception ignored: $_.Exception.Message - moving forward...") + } + } + + if($ProxyAddress) { + $HttpClientHandler = New-Object System.Net.Http.HttpClientHandler + $HttpClientHandler.Proxy = New-Object System.Net.WebProxy -Property @{ + Address=$ProxyAddress; + UseDefaultCredentials=$ProxyUseDefaultCredentials; + BypassList = $ProxyBypassList; + } + $HttpClient = New-Object System.Net.Http.HttpClient -ArgumentList $HttpClientHandler + } + else { + + $HttpClient = New-Object System.Net.Http.HttpClient + } + # Default timeout for HttpClient is 100s. For a 50 MB download this assumes 500 KB/s average, any less will time out + # 20 minutes allows it to work over much slower connections. + $HttpClient.Timeout = New-TimeSpan -Minutes 20 + $Response = $HttpClient.GetAsync("${Uri}${FeedCredential}").Result + if (($Response -eq $null) -or (-not ($Response.IsSuccessStatusCode))) { + # The feed credential is potentially sensitive info. Do not log FeedCredential to console output. + $ErrorMsg = "Failed to download $Uri." + if ($Response -ne $null) { + $ErrorMsg += " $Response" + } + + throw $ErrorMsg + } + + return $Response + } + finally { + if ($HttpClient -ne $null) { + $HttpClient.Dispose() + } + } + }) +} + +function Get-Latest-Version-Info([string]$AzureFeed, [string]$Channel, [bool]$Coherent) { + Say-Invocation $MyInvocation + + $VersionFileUrl = $null + if ($Runtime -eq "dotnet") { + $VersionFileUrl = "$UncachedFeed/Runtime/$Channel/latest.version" + } + elseif ($Runtime -eq "aspnetcore") { + $VersionFileUrl = "$UncachedFeed/aspnetcore/Runtime/$Channel/latest.version" + } + # Currently, the WindowsDesktop runtime is manufactured with the .Net core runtime + elseif ($Runtime -eq "windowsdesktop") { + $VersionFileUrl = "$UncachedFeed/Runtime/$Channel/latest.version" + } + elseif (-not $Runtime) { + if ($Coherent) { + $VersionFileUrl = "$UncachedFeed/Sdk/$Channel/latest.coherent.version" + } + else { + $VersionFileUrl = "$UncachedFeed/Sdk/$Channel/latest.version" + } + } + else { + throw "Invalid value for `$Runtime" + } + try { + $Response = GetHTTPResponse -Uri $VersionFileUrl + } + catch { + throw "Could not resolve version information." + } + $StringContent = $Response.Content.ReadAsStringAsync().Result + + switch ($Response.Content.Headers.ContentType) { + { ($_ -eq "application/octet-stream") } { $VersionText = $StringContent } + { ($_ -eq "text/plain") } { $VersionText = $StringContent } + { ($_ -eq "text/plain; charset=UTF-8") } { $VersionText = $StringContent } + default { throw "``$Response.Content.Headers.ContentType`` is an unknown .version file content type." } + } + + $VersionInfo = Get-Version-Info-From-Version-Text $VersionText + + return $VersionInfo +} + +function Parse-Jsonfile-For-Version([string]$JSonFile) { + Say-Invocation $MyInvocation + + If (-Not (Test-Path $JSonFile)) { + throw "Unable to find '$JSonFile'" + } + try { + $JSonContent = Get-Content($JSonFile) -Raw | ConvertFrom-Json | Select-Object -expand "sdk" -ErrorAction SilentlyContinue + } + catch { + throw "Json file unreadable: '$JSonFile'" + } + if ($JSonContent) { + try { + $JSonContent.PSObject.Properties | ForEach-Object { + $PropertyName = $_.Name + if ($PropertyName -eq "version") { + $Version = $_.Value + Say-Verbose "Version = $Version" + } + } + } + catch { + throw "Unable to parse the SDK node in '$JSonFile'" + } + } + else { + throw "Unable to find the SDK node in '$JSonFile'" + } + If ($Version -eq $null) { + throw "Unable to find the SDK:version node in '$JSonFile'" + } + return $Version +} + +function Get-Specific-Version-From-Version([string]$AzureFeed, [string]$Channel, [string]$Version, [string]$JSonFile) { + Say-Invocation $MyInvocation + + if (-not $JSonFile) { + switch ($Version.ToLower()) { + { $_ -eq "latest" } { + $LatestVersionInfo = Get-Latest-Version-Info -AzureFeed $AzureFeed -Channel $Channel -Coherent $False + return $LatestVersionInfo.Version + } + { $_ -eq "coherent" } { + $LatestVersionInfo = Get-Latest-Version-Info -AzureFeed $AzureFeed -Channel $Channel -Coherent $True + return $LatestVersionInfo.Version + } + default { return $Version } + } + } + else { + return Parse-Jsonfile-For-Version $JSonFile + } +} + +function Get-Download-Link([string]$AzureFeed, [string]$SpecificVersion, [string]$CLIArchitecture) { + Say-Invocation $MyInvocation + + # If anything fails in this lookup it will default to $SpecificVersion + $SpecificProductVersion = Get-Product-Version -AzureFeed $AzureFeed -SpecificVersion $SpecificVersion + + if ($Runtime -eq "dotnet") { + $PayloadURL = "$AzureFeed/Runtime/$SpecificVersion/dotnet-runtime-$SpecificProductVersion-win-$CLIArchitecture.zip" + } + elseif ($Runtime -eq "aspnetcore") { + $PayloadURL = "$AzureFeed/aspnetcore/Runtime/$SpecificVersion/aspnetcore-runtime-$SpecificProductVersion-win-$CLIArchitecture.zip" + } + elseif ($Runtime -eq "windowsdesktop") { + $PayloadURL = "$AzureFeed/Runtime/$SpecificVersion/windowsdesktop-runtime-$SpecificProductVersion-win-$CLIArchitecture.zip" + } + elseif (-not $Runtime) { + $PayloadURL = "$AzureFeed/Sdk/$SpecificVersion/dotnet-sdk-$SpecificProductVersion-win-$CLIArchitecture.zip" + } + else { + throw "Invalid value for `$Runtime" + } + + Say-Verbose "Constructed primary named payload URL: $PayloadURL" + + return $PayloadURL, $SpecificProductVersion +} + +function Get-LegacyDownload-Link([string]$AzureFeed, [string]$SpecificVersion, [string]$CLIArchitecture) { + Say-Invocation $MyInvocation + + if (-not $Runtime) { + $PayloadURL = "$AzureFeed/Sdk/$SpecificVersion/dotnet-dev-win-$CLIArchitecture.$SpecificVersion.zip" + } + elseif ($Runtime -eq "dotnet") { + $PayloadURL = "$AzureFeed/Runtime/$SpecificVersion/dotnet-win-$CLIArchitecture.$SpecificVersion.zip" + } + else { + return $null + } + + Say-Verbose "Constructed legacy named payload URL: $PayloadURL" + + return $PayloadURL +} + +function Get-Product-Version([string]$AzureFeed, [string]$SpecificVersion) { + Say-Invocation $MyInvocation + + if ($Runtime -eq "dotnet") { + $ProductVersionTxtURL = "$AzureFeed/Runtime/$SpecificVersion/productVersion.txt" + } + elseif ($Runtime -eq "aspnetcore") { + $ProductVersionTxtURL = "$AzureFeed/aspnetcore/Runtime/$SpecificVersion/productVersion.txt" + } + elseif ($Runtime -eq "windowsdesktop") { + $ProductVersionTxtURL = "$AzureFeed/Runtime/$SpecificVersion/productVersion.txt" + } + elseif (-not $Runtime) { + $ProductVersionTxtURL = "$AzureFeed/Sdk/$SpecificVersion/productVersion.txt" + } + else { + throw "Invalid value specified for `$Runtime" + } + + Say-Verbose "Checking for existence of $ProductVersionTxtURL" + + try { + $productVersionResponse = GetHTTPResponse($productVersionTxtUrl) + + if ($productVersionResponse.StatusCode -eq 200) { + $productVersion = $productVersionResponse.Content.ReadAsStringAsync().Result.Trim() + if ($productVersion -ne $SpecificVersion) + { + Say "Using alternate version $productVersion found in $ProductVersionTxtURL" + } + + return $productVersion + } + else { + Say-Verbose "Got StatusCode $($productVersionResponse.StatusCode) trying to get productVersion.txt at $productVersionTxtUrl, so using default value of $SpecificVersion" + $productVersion = $SpecificVersion + } + } catch { + Say-Verbose "Could not read productVersion.txt at $productVersionTxtUrl, so using default value of $SpecificVersion" + $productVersion = $SpecificVersion + } + + return $productVersion +} + +function Get-User-Share-Path() { + Say-Invocation $MyInvocation + + $InstallRoot = $env:DOTNET_INSTALL_DIR + if (!$InstallRoot) { + $InstallRoot = "$env:LocalAppData\Microsoft\dotnet" + } + return $InstallRoot +} + +function Resolve-Installation-Path([string]$InstallDir) { + Say-Invocation $MyInvocation + + if ($InstallDir -eq "") { + return Get-User-Share-Path + } + return $InstallDir +} + +function Is-Dotnet-Package-Installed([string]$InstallRoot, [string]$RelativePathToPackage, [string]$SpecificVersion) { + Say-Invocation $MyInvocation + + $DotnetPackagePath = Join-Path -Path $InstallRoot -ChildPath $RelativePathToPackage | Join-Path -ChildPath $SpecificVersion + Say-Verbose "Is-Dotnet-Package-Installed: DotnetPackagePath=$DotnetPackagePath" + return Test-Path $DotnetPackagePath -PathType Container +} + +function Get-Absolute-Path([string]$RelativeOrAbsolutePath) { + # Too much spam + # Say-Invocation $MyInvocation + + return $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($RelativeOrAbsolutePath) +} + +function Get-Path-Prefix-With-Version($path) { + $match = [regex]::match($path, $VersionRegEx) + if ($match.Success) { + return $entry.FullName.Substring(0, $match.Index + $match.Length) + } + + return $null +} + +function Get-List-Of-Directories-And-Versions-To-Unpack-From-Dotnet-Package([System.IO.Compression.ZipArchive]$Zip, [string]$OutPath) { + Say-Invocation $MyInvocation + + $ret = @() + foreach ($entry in $Zip.Entries) { + $dir = Get-Path-Prefix-With-Version $entry.FullName + if ($dir -ne $null) { + $path = Get-Absolute-Path $(Join-Path -Path $OutPath -ChildPath $dir) + if (-Not (Test-Path $path -PathType Container)) { + $ret += $dir + } + } + } + + $ret = $ret | Sort-Object | Get-Unique + + $values = ($ret | foreach { "$_" }) -join ";" + Say-Verbose "Directories to unpack: $values" + + return $ret +} + +# Example zip content and extraction algorithm: +# Rule: files if extracted are always being extracted to the same relative path locally +# .\ +# a.exe # file does not exist locally, extract +# b.dll # file exists locally, override only if $OverrideFiles set +# aaa\ # same rules as for files +# ... +# abc\1.0.0\ # directory contains version and exists locally +# ... # do not extract content under versioned part +# abc\asd\ # same rules as for files +# ... +# def\ghi\1.0.1\ # directory contains version and does not exist locally +# ... # extract content +function Extract-Dotnet-Package([string]$ZipPath, [string]$OutPath) { + Say-Invocation $MyInvocation + + Load-Assembly -Assembly System.IO.Compression.FileSystem + Set-Variable -Name Zip + try { + $Zip = [System.IO.Compression.ZipFile]::OpenRead($ZipPath) + + $DirectoriesToUnpack = Get-List-Of-Directories-And-Versions-To-Unpack-From-Dotnet-Package -Zip $Zip -OutPath $OutPath + + foreach ($entry in $Zip.Entries) { + $PathWithVersion = Get-Path-Prefix-With-Version $entry.FullName + if (($PathWithVersion -eq $null) -Or ($DirectoriesToUnpack -contains $PathWithVersion)) { + $DestinationPath = Get-Absolute-Path $(Join-Path -Path $OutPath -ChildPath $entry.FullName) + $DestinationDir = Split-Path -Parent $DestinationPath + $OverrideFiles=$OverrideNonVersionedFiles -Or (-Not (Test-Path $DestinationPath)) + if ((-Not $DestinationPath.EndsWith("\")) -And $OverrideFiles) { + New-Item -ItemType Directory -Force -Path $DestinationDir | Out-Null + [System.IO.Compression.ZipFileExtensions]::ExtractToFile($entry, $DestinationPath, $OverrideNonVersionedFiles) + } + } + } + } + finally { + if ($Zip -ne $null) { + $Zip.Dispose() + } + } +} + +function DownloadFile($Source, [string]$OutPath) { + if ($Source -notlike "http*") { + # Using System.IO.Path.GetFullPath to get the current directory + # does not work in this context - $pwd gives the current directory + if (![System.IO.Path]::IsPathRooted($Source)) { + $Source = $(Join-Path -Path $pwd -ChildPath $Source) + } + $Source = Get-Absolute-Path $Source + Say "Copying file from $Source to $OutPath" + Copy-Item $Source $OutPath + return + } + + $Stream = $null + + try { + $Response = GetHTTPResponse -Uri $Source + $Stream = $Response.Content.ReadAsStreamAsync().Result + $File = [System.IO.File]::Create($OutPath) + $Stream.CopyTo($File) + $File.Close() + } + finally { + if ($Stream -ne $null) { + $Stream.Dispose() + } + } +} + +function Prepend-Sdk-InstallRoot-To-Path([string]$InstallRoot, [string]$BinFolderRelativePath) { + $BinPath = Get-Absolute-Path $(Join-Path -Path $InstallRoot -ChildPath $BinFolderRelativePath) + if (-Not $NoPath) { + $SuffixedBinPath = "$BinPath;" + if (-Not $env:path.Contains($SuffixedBinPath)) { + Say "Adding to current process PATH: `"$BinPath`". Note: This change will not be visible if PowerShell was run as a child process." + $env:path = $SuffixedBinPath + $env:path + } else { + Say-Verbose "Current process PATH already contains `"$BinPath`"" + } + } + else { + Say "Binaries of dotnet can be found in $BinPath" + } +} + +$CLIArchitecture = Get-CLIArchitecture-From-Architecture $Architecture +$SpecificVersion = Get-Specific-Version-From-Version -AzureFeed $AzureFeed -Channel $Channel -Version $Version -JSonFile $JSonFile +$DownloadLink, $EffectiveVersion = Get-Download-Link -AzureFeed $AzureFeed -SpecificVersion $SpecificVersion -CLIArchitecture $CLIArchitecture +$LegacyDownloadLink = Get-LegacyDownload-Link -AzureFeed $AzureFeed -SpecificVersion $SpecificVersion -CLIArchitecture $CLIArchitecture + +$InstallRoot = Resolve-Installation-Path $InstallDir +Say-Verbose "InstallRoot: $InstallRoot" +$ScriptName = $MyInvocation.MyCommand.Name + +if ($DryRun) { + Say "Payload URLs:" + Say "Primary named payload URL: $DownloadLink" + if ($LegacyDownloadLink) { + Say "Legacy named payload URL: $LegacyDownloadLink" + } + $RepeatableCommand = ".\$ScriptName -Version `"$SpecificVersion`" -InstallDir `"$InstallRoot`" -Architecture `"$CLIArchitecture`"" + if ($Runtime -eq "dotnet") { + $RepeatableCommand+=" -Runtime `"dotnet`"" + } + elseif ($Runtime -eq "aspnetcore") { + $RepeatableCommand+=" -Runtime `"aspnetcore`"" + } + foreach ($key in $MyInvocation.BoundParameters.Keys) { + if (-not (@("Architecture","Channel","DryRun","InstallDir","Runtime","SharedRuntime","Version") -contains $key)) { + $RepeatableCommand+=" -$key `"$($MyInvocation.BoundParameters[$key])`"" + } + } + Say "Repeatable invocation: $RepeatableCommand" + exit 0 +} + +if ($Runtime -eq "dotnet") { + $assetName = ".NET Core Runtime" + $dotnetPackageRelativePath = "shared\Microsoft.NETCore.App" +} +elseif ($Runtime -eq "aspnetcore") { + $assetName = "ASP.NET Core Runtime" + $dotnetPackageRelativePath = "shared\Microsoft.AspNetCore.App" +} +elseif ($Runtime -eq "windowsdesktop") { + $assetName = ".NET Core Windows Desktop Runtime" + $dotnetPackageRelativePath = "shared\Microsoft.WindowsDesktop.App" +} +elseif (-not $Runtime) { + $assetName = ".NET Core SDK" + $dotnetPackageRelativePath = "sdk" +} +else { + throw "Invalid value for `$Runtime" +} + +if ($SpecificVersion -ne $EffectiveVersion) +{ + Say "Performing installation checks for effective version: $EffectiveVersion" + $SpecificVersion = $EffectiveVersion +} + +# Check if the SDK version is already installed. +$isAssetInstalled = Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $SpecificVersion +if ($isAssetInstalled) { + Say "$assetName version $SpecificVersion is already installed." + Prepend-Sdk-InstallRoot-To-Path -InstallRoot $InstallRoot -BinFolderRelativePath $BinFolderRelativePath + exit 0 +} + +New-Item -ItemType Directory -Force -Path $InstallRoot | Out-Null + +$installDrive = $((Get-Item $InstallRoot).PSDrive.Name); +$diskInfo = Get-PSDrive -Name $installDrive +if ($diskInfo.Free / 1MB -le 100) { + Say "There is not enough disk space on drive ${installDrive}:" + exit 0 +} + +$ZipPath = [System.IO.Path]::combine([System.IO.Path]::GetTempPath(), [System.IO.Path]::GetRandomFileName()) +Say-Verbose "Zip path: $ZipPath" + +$DownloadFailed = $false +Say "Downloading link: $DownloadLink" +try { + DownloadFile -Source $DownloadLink -OutPath $ZipPath +} +catch { + Say "Cannot download: $DownloadLink" + if ($LegacyDownloadLink) { + $DownloadLink = $LegacyDownloadLink + $ZipPath = [System.IO.Path]::combine([System.IO.Path]::GetTempPath(), [System.IO.Path]::GetRandomFileName()) + Say-Verbose "Legacy zip path: $ZipPath" + Say "Downloading legacy link: $DownloadLink" + try { + DownloadFile -Source $DownloadLink -OutPath $ZipPath + } + catch { + Say "Cannot download: $DownloadLink" + $DownloadFailed = $true + } + } + else { + $DownloadFailed = $true + } +} + +if ($DownloadFailed) { + throw "Could not find/download: `"$assetName`" with version = $SpecificVersion`nRefer to: https://aka.ms/dotnet-os-lifecycle for information on .NET Core support" +} + +Say "Extracting zip from $DownloadLink" +Extract-Dotnet-Package -ZipPath $ZipPath -OutPath $InstallRoot + +# Check if the SDK version is installed; if not, fail the installation. +$isAssetInstalled = $false + +# if the version contains "RTM" or "servicing"; check if a 'release-type' SDK version is installed. +if ($SpecificVersion -Match "rtm" -or $SpecificVersion -Match "servicing") { + $ReleaseVersion = $SpecificVersion.Split("-")[0] + Say-Verbose "Checking installation: version = $ReleaseVersion" + $isAssetInstalled = Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $ReleaseVersion +} + +# Check if the SDK version is installed. +if (!$isAssetInstalled) { + Say-Verbose "Checking installation: version = $SpecificVersion" + $isAssetInstalled = Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $SpecificVersion +} + +if (!$isAssetInstalled) { + throw "`"$assetName`" with version = $SpecificVersion failed to install with an unknown error." +} + +Remove-Item $ZipPath + +Prepend-Sdk-InstallRoot-To-Path -InstallRoot $InstallRoot -BinFolderRelativePath $BinFolderRelativePath + +Say "Installation finished" +exit 0 \ No newline at end of file diff --git a/eng/common/dotnet-install-scripts/dotnet-install.sh b/eng/common/dotnet-install-scripts/dotnet-install.sh new file mode 100644 index 00000000..92161141 --- /dev/null +++ b/eng/common/dotnet-install-scripts/dotnet-install.sh @@ -0,0 +1,1133 @@ +#!/usr/bin/env bash +# Copyright (c) .NET Foundation and contributors. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. +# + +# Stop script on NZEC +set -e +# Stop script if unbound variable found (use ${var:-} if intentional) +set -u +# By default cmd1 | cmd2 returns exit code of cmd2 regardless of cmd1 success +# This is causing it to fail +set -o pipefail + +# Use in the the functions: eval $invocation +invocation='say_verbose "Calling: ${yellow:-}${FUNCNAME[0]} ${green:-}$*${normal:-}"' + +# standard output may be used as a return value in the functions +# we need a way to write text on the screen in the functions so that +# it won't interfere with the return value. +# Exposing stream 3 as a pipe to standard output of the script itself +exec 3>&1 + +# Setup some colors to use. These need to work in fairly limited shells, like the Ubuntu Docker container where there are only 8 colors. +# See if stdout is a terminal +if [ -t 1 ] && command -v tput > /dev/null; then + # see if it supports colors + ncolors=$(tput colors) + if [ -n "$ncolors" ] && [ $ncolors -ge 8 ]; then + bold="$(tput bold || echo)" + normal="$(tput sgr0 || echo)" + black="$(tput setaf 0 || echo)" + red="$(tput setaf 1 || echo)" + green="$(tput setaf 2 || echo)" + yellow="$(tput setaf 3 || echo)" + blue="$(tput setaf 4 || echo)" + magenta="$(tput setaf 5 || echo)" + cyan="$(tput setaf 6 || echo)" + white="$(tput setaf 7 || echo)" + fi +fi + +say_warning() { + printf "%b\n" "${yellow:-}dotnet_install: Warning: $1${normal:-}" +} + +say_err() { + printf "%b\n" "${red:-}dotnet_install: Error: $1${normal:-}" >&2 +} + +say() { + # using stream 3 (defined in the beginning) to not interfere with stdout of functions + # which may be used as return value + printf "%b\n" "${cyan:-}dotnet-install:${normal:-} $1" >&3 +} + +say_verbose() { + if [ "$verbose" = true ]; then + say "$1" + fi +} + +# This platform list is finite - if the SDK/Runtime has supported Linux distribution-specific assets, +# then and only then should the Linux distribution appear in this list. +# Adding a Linux distribution to this list does not imply distribution-specific support. +get_legacy_os_name_from_platform() { + eval $invocation + + platform="$1" + case "$platform" in + "centos.7") + echo "centos" + return 0 + ;; + "debian.8") + echo "debian" + return 0 + ;; + "debian.9") + echo "debian.9" + return 0 + ;; + "fedora.23") + echo "fedora.23" + return 0 + ;; + "fedora.24") + echo "fedora.24" + return 0 + ;; + "fedora.27") + echo "fedora.27" + return 0 + ;; + "fedora.28") + echo "fedora.28" + return 0 + ;; + "opensuse.13.2") + echo "opensuse.13.2" + return 0 + ;; + "opensuse.42.1") + echo "opensuse.42.1" + return 0 + ;; + "opensuse.42.3") + echo "opensuse.42.3" + return 0 + ;; + "rhel.7"*) + echo "rhel" + return 0 + ;; + "ubuntu.14.04") + echo "ubuntu" + return 0 + ;; + "ubuntu.16.04") + echo "ubuntu.16.04" + return 0 + ;; + "ubuntu.16.10") + echo "ubuntu.16.10" + return 0 + ;; + "ubuntu.18.04") + echo "ubuntu.18.04" + return 0 + ;; + "alpine.3.4.3") + echo "alpine" + return 0 + ;; + esac + return 1 +} + +get_linux_platform_name() { + eval $invocation + + if [ -n "$runtime_id" ]; then + echo "${runtime_id%-*}" + return 0 + else + if [ -e /etc/os-release ]; then + . /etc/os-release + echo "$ID${VERSION_ID:+.${VERSION_ID}}" + return 0 + elif [ -e /etc/redhat-release ]; then + local redhatRelease=$(&1 || true) | grep -q musl +} + +get_current_os_name() { + eval $invocation + + local uname=$(uname) + if [ "$uname" = "Darwin" ]; then + echo "osx" + return 0 + elif [ "$uname" = "FreeBSD" ]; then + echo "freebsd" + return 0 + elif [ "$uname" = "Linux" ]; then + local linux_platform_name + linux_platform_name="$(get_linux_platform_name)" || { echo "linux" && return 0 ; } + + if [ "$linux_platform_name" = "rhel.6" ]; then + echo $linux_platform_name + return 0 + elif is_musl_based_distro; then + echo "linux-musl" + return 0 + else + echo "linux" + return 0 + fi + fi + + say_err "OS name could not be detected: UName = $uname" + return 1 +} + +get_legacy_os_name() { + eval $invocation + + local uname=$(uname) + if [ "$uname" = "Darwin" ]; then + echo "osx" + return 0 + elif [ -n "$runtime_id" ]; then + echo $(get_legacy_os_name_from_platform "${runtime_id%-*}" || echo "${runtime_id%-*}") + return 0 + else + if [ -e /etc/os-release ]; then + . /etc/os-release + os=$(get_legacy_os_name_from_platform "$ID${VERSION_ID:+.${VERSION_ID}}" || echo "") + if [ -n "$os" ]; then + echo "$os" + return 0 + fi + fi + fi + + say_verbose "Distribution specific OS name and version could not be detected: UName = $uname" + return 1 +} + +machine_has() { + eval $invocation + + hash "$1" > /dev/null 2>&1 + return $? +} + + +check_min_reqs() { + local hasMinimum=false + if machine_has "curl"; then + hasMinimum=true + elif machine_has "wget"; then + hasMinimum=true + fi + + if [ "$hasMinimum" = "false" ]; then + say_err "curl (recommended) or wget are required to download dotnet. Install missing prerequisite to proceed." + return 1 + fi + return 0 +} + +check_pre_reqs() { + eval $invocation + + if [ "${DOTNET_INSTALL_SKIP_PREREQS:-}" = "1" ]; then + return 0 + fi + + if [ "$(uname)" = "Linux" ]; then + if is_musl_based_distro; then + if ! command -v scanelf > /dev/null; then + say_warning "scanelf not found, please install pax-utils package." + return 0 + fi + LDCONFIG_COMMAND="scanelf --ldpath -BF '%f'" + [ -z "$($LDCONFIG_COMMAND 2>/dev/null | grep libintl)" ] && say_warning "Unable to locate libintl. Probable prerequisite missing; install libintl (or gettext)." + else + if [ ! -x "$(command -v ldconfig)" ]; then + say_verbose "ldconfig is not in PATH, trying /sbin/ldconfig." + LDCONFIG_COMMAND="/sbin/ldconfig" + else + LDCONFIG_COMMAND="ldconfig" + fi + local librarypath=${LD_LIBRARY_PATH:-} + LDCONFIG_COMMAND="$LDCONFIG_COMMAND -NXv ${librarypath//:/ }" + fi + + [ -z "$($LDCONFIG_COMMAND 2>/dev/null | grep zlib)" ] && say_warning "Unable to locate zlib. Probable prerequisite missing; install zlib." + [ -z "$($LDCONFIG_COMMAND 2>/dev/null | grep ssl)" ] && say_warning "Unable to locate libssl. Probable prerequisite missing; install libssl." + [ -z "$($LDCONFIG_COMMAND 2>/dev/null | grep libicu)" ] && say_warning "Unable to locate libicu. Probable prerequisite missing; install libicu." + [ -z "$($LDCONFIG_COMMAND 2>/dev/null | grep lttng)" ] && say_warning "Unable to locate liblttng. Probable prerequisite missing; install libcurl." + [ -z "$($LDCONFIG_COMMAND 2>/dev/null | grep libcurl)" ] && say_warning "Unable to locate libcurl. Probable prerequisite missing; install libcurl." + fi + + return 0 +} + +# args: +# input - $1 +to_lowercase() { + #eval $invocation + + echo "$1" | tr '[:upper:]' '[:lower:]' + return 0 +} + +# args: +# input - $1 +remove_trailing_slash() { + #eval $invocation + + local input="${1:-}" + echo "${input%/}" + return 0 +} + +# args: +# input - $1 +remove_beginning_slash() { + #eval $invocation + + local input="${1:-}" + echo "${input#/}" + return 0 +} + +# args: +# root_path - $1 +# child_path - $2 - this parameter can be empty +combine_paths() { + eval $invocation + + # TODO: Consider making it work with any number of paths. For now: + if [ ! -z "${3:-}" ]; then + say_err "combine_paths: Function takes two parameters." + return 1 + fi + + local root_path="$(remove_trailing_slash "$1")" + local child_path="$(remove_beginning_slash "${2:-}")" + say_verbose "combine_paths: root_path=$root_path" + say_verbose "combine_paths: child_path=$child_path" + echo "$root_path/$child_path" + return 0 +} + +get_machine_architecture() { + eval $invocation + + if command -v uname > /dev/null; then + CPUName=$(uname -m) + case $CPUName in + armv7l) + echo "arm" + return 0 + ;; + aarch64) + echo "arm64" + return 0 + ;; + esac + fi + + # Always default to 'x64' + echo "x64" + return 0 +} + +# args: +# architecture - $1 +get_normalized_architecture_from_architecture() { + eval $invocation + + local architecture="$(to_lowercase "$1")" + case "$architecture" in + \) + echo "$(get_normalized_architecture_from_architecture "$(get_machine_architecture)")" + return 0 + ;; + amd64|x64) + echo "x64" + return 0 + ;; + arm) + echo "arm" + return 0 + ;; + arm64) + echo "arm64" + return 0 + ;; + esac + + say_err "Architecture \`$architecture\` not supported. If you think this is a bug, report it at https://github.com/dotnet/sdk/issues" + return 1 +} + +# The version text returned from the feeds is a 1-line or 2-line string: +# For the SDK and the dotnet runtime (2 lines): +# Line 1: # commit_hash +# Line 2: # 4-part version +# For the aspnetcore runtime (1 line): +# Line 1: # 4-part version + +# args: +# version_text - stdin +get_version_from_version_info() { + eval $invocation + + cat | tail -n 1 | sed 's/\r$//' + return 0 +} + +# args: +# install_root - $1 +# relative_path_to_package - $2 +# specific_version - $3 +is_dotnet_package_installed() { + eval $invocation + + local install_root="$1" + local relative_path_to_package="$2" + local specific_version="${3//[$'\t\r\n']}" + + local dotnet_package_path="$(combine_paths "$(combine_paths "$install_root" "$relative_path_to_package")" "$specific_version")" + say_verbose "is_dotnet_package_installed: dotnet_package_path=$dotnet_package_path" + + if [ -d "$dotnet_package_path" ]; then + return 0 + else + return 1 + fi +} + +# args: +# azure_feed - $1 +# channel - $2 +# normalized_architecture - $3 +# coherent - $4 +get_latest_version_info() { + eval $invocation + + local azure_feed="$1" + local channel="$2" + local normalized_architecture="$3" + local coherent="$4" + + local version_file_url=null + if [[ "$runtime" == "dotnet" ]]; then + version_file_url="$uncached_feed/Runtime/$channel/latest.version" + elif [[ "$runtime" == "aspnetcore" ]]; then + version_file_url="$uncached_feed/aspnetcore/Runtime/$channel/latest.version" + elif [ -z "$runtime" ]; then + if [ "$coherent" = true ]; then + version_file_url="$uncached_feed/Sdk/$channel/latest.coherent.version" + else + version_file_url="$uncached_feed/Sdk/$channel/latest.version" + fi + else + say_err "Invalid value for \$runtime" + return 1 + fi + say_verbose "get_latest_version_info: latest url: $version_file_url" + + download "$version_file_url" + return $? +} + +# args: +# json_file - $1 +parse_jsonfile_for_version() { + eval $invocation + + local json_file="$1" + if [ ! -f "$json_file" ]; then + say_err "Unable to find \`$json_file\`" + return 1 + fi + + sdk_section=$(cat $json_file | awk '/"sdk"/,/}/') + if [ -z "$sdk_section" ]; then + say_err "Unable to parse the SDK node in \`$json_file\`" + return 1 + fi + + sdk_list=$(echo $sdk_section | awk -F"[{}]" '{print $2}') + sdk_list=${sdk_list//[\" ]/} + sdk_list=${sdk_list//,/$'\n'} + sdk_list="$(echo -e "${sdk_list}" | tr -d '[[:space:]]')" + + local version_info="" + while read -r line; do + IFS=: + while read -r key value; do + if [[ "$key" == "version" ]]; then + version_info=$value + fi + done <<< "$line" + done <<< "$sdk_list" + if [ -z "$version_info" ]; then + say_err "Unable to find the SDK:version node in \`$json_file\`" + return 1 + fi + + unset IFS; + echo "$version_info" + return 0 +} + +# args: +# azure_feed - $1 +# channel - $2 +# normalized_architecture - $3 +# version - $4 +# json_file - $5 +get_specific_version_from_version() { + eval $invocation + + local azure_feed="$1" + local channel="$2" + local normalized_architecture="$3" + local version="$(to_lowercase "$4")" + local json_file="$5" + + if [ -z "$json_file" ]; then + case "$version" in + latest) + local version_info + version_info="$(get_latest_version_info "$azure_feed" "$channel" "$normalized_architecture" false)" || return 1 + say_verbose "get_specific_version_from_version: version_info=$version_info" + echo "$version_info" | get_version_from_version_info + return 0 + ;; + coherent) + local version_info + version_info="$(get_latest_version_info "$azure_feed" "$channel" "$normalized_architecture" true)" || return 1 + say_verbose "get_specific_version_from_version: version_info=$version_info" + echo "$version_info" | get_version_from_version_info + return 0 + ;; + *) + echo "$version" + return 0 + ;; + esac + else + local version_info + version_info="$(parse_jsonfile_for_version "$json_file")" || return 1 + echo "$version_info" + return 0 + fi +} + +# args: +# azure_feed - $1 +# channel - $2 +# normalized_architecture - $3 +# specific_version - $4 +construct_download_link() { + eval $invocation + + local azure_feed="$1" + local channel="$2" + local normalized_architecture="$3" + local specific_version="${4//[$'\t\r\n']}" + local specific_product_version="$(get_specific_product_version "$1" "$4")" + + local osname + osname="$(get_current_os_name)" || return 1 + + local download_link=null + if [[ "$runtime" == "dotnet" ]]; then + download_link="$azure_feed/Runtime/$specific_version/dotnet-runtime-$specific_product_version-$osname-$normalized_architecture.tar.gz" + elif [[ "$runtime" == "aspnetcore" ]]; then + download_link="$azure_feed/aspnetcore/Runtime/$specific_version/aspnetcore-runtime-$specific_product_version-$osname-$normalized_architecture.tar.gz" + elif [ -z "$runtime" ]; then + download_link="$azure_feed/Sdk/$specific_version/dotnet-sdk-$specific_product_version-$osname-$normalized_architecture.tar.gz" + else + return 1 + fi + + echo "$download_link" + return 0 +} + +# args: +# azure_feed - $1 +# specific_version - $2 +get_specific_product_version() { + # If we find a 'productVersion.txt' at the root of any folder, we'll use its contents + # to resolve the version of what's in the folder, superseding the specified version. + eval $invocation + + local azure_feed="$1" + local specific_version="${2//[$'\t\r\n']}" + local specific_product_version=$specific_version + + local download_link=null + if [[ "$runtime" == "dotnet" ]]; then + download_link="$azure_feed/Runtime/$specific_version/productVersion.txt${feed_credential}" + elif [[ "$runtime" == "aspnetcore" ]]; then + download_link="$azure_feed/aspnetcore/Runtime/$specific_version/productVersion.txt${feed_credential}" + elif [ -z "$runtime" ]; then + download_link="$azure_feed/Sdk/$specific_version/productVersion.txt${feed_credential}" + else + return 1 + fi + + specific_product_version=$(curl -s --fail "$download_link") + if [ $? -ne 0 ] + then + specific_product_version=$(wget -qO- "$download_link") + if [ $? -ne 0 ] + then + specific_product_version=$specific_version + fi + fi + specific_product_version="${specific_product_version//[$'\t\r\n']}" + + echo "$specific_product_version" + return 0 +} + +# args: +# azure_feed - $1 +# channel - $2 +# normalized_architecture - $3 +# specific_version - $4 +construct_legacy_download_link() { + eval $invocation + + local azure_feed="$1" + local channel="$2" + local normalized_architecture="$3" + local specific_version="${4//[$'\t\r\n']}" + + local distro_specific_osname + distro_specific_osname="$(get_legacy_os_name)" || return 1 + + local legacy_download_link=null + if [[ "$runtime" == "dotnet" ]]; then + legacy_download_link="$azure_feed/Runtime/$specific_version/dotnet-$distro_specific_osname-$normalized_architecture.$specific_version.tar.gz" + elif [ -z "$runtime" ]; then + legacy_download_link="$azure_feed/Sdk/$specific_version/dotnet-dev-$distro_specific_osname-$normalized_architecture.$specific_version.tar.gz" + else + return 1 + fi + + echo "$legacy_download_link" + return 0 +} + +get_user_install_path() { + eval $invocation + + if [ ! -z "${DOTNET_INSTALL_DIR:-}" ]; then + echo "$DOTNET_INSTALL_DIR" + else + echo "$HOME/.dotnet" + fi + return 0 +} + +# args: +# install_dir - $1 +resolve_installation_path() { + eval $invocation + + local install_dir=$1 + if [ "$install_dir" = "" ]; then + local user_install_path="$(get_user_install_path)" + say_verbose "resolve_installation_path: user_install_path=$user_install_path" + echo "$user_install_path" + return 0 + fi + + echo "$install_dir" + return 0 +} + +# args: +# relative_or_absolute_path - $1 +get_absolute_path() { + eval $invocation + + local relative_or_absolute_path=$1 + echo "$(cd "$(dirname "$1")" && pwd -P)/$(basename "$1")" + return 0 +} + +# args: +# input_files - stdin +# root_path - $1 +# out_path - $2 +# override - $3 +copy_files_or_dirs_from_list() { + eval $invocation + + local root_path="$(remove_trailing_slash "$1")" + local out_path="$(remove_trailing_slash "$2")" + local override="$3" + local osname="$(get_current_os_name)" + local override_switch=$( + if [ "$override" = false ]; then + if [ "$osname" = "linux-musl" ]; then + printf -- "-u"; + else + printf -- "-n"; + fi + fi) + + cat | uniq | while read -r file_path; do + local path="$(remove_beginning_slash "${file_path#$root_path}")" + local target="$out_path/$path" + if [ "$override" = true ] || (! ([ -d "$target" ] || [ -e "$target" ])); then + mkdir -p "$out_path/$(dirname "$path")" + if [ -d "$target" ]; then + rm -rf "$target" + fi + cp -R $override_switch "$root_path/$path" "$target" + fi + done +} + +# args: +# zip_path - $1 +# out_path - $2 +extract_dotnet_package() { + eval $invocation + + local zip_path="$1" + local out_path="$2" + + local temp_out_path="$(mktemp -d "$temporary_file_template")" + + local failed=false + tar -xzf "$zip_path" -C "$temp_out_path" > /dev/null || failed=true + + local folders_with_version_regex='^.*/[0-9]+\.[0-9]+[^/]+/' + find "$temp_out_path" -type f | grep -Eo "$folders_with_version_regex" | sort | copy_files_or_dirs_from_list "$temp_out_path" "$out_path" false + find "$temp_out_path" -type f | grep -Ev "$folders_with_version_regex" | copy_files_or_dirs_from_list "$temp_out_path" "$out_path" "$override_non_versioned_files" + + rm -rf "$temp_out_path" + + if [ "$failed" = true ]; then + say_err "Extraction failed" + return 1 + fi +} + +# args: +# remote_path - $1 +# [out_path] - $2 - stdout if not provided +download() { + eval $invocation + + local remote_path="$1" + local out_path="${2:-}" + + if [[ "$remote_path" != "http"* ]]; then + cp "$remote_path" "$out_path" + return $? + fi + + local failed=false + if machine_has "curl"; then + downloadcurl "$remote_path" "$out_path" || failed=true + elif machine_has "wget"; then + downloadwget "$remote_path" "$out_path" || failed=true + else + failed=true + fi + if [ "$failed" = true ]; then + say_verbose "Download failed: $remote_path" + return 1 + fi + return 0 +} + +downloadcurl() { + eval $invocation + local remote_path="$1" + local out_path="${2:-}" + + # Append feed_credential as late as possible before calling curl to avoid logging feed_credential + remote_path="${remote_path}${feed_credential}" + + local curl_options="--retry 20 --retry-delay 2 --connect-timeout 15 -sSL -f --create-dirs " + local failed=false + if [ -z "$out_path" ]; then + curl $curl_options "$remote_path" || failed=true + else + curl $curl_options -o "$out_path" "$remote_path" || failed=true + fi + if [ "$failed" = true ]; then + say_verbose "Curl download failed" + return 1 + fi + return 0 +} + +downloadwget() { + eval $invocation + local remote_path="$1" + local out_path="${2:-}" + + # Append feed_credential as late as possible before calling wget to avoid logging feed_credential + remote_path="${remote_path}${feed_credential}" + local wget_options="--tries 20 --waitretry 2 --connect-timeout 15 " + local failed=false + if [ -z "$out_path" ]; then + wget -q $wget_options -O - "$remote_path" || failed=true + else + wget $wget_options -O "$out_path" "$remote_path" || failed=true + fi + if [ "$failed" = true ]; then + say_verbose "Wget download failed" + return 1 + fi + return 0 +} + +calculate_vars() { + eval $invocation + valid_legacy_download_link=true + + normalized_architecture="$(get_normalized_architecture_from_architecture "$architecture")" + say_verbose "normalized_architecture=$normalized_architecture" + + specific_version="$(get_specific_version_from_version "$azure_feed" "$channel" "$normalized_architecture" "$version" "$json_file")" + specific_product_version="$(get_specific_product_version "$azure_feed" "$specific_version")" + say_verbose "specific_version=$specific_version" + if [ -z "$specific_version" ]; then + say_err "Could not resolve version information." + return 1 + fi + + download_link="$(construct_download_link "$azure_feed" "$channel" "$normalized_architecture" "$specific_version")" + say_verbose "Constructed primary named payload URL: $download_link" + + legacy_download_link="$(construct_legacy_download_link "$azure_feed" "$channel" "$normalized_architecture" "$specific_version")" || valid_legacy_download_link=false + + if [ "$valid_legacy_download_link" = true ]; then + say_verbose "Constructed legacy named payload URL: $legacy_download_link" + else + say_verbose "Cound not construct a legacy_download_link; omitting..." + fi + + install_root="$(resolve_installation_path "$install_dir")" + say_verbose "InstallRoot: $install_root" +} + +install_dotnet() { + eval $invocation + local download_failed=false + local asset_name='' + local asset_relative_path='' + + if [[ "$runtime" == "dotnet" ]]; then + asset_relative_path="shared/Microsoft.NETCore.App" + asset_name=".NET Core Runtime" + elif [[ "$runtime" == "aspnetcore" ]]; then + asset_relative_path="shared/Microsoft.AspNetCore.App" + asset_name="ASP.NET Core Runtime" + elif [ -z "$runtime" ]; then + asset_relative_path="sdk" + asset_name=".NET Core SDK" + else + say_err "Invalid value for \$runtime" + return 1 + fi + + # Check if the SDK version is already installed. + if is_dotnet_package_installed "$install_root" "$asset_relative_path" "$specific_version"; then + say "$asset_name version $specific_version is already installed." + return 0 + fi + + mkdir -p "$install_root" + zip_path="$(mktemp "$temporary_file_template")" + say_verbose "Zip path: $zip_path" + + say "Downloading link: $download_link" + + # Failures are normal in the non-legacy case for ultimately legacy downloads. + # Do not output to stderr, since output to stderr is considered an error. + download "$download_link" "$zip_path" 2>&1 || download_failed=true + + # if the download fails, download the legacy_download_link + if [ "$download_failed" = true ]; then + say "Cannot download: $download_link" + + if [ "$valid_legacy_download_link" = true ]; then + download_failed=false + download_link="$legacy_download_link" + zip_path="$(mktemp "$temporary_file_template")" + say_verbose "Legacy zip path: $zip_path" + say "Downloading legacy link: $download_link" + download "$download_link" "$zip_path" 2>&1 || download_failed=true + + if [ "$download_failed" = true ]; then + say "Cannot download: $download_link" + fi + fi + fi + + if [ "$download_failed" = true ]; then + say_err "Could not find/download: \`$asset_name\` with version = $specific_version" + say_err "Refer to: https://aka.ms/dotnet-os-lifecycle for information on .NET Core support" + return 1 + fi + + say "Extracting zip from $download_link" + extract_dotnet_package "$zip_path" "$install_root" + + # Check if the SDK version is installed; if not, fail the installation. + # if the version contains "RTM" or "servicing"; check if a 'release-type' SDK version is installed. + if [[ $specific_version == *"rtm"* || $specific_version == *"servicing"* ]]; then + IFS='-' + read -ra verArr <<< "$specific_version" + release_version="${verArr[0]}" + unset IFS; + say_verbose "Checking installation: version = $release_version" + if is_dotnet_package_installed "$install_root" "$asset_relative_path" "$release_version"; then + return 0 + fi + fi + + # Check if the standard SDK version is installed. + say_verbose "Checking installation: version = $specific_product_version" + if is_dotnet_package_installed "$install_root" "$asset_relative_path" "$specific_product_version"; then + return 0 + fi + + say_err "\`$asset_name\` with version = $specific_product_version failed to install with an unknown error." + return 1 +} + +args=("$@") + +local_version_file_relative_path="/.version" +bin_folder_relative_path="" +temporary_file_template="${TMPDIR:-/tmp}/dotnet.XXXXXXXXX" + +channel="LTS" +version="Latest" +json_file="" +install_dir="" +architecture="" +dry_run=false +no_path=false +no_cdn=false +azure_feed="https://dotnetcli.azureedge.net/dotnet" +uncached_feed="https://dotnetcli.blob.core.windows.net/dotnet" +feed_credential="" +verbose=false +runtime="" +runtime_id="" +override_non_versioned_files=true +non_dynamic_parameters="" + +while [ $# -ne 0 ] +do + name="$1" + case "$name" in + -c|--channel|-[Cc]hannel) + shift + channel="$1" + ;; + -v|--version|-[Vv]ersion) + shift + version="$1" + ;; + -i|--install-dir|-[Ii]nstall[Dd]ir) + shift + install_dir="$1" + ;; + --arch|--architecture|-[Aa]rch|-[Aa]rchitecture) + shift + architecture="$1" + ;; + --shared-runtime|-[Ss]hared[Rr]untime) + say_warning "The --shared-runtime flag is obsolete and may be removed in a future version of this script. The recommended usage is to specify '--runtime dotnet'." + if [ -z "$runtime" ]; then + runtime="dotnet" + fi + ;; + --runtime|-[Rr]untime) + shift + runtime="$1" + if [[ "$runtime" != "dotnet" ]] && [[ "$runtime" != "aspnetcore" ]]; then + say_err "Unsupported value for --runtime: '$1'. Valid values are 'dotnet' and 'aspnetcore'." + if [[ "$runtime" == "windowsdesktop" ]]; then + say_err "WindowsDesktop archives are manufactured for Windows platforms only." + fi + exit 1 + fi + ;; + --dry-run|-[Dd]ry[Rr]un) + dry_run=true + ;; + --no-path|-[Nn]o[Pp]ath) + no_path=true + non_dynamic_parameters+=" $name" + ;; + --verbose|-[Vv]erbose) + verbose=true + non_dynamic_parameters+=" $name" + ;; + --no-cdn|-[Nn]o[Cc]dn) + no_cdn=true + non_dynamic_parameters+=" $name" + ;; + --azure-feed|-[Aa]zure[Ff]eed) + shift + azure_feed="$1" + non_dynamic_parameters+=" $name "\""$1"\""" + ;; + --uncached-feed|-[Uu]ncached[Ff]eed) + shift + uncached_feed="$1" + non_dynamic_parameters+=" $name "\""$1"\""" + ;; + --feed-credential|-[Ff]eed[Cc]redential) + shift + feed_credential="$1" + non_dynamic_parameters+=" $name "\""$1"\""" + ;; + --runtime-id|-[Rr]untime[Ii]d) + shift + runtime_id="$1" + non_dynamic_parameters+=" $name "\""$1"\""" + ;; + --jsonfile|-[Jj][Ss]on[Ff]ile) + shift + json_file="$1" + ;; + --skip-non-versioned-files|-[Ss]kip[Nn]on[Vv]ersioned[Ff]iles) + override_non_versioned_files=false + non_dynamic_parameters+=" $name" + ;; + -?|--?|-h|--help|-[Hh]elp) + script_name="$(basename "$0")" + echo ".NET Tools Installer" + echo "Usage: $script_name [-c|--channel ] [-v|--version ] [-p|--prefix ]" + echo " $script_name -h|-?|--help" + echo "" + echo "$script_name is a simple command line interface for obtaining dotnet cli." + echo "" + echo "Options:" + echo " -c,--channel Download from the channel specified, Defaults to \`$channel\`." + echo " -Channel" + echo " Possible values:" + echo " - Current - most current release" + echo " - LTS - most current supported release" + echo " - 2-part version in a format A.B - represents a specific release" + echo " examples: 2.0; 1.0" + echo " - Branch name" + echo " examples: release/2.0.0; Master" + echo " Note: The version parameter overrides the channel parameter." + echo " -v,--version Use specific VERSION, Defaults to \`$version\`." + echo " -Version" + echo " Possible values:" + echo " - latest - most latest build on specific channel" + echo " - coherent - most latest coherent build on specific channel" + echo " coherent applies only to SDK downloads" + echo " - 3-part version in a format A.B.C - represents specific version of build" + echo " examples: 2.0.0-preview2-006120; 1.1.0" + echo " -i,--install-dir Install under specified location (see Install Location below)" + echo " -InstallDir" + echo " --architecture Architecture of dotnet binaries to be installed, Defaults to \`$architecture\`." + echo " --arch,-Architecture,-Arch" + echo " Possible values: x64, arm, and arm64" + echo " --runtime Installs a shared runtime only, without the SDK." + echo " -Runtime" + echo " Possible values:" + echo " - dotnet - the Microsoft.NETCore.App shared runtime" + echo " - aspnetcore - the Microsoft.AspNetCore.App shared runtime" + echo " --dry-run,-DryRun Do not perform installation. Display download link." + echo " --no-path, -NoPath Do not set PATH for the current process." + echo " --verbose,-Verbose Display diagnostics information." + echo " --azure-feed,-AzureFeed Azure feed location. Defaults to $azure_feed, This parameter typically is not changed by the user." + echo " --uncached-feed,-UncachedFeed Uncached feed location. This parameter typically is not changed by the user." + echo " --feed-credential,-FeedCredential Azure feed shared access token. This parameter typically is not specified." + echo " --skip-non-versioned-files Skips non-versioned files if they already exist, such as the dotnet executable." + echo " -SkipNonVersionedFiles" + echo " --no-cdn,-NoCdn Disable downloading from the Azure CDN, and use the uncached feed directly." + echo " --jsonfile Determines the SDK version from a user specified global.json file." + echo " Note: global.json must have a value for 'SDK:Version'" + echo " --runtime-id Installs the .NET Tools for the given platform (use linux-x64 for portable linux)." + echo " -RuntimeId" + echo " -?,--?,-h,--help,-Help Shows this help message" + echo "" + echo "Obsolete parameters:" + echo " --shared-runtime The recommended alternative is '--runtime dotnet'." + echo " This parameter is obsolete and may be removed in a future version of this script." + echo " Installs just the shared runtime bits, not the entire SDK." + echo "" + echo "Install Location:" + echo " Location is chosen in following order:" + echo " - --install-dir option" + echo " - Environmental variable DOTNET_INSTALL_DIR" + echo " - $HOME/.dotnet" + exit 0 + ;; + *) + say_err "Unknown argument \`$name\`" + exit 1 + ;; + esac + + shift +done + +if [ "$no_cdn" = true ]; then + azure_feed="$uncached_feed" +fi + +check_min_reqs +calculate_vars +script_name=$(basename "$0") + +if [ "$dry_run" = true ]; then + say "Payload URLs:" + say "Primary named payload URL: $download_link" + if [ "$valid_legacy_download_link" = true ]; then + say "Legacy named payload URL: $legacy_download_link" + fi + repeatable_command="./$script_name --version "\""$specific_version"\"" --install-dir "\""$install_root"\"" --architecture "\""$normalized_architecture"\""" + if [[ "$runtime" == "dotnet" ]]; then + repeatable_command+=" --runtime "\""dotnet"\""" + elif [[ "$runtime" == "aspnetcore" ]]; then + repeatable_command+=" --runtime "\""aspnetcore"\""" + fi + repeatable_command+="$non_dynamic_parameters" + say "Repeatable invocation: $repeatable_command" + exit 0 +fi + +check_pre_reqs +install_dotnet + +bin_path="$(get_absolute_path "$(combine_paths "$install_root" "$bin_folder_relative_path")")" +if [ "$no_path" = false ]; then + say "Adding to current process PATH: \`$bin_path\`. Note: This change will be visible only when sourcing script." + export PATH="$bin_path":"$PATH" +else + say "Binaries of dotnet can be found in $bin_path" +fi + +say "Installation finished successfully." diff --git a/eng/common/internal/Directory.Build.props b/eng/common/internal/Directory.Build.props index e33179ef..dbf99d82 100644 --- a/eng/common/internal/Directory.Build.props +++ b/eng/common/internal/Directory.Build.props @@ -1,4 +1,4 @@ - + diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1 index f044afe8..650b13b0 100644 --- a/eng/common/post-build/publish-using-darc.ps1 +++ b/eng/common/post-build/publish-using-darc.ps1 @@ -8,7 +8,7 @@ param( [Parameter(Mandatory=$false)][string] $EnableSourceLinkValidation, [Parameter(Mandatory=$false)][string] $EnableSigningValidation, [Parameter(Mandatory=$false)][string] $EnableNugetValidation, - [Parameter(Mandatory=$true)][string] $PublishInstallersAndChecksums, + [Parameter(Mandatory=$false)][string] $PublishInstallersAndChecksums, [Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters, [Parameter(Mandatory=$false)][string] $SigningValidationAdditionalParameters ) @@ -16,7 +16,7 @@ param( try { . $PSScriptRoot\post-build-utils.ps1 # Hard coding darc version till the next arcade-services roll out, cos this version has required API changes for darc add-build-to-channel - . $PSScriptRoot\..\darc-init.ps1 -darcVersion "1.1.0-beta.20418.1" + $darc = Get-Darc "1.1.0-beta.20418.1" $optionalParams = [System.Collections.ArrayList]::new() @@ -29,7 +29,7 @@ try { $optionalParams.Add("--no-wait") | Out-Null } - if ("true" -eq $PublishInstallersAndChecksums) { + if ("false" -ne $PublishInstallersAndChecksums) { $optionalParams.Add("--publish-installers-and-checksums") | Out-Null } @@ -50,7 +50,7 @@ try { } } - & darc add-build-to-channel ` + & $darc add-build-to-channel ` --id $buildId ` --publishing-infra-version $PublishingInfraVersion ` --default-channels ` diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 055304ad..d0c3cc2b 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -67,6 +67,7 @@ jobs: /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} /p:Configuration=$(_BuildConfig) + /p:OfficialBuildId=$(Build.BuildNumber) condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} @@ -87,11 +88,6 @@ jobs: ArtifactName: ReleaseConfigs - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - - task: PublishBuildArtifacts@1 - displayName: Publish Logs to VSTS - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' - PublishLocation: Container - ArtifactName: $(Agent.Os)_PublishBuildAssets - continueOnError: true - condition: always() + - template: /eng/common/templates/steps/publish-logs.yml + parameters: + JobLabel: 'Publish_Artifacts_Logs' diff --git a/eng/common/templates/post-build/channels/generic-internal-channel.yml b/eng/common/templates/post-build/channels/generic-internal-channel.yml index 59eb93a4..7ae52559 100644 --- a/eng/common/templates/post-build/channels/generic-internal-channel.yml +++ b/eng/common/templates/post-build/channels/generic-internal-channel.yml @@ -4,7 +4,7 @@ parameters: artifactsPublishingAdditionalParameters: '' dependsOn: - Validate - publishInstallersAndChecksums: false + publishInstallersAndChecksums: true symbolPublishingAdditionalParameters: '' stageName: '' channelName: '' @@ -158,7 +158,7 @@ stages: /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' /p:Configuration=Release - /p:PublishInstallersAndChecksums=true + /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }} /p:ChecksumsTargetStaticFeed=$(InternalChecksumsBlobFeedUrl) /p:ChecksumsAzureAccountKey=$(InternalChecksumsBlobFeedKey) /p:InstallersTargetStaticFeed=$(InternalInstallersBlobFeedUrl) diff --git a/eng/common/templates/post-build/channels/generic-public-channel.yml b/eng/common/templates/post-build/channels/generic-public-channel.yml index 7e80a621..6cf39dbb 100644 --- a/eng/common/templates/post-build/channels/generic-public-channel.yml +++ b/eng/common/templates/post-build/channels/generic-public-channel.yml @@ -4,7 +4,7 @@ parameters: artifactsPublishingAdditionalParameters: '' dependsOn: - Validate - publishInstallersAndChecksums: false + publishInstallersAndChecksums: true symbolPublishingAdditionalParameters: '' stageName: '' channelName: '' diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 630a99d4..df06f537 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -19,7 +19,7 @@ parameters: enableSigningValidation: true enableSymbolValidation: false enableNugetValidation: true - publishInstallersAndChecksums: false + publishInstallersAndChecksums: true SDLValidationParameters: enable: false continueOnError: false diff --git a/eng/common/templates/steps/publish-logs.yml b/eng/common/templates/steps/publish-logs.yml index f91751fe..88f238f3 100644 --- a/eng/common/templates/steps/publish-logs.yml +++ b/eng/common/templates/steps/publish-logs.yml @@ -12,7 +12,7 @@ steps: Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ continueOnError: true condition: always() - + - task: PublishBuildArtifacts@1 displayName: Publish Logs inputs: diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index c2e575c9..37a3a3d6 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -57,11 +57,15 @@ set-strictmode -version 2.0 $ErrorActionPreference = 'Stop' [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 -# If specifies, provides an alternate path for getting .NET Core SDKs and Runtimes. This script will still try public sources first. +# If specified, provides an alternate path for getting .NET Core SDKs and Runtimes. This script will still try public sources first. [string]$runtimeSourceFeed = if (Test-Path variable:runtimeSourceFeed) { $runtimeSourceFeed } else { $null } # Base-64 encoded SAS token that has permission to storage container described by $runtimeSourceFeed [string]$runtimeSourceFeedKey = if (Test-Path variable:runtimeSourceFeedKey) { $runtimeSourceFeedKey } else { $null } +# If false, use copy of dotnet-install from /eng/common/dotnet-install-scripts (for custom behaviors). +# otherwise will fetch from public location. +[bool]$useDefaultDotnetInstall = if (Test-Path variable:useDefaultDotnetInstall) { $useDefaultDotnetInstall } else { $false } + function Create-Directory ([string[]] $path) { New-Item -Path $path -Force -ItemType 'Directory' | Out-Null } @@ -193,42 +197,51 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { function GetDotNetInstallScript([string] $dotnetRoot) { $installScript = Join-Path $dotnetRoot 'dotnet-install.ps1' if (!(Test-Path $installScript)) { - Create-Directory $dotnetRoot - $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit - - $maxRetries = 5 - $retries = 1 - - $uri = "https://dot.net/$dotnetInstallScriptVersion/dotnet-install.ps1" - - while($true) { - try { - Write-Host "GET $uri" - Invoke-WebRequest $uri -OutFile $installScript - break - } - catch { - Write-Host "Failed to download '$uri'" - Write-Error $_.Exception.Message -ErrorAction Continue - } - - if (++$retries -le $maxRetries) { - $delayInSeconds = [math]::Pow(2, $retries) - 1 # Exponential backoff - Write-Host "Retrying. Waiting for $delayInSeconds seconds before next attempt ($retries of $maxRetries)." - Start-Sleep -Seconds $delayInSeconds - } - else { - throw "Unable to download file in $maxRetries attempts." + create-directory $dotnetroot + + if ($useDefaultDotnetInstall) + { + $progresspreference = 'silentlycontinue' # don't display the console progress ui - it's a huge perf hit + + $maxretries = 5 + $retries = 1 + + $uri = "https://dot.net/$dotnetinstallscriptversion/dotnet-install.ps1" + + while($true) { + try { + write-host "get $uri" + invoke-webrequest $uri -outfile $installscript + break + } + catch { + write-host "failed to download '$uri'" + write-error $_.exception.message -erroraction continue + } + + if (++$retries -le $maxretries) { + $delayinseconds = [math]::pow(2, $retries) - 1 # exponential backoff + write-host "retrying. waiting for $delayinseconds seconds before next attempt ($retries of $maxretries)." + start-sleep -seconds $delayinseconds + } + else { + throw "unable to download file in $maxretries attempts." + } } - + } + else + { + # Use a special version of the script from eng/common that understands the existence of a "productVersion.txt" in a dotnet path. + # See https://github.com/dotnet/arcade/issues/6047 for details + $engCommonCopy = Resolve-Path (Join-Path $PSScriptRoot 'dotnet-install-scripts\dotnet-install.ps1') + Copy-Item $engCommonCopy -Destination $installScript -Force } } - return $installScript } -function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $architecture = '') { - InstallDotNet $dotnetRoot $version $architecture '' $false $runtimeSourceFeed $runtimeSourceFeedKey +function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $architecture = '', [switch] $noPath) { + InstallDotNet $dotnetRoot $version $architecture '' $false $runtimeSourceFeed $runtimeSourceFeedKey -noPath:$noPath } function InstallDotNet([string] $dotnetRoot, @@ -237,7 +250,8 @@ function InstallDotNet([string] $dotnetRoot, [string] $runtime = '', [bool] $skipNonVersionedFiles = $false, [string] $runtimeSourceFeed = '', - [string] $runtimeSourceFeedKey = '') { + [string] $runtimeSourceFeedKey = '', + [switch] $noPath) { $installScript = GetDotNetInstallScript $dotnetRoot $installParameters = @{ @@ -248,6 +262,7 @@ function InstallDotNet([string] $dotnetRoot, if ($architecture) { $installParameters.Architecture = $architecture } if ($runtime) { $installParameters.Runtime = $runtime } if ($skipNonVersionedFiles) { $installParameters.SkipNonVersionedFiles = $skipNonVersionedFiles } + if ($noPath) { $installParameters.NoPath = $True } try { & $installScript @installParameters @@ -703,6 +718,16 @@ function IsWindowsPlatform() { return [environment]::OSVersion.Platform -eq [PlatformID]::Win32NT } +function Get-Darc($version) { + $darcPath = "$TempDir\darc\$(New-Guid)" + if ($version -ne $null) { + & $PSScriptRoot\darc-init.ps1 -toolpath $darcPath -darcVersion $version | Out-Host + } else { + & $PSScriptRoot\darc-init.ps1 -toolpath $darcPath | Out-Host + } + return "$darcPath\darc.exe" +} + . $PSScriptRoot\pipeline-logging-functions.ps1 $RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..') diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 2be5cba3..c722a058 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -68,6 +68,10 @@ fi runtime_source_feed=${runtime_source_feed:-''} runtime_source_feed_key=${runtime_source_feed_key:-''} +# Determines if dotnet-install.sh comes from the eng/common folder or the internet +# (default = public version) +use_default_dotnet_install=${use_default_dotnet_install:-false} + # Resolve any symlinks in the given path. function ResolvePath { local path=$1 @@ -267,23 +271,30 @@ function GetDotNetInstallScript { if [[ ! -a "$install_script" ]]; then mkdir -p "$root" - echo "Downloading '$install_script_url'" + if [[ "$use_default_dotnet_install" == true ]]; then + echo "Downloading '$install_script_url'" - # Use curl if available, otherwise use wget - if command -v curl > /dev/null; then - with_retries curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || { - local exit_code=$? - Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." - ExitWithExitCode $exit_code - } + # Use curl if available, otherwise use wget + if command -v curl > /dev/null; then + with_retries curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || { + local exit_code=$? + Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." + ExitWithExitCode $exit_code + } + else + with_retries wget -v -O "$install_script" "$install_script_url" || { + local exit_code=$? + Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." + ExitWithExitCode $exit_code + } + fi else - with_retries wget -v -O "$install_script" "$install_script_url" || { - local exit_code=$? - Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." - ExitWithExitCode $exit_code - } + # Use a special version of the script from eng/common that understands the existence of a "productVersion.txt" in a dotnet path. + # See https://github.com/dotnet/arcade/issues/6047 for details + cp $repo_root/eng/common/dotnet-install-scripts/dotnet-install.sh $install_script fi fi + # return value _GetDotNetInstallScript="$install_script" } diff --git a/global.json b/global.json index eb71a354..3e2276cb 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "5.0.100-preview.6.20310.4" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20427.5" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20453.7" } } From 8a3edd1902dbfe3adba65f22e3bb7aa2cc73e97f Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 14 Sep 2020 12:26:19 +0000 Subject: [PATCH 095/161] Update dependencies from https://github.com/dotnet/arcade build 20200909.8 (#649) [master] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index ee6e3fd3..80677ec7 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 16b71a2f216c3c5be5860977c4cb03a95ee2f0e3 + 91470b0b14ba016c1fb78211b12775287c17b34e diff --git a/global.json b/global.json index 3e2276cb..e1e835d4 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "5.0.100-preview.6.20310.4" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20453.7" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20459.8" } } From 81357f3e90241d3b262289b1ffa285c9ba51d335 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Sat, 26 Sep 2020 00:32:50 +0000 Subject: [PATCH 096/161] [master] Update dependencies from dotnet/arcade (#650) [master] Update dependencies from dotnet/arcade - Add NuGetPackageFolders into expected SourceRoots --- eng/Version.Details.xml | 4 +- eng/common/performance/performance-setup.sh | 2 +- .../post-build/sourcelink-validation.ps1 | 26 ++++++-- eng/common/post-build/symbols-validation.ps1 | 7 +- eng/common/sdk-task.ps1 | 2 +- eng/common/templates/job/source-build.yml | 49 ++++++++++++++ eng/common/templates/jobs/jobs.yml | 16 +++++ eng/common/templates/jobs/source-build.yml | 48 ++++++++++++++ .../templates/post-build/post-build.yml | 31 ++++----- .../templates/steps/perf-send-to-helix.yml | 34 +++------- eng/common/templates/steps/source-build.yml | 66 +++++++++++++++++++ eng/common/tools.ps1 | 29 +++++--- eng/common/tools.sh | 2 + global.json | 4 +- .../AzureDevOpsServerTests.cs | 2 + .../AzureReposAndGitHubTests.cs | 8 ++- .../AzureReposTests.cs | 2 + .../BitbucketGitTests.cs | 8 +++ .../GitHubTests.cs | 5 +- .../GitLabTests.cs | 2 + .../GitWebTests.cs | 1 + .../TargetTests.cs | 5 +- .../DotNetSdk/DotNetSdkTestBase.cs | 8 ++- 23 files changed, 288 insertions(+), 73 deletions(-) create mode 100644 eng/common/templates/job/source-build.yml create mode 100644 eng/common/templates/jobs/source-build.yml create mode 100644 eng/common/templates/steps/source-build.yml diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 80677ec7..12711821 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 91470b0b14ba016c1fb78211b12775287c17b34e + 61cde6e8fb9d5c9790867b279deb41783a780cd8 diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index 806e56c6..99d1b7bc 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -201,7 +201,6 @@ if [[ "$internal" == true ]]; then fi if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "false" ]]; then - configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot" extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoMono" fi @@ -211,6 +210,7 @@ if [[ "$wasm_runtime_loc" != "" ]]; then fi if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "true" ]]; then + configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot" extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter NoMono" fi diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1 index cdd1bba4..c7e7ae67 100644 --- a/eng/common/post-build/sourcelink-validation.ps1 +++ b/eng/common/post-build/sourcelink-validation.ps1 @@ -144,11 +144,27 @@ $ValidatePackage = { if ($FailedFiles -eq 0) { Write-Host 'Passed.' - return 0 + return [pscustomobject]@{ + result = 0 + packagePath = $PackagePath + } } else { Write-PipelineTelemetryError -Category 'SourceLink' -Message "$PackagePath has broken SourceLink links." - return 1 + return [pscustomobject]@{ + result = 1 + packagePath = $PackagePath + } + } +} + +function CheckJobResult( + $result, + $packagePath, + [ref]$ValidationFailures) { + if ($jobResult.result -ne '0') { + Write-PipelineTelemetryError -Category 'SourceLink' -Message "$packagePath has broken SourceLink links." + $ValidationFailures.Value++ } } @@ -211,10 +227,8 @@ function ValidateSourceLinkLinks { } foreach ($Job in @(Get-Job -State 'Completed')) { - $jobResult = Receive-Job -Id $Job.Id - if ($jobResult -ne '0') { - $ValidationFailures++ - } + $jobResult = Wait-Job -Id $Job.Id | Receive-Job + CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures) Remove-Job -Id $Job.Id } } diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 index a36fa6f2..fcc6019b 100644 --- a/eng/common/post-build/symbols-validation.ps1 +++ b/eng/common/post-build/symbols-validation.ps1 @@ -141,11 +141,6 @@ $CountMissingSymbols = { if ($using:Clean) { Remove-Item $ExtractPath -Recurse -Force } - - if ($MissingSymbols -ne 0) - { - Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $MissingSymbols modules in the package $PackagePath" - } Pop-Location @@ -165,6 +160,7 @@ function CheckJobResult( $DupedSymbols.Value++ } elseif ($jobResult.result -ne '0') { + Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $result modules in the package $packagePath" $TotalFailures.Value++ } } @@ -201,7 +197,6 @@ function CheckSymbolsAvailable { Start-Job -ScriptBlock $CountMissingSymbols -ArgumentList $FullName | Out-Null $NumJobs = @(Get-Job -State 'Running').Count - Write-Host $NumJobs while ($NumJobs -ge $MaxParallelJobs) { Write-Host "There are $NumJobs validation jobs running right now. Waiting $SecondsBetweenLoadChecks seconds to check again." diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index e159c6f1..f55c43c6 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -64,7 +64,7 @@ try { $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty } if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { - $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.5.0-alpha" -MemberType NoteProperty + $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.8.0-preview3" -MemberType NoteProperty } if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml new file mode 100644 index 00000000..9332f5ec --- /dev/null +++ b/eng/common/templates/job/source-build.yml @@ -0,0 +1,49 @@ +parameters: + # This template adds arcade-powered source-build to CI. The template produces a server job with a + # default ID 'Source_Build_Complete' to put in a dependency list if necessary. + + # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed. + jobNamePrefix: 'Source_Build' + + # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for + # managed-only repositories. This is an object with these properties: + # + # name: '' + # The name of the job. This is included in the job ID. + # targetRID: '' + # The name of the target RID to use, instead of the one auto-detected by Arcade. + # nonPortable: false + # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than + # linux-x64), and compiling against distro-provided packages rather than portable ones. + # container: '' + # A container to use. Runs in docker. + # pool: {} + # A pool to use. Runs directly on an agent. + # buildScript: '' + # Specifies the build script to invoke to perform the build in the repo. The default + # './build.sh' should work for typical Arcade repositories, but this is customizable for + # difficult situations. + # jobProperties: {} + # A list of job properties to inject at the top level, for potential extensibility beyond + # container and pool. + platform: {} + +jobs: +- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} + displayName: Source-Build (${{ parameters.platform.name }}) + + ${{ each property in parameters.platform.jobProperties }}: + ${{ property.key }}: ${{ property.value }} + + ${{ if ne(parameters.platform.container, '') }}: + container: ${{ parameters.platform.container }} + ${{ if ne(parameters.platform.pool, '') }}: + pool: ${{ parameters.platform.pool }} + + workspace: + clean: all + + steps: + - template: /eng/common/templates/steps/source-build.yml + parameters: + platform: ${{ parameters.platform }} diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml index c08225a9..08845950 100644 --- a/eng/common/templates/jobs/jobs.yml +++ b/eng/common/templates/jobs/jobs.yml @@ -24,6 +24,13 @@ parameters: # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. runAsPublic: false + # Optional: Enable running the source-build jobs to build repo from source + runSourceBuild: false + + # Optional: Parameters for source-build template. + # See /eng/common/templates/jobs/source-build.yml for options + sourceBuildParameters: [] + # Internal resources (telemetry, microbuild) can only be accessed from non-public projects, # and some (Microbuild) should only be applied to non-PR cases for internal builds. @@ -43,6 +50,13 @@ jobs: name: ${{ job.job }} +- ${{ if eq(parameters.runSourceBuild, true) }}: + - template: /eng/common/templates/jobs/source-build.yml + parameters: + allCompletedJobId: Source_Build_Complete + ${{ each parameter in parameters.sourceBuildParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: - template: ../job/publish-build-assets.yml @@ -55,6 +69,8 @@ jobs: - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: - ${{ each job in parameters.jobs }}: - ${{ job.job }} + - ${{ if eq(parameters.runSourceBuild, true) }}: + - Source_Build_Complete pool: vmImage: vs2017-win2016 runAsPublic: ${{ parameters.runAsPublic }} diff --git a/eng/common/templates/jobs/source-build.yml b/eng/common/templates/jobs/source-build.yml new file mode 100644 index 00000000..f463011e --- /dev/null +++ b/eng/common/templates/jobs/source-build.yml @@ -0,0 +1,48 @@ +parameters: + # This template adds arcade-powered source-build to CI. A job is created for each platform, as + # well as an optional server job that completes when all platform jobs complete. + + # The name of the "join" job for all source-build platforms. If set to empty string, the job is + # not included. Existing repo pipelines can use this job depend on all source-build jobs + # completing without maintaining a separate list of every single job ID: just depend on this one + # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'. + allCompletedJobId: '' + + # See /eng/common/templates/job/source-build.yml + jobNamePrefix: 'Source_Build' + + # If changed to true, causes this template to include the default platform for a managed-only + # repo. The exact Docker image used for this build will be provided by Arcade. This has some risk, + # but since the repo is supposed to be managed-only, the risk should be very low. + includeDefaultManagedPlatform: false + defaultManagedPlatform: + name: 'Managed' + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-3e800f1-20190501005343' + + # Defines the platforms on which to run build jobs. One job is created for each platform, and the + # object in this array is sent to the job template as 'platform'. + platforms: [] + +jobs: + +- ${{ if ne(parameters.allCompletedJobId, '') }}: + - job: ${{ parameters.allCompletedJobId }} + displayName: Source-Build Complete + pool: server + dependsOn: + - ${{ each platform in parameters.platforms }}: + - ${{ parameters.jobNamePrefix }}_${{ platform.name }} + - ${{ if eq(parameters.includeDefaultManagedPlatform, true) }}: + - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }} + +- ${{ each platform in parameters.platforms }}: + - template: /eng/common/templates/job/source-build.yml + parameters: + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ platform }} + +- ${{ if eq(parameters.includeDefaultManagedPlatform, true) }}: + - template: /eng/common/templates/job/source-build.yml + parameters: + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ parameters.defaultManagedPlatform }} diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index df06f537..0854e489 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -78,21 +78,22 @@ stages: BARBuildId: ${{ parameters.BARBuildId }} PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - job: - displayName: Post-build Checks - dependsOn: setupMaestroVars - variables: - - name: TargetChannels - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ] - pool: - vmImage: 'windows-2019' - steps: - - task: PowerShell@2 - displayName: Maestro Channels Consistency - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 - arguments: -PromoteToChannels "$(TargetChannels)" - -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview8ChannelId}},${{parameters.Net5RC1ChannelId}},${{parameters.Net5RC2ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}} + - ${{ if and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')) }}: + - job: + displayName: Post-build Checks + dependsOn: setupMaestroVars + variables: + - name: TargetChannels + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ] + pool: + vmImage: 'windows-2019' + steps: + - task: PowerShell@2 + displayName: Maestro Channels Consistency + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 + arguments: -PromoteToChannels "$(TargetChannels)" + -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview8ChannelId}},${{parameters.Net5RC1ChannelId}},${{parameters.Net5RC2ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}} - job: displayName: NuGet Validation diff --git a/eng/common/templates/steps/perf-send-to-helix.yml b/eng/common/templates/steps/perf-send-to-helix.yml index e003fe2e..8427de59 100644 --- a/eng/common/templates/steps/perf-send-to-helix.yml +++ b/eng/common/templates/steps/perf-send-to-helix.yml @@ -19,34 +19,18 @@ parameters: DisplayNamePrefix: 'Send job to Helix' # optional -- rename the beginning of the displayName of the steps in AzDO condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false + osGroup: '' # required -- operating system for the job steps: - - powershell: $(Build.SourcesDirectory)\eng\common\msbuild.ps1 $(Build.SourcesDirectory)\eng\common\performance\${{ parameters.ProjectFile }} /restore /t:Test /bl:$(Build.SourcesDirectory)\artifacts\log\$env:BuildConfig\SendToHelix.binlog - displayName: ${{ parameters.DisplayNamePrefix }} (Windows) - env: - BuildConfig: $(_BuildConfig) - HelixSource: ${{ parameters.HelixSource }} - HelixType: ${{ parameters.HelixType }} - HelixBuild: ${{ parameters.HelixBuild }} - HelixTargetQueues: ${{ parameters.HelixTargetQueues }} - HelixAccessToken: ${{ parameters.HelixAccessToken }} - HelixPreCommands: ${{ parameters.HelixPreCommands }} - HelixPostCommands: ${{ parameters.HelixPostCommands }} - WorkItemDirectory: ${{ parameters.WorkItemDirectory }} - CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} - IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} - DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} - DotNetCliVersion: ${{ parameters.DotNetCliVersion }} - EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} - WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - Creator: ${{ parameters.Creator }} - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) +- template: /eng/pipelines/common/templates/runtimes/send-to-helix-inner-step.yml + parameters: + osGroup: ${{ parameters.osGroup }} + sendParams: $(Build.SourcesDirectory)/eng/common/performance/${{ parameters.ProjectFile }} /restore /t:Test /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/SendToHelix.binlog + displayName: ${{ parameters.DisplayNamePrefix }} + condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} - - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/performance/${{ parameters.ProjectFile }} /restore /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog - displayName: ${{ parameters.DisplayNamePrefix }} (Unix) - env: + environment: BuildConfig: $(_BuildConfig) HelixSource: ${{ parameters.HelixSource }} HelixType: ${{ parameters.HelixType }} @@ -64,5 +48,3 @@ steps: WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} Creator: ${{ parameters.Creator }} SYSTEM_ACCESSTOKEN: $(System.AccessToken) - condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml new file mode 100644 index 00000000..8e336b7d --- /dev/null +++ b/eng/common/templates/steps/source-build.yml @@ -0,0 +1,66 @@ +parameters: + # This template adds arcade-powered source-build to CI. + + # This is a 'steps' template, and is intended for advanced scenarios where the existing build + # infra has a careful build methodology that must be followed. For example, a repo + # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline + # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to + # GitHub. Using this steps template leaves room for that infra to be included. + + # Defines the platform on which to run the steps. See 'eng/common/templates/job/source-build.yml' + # for details. The entire object is described in the 'job' template for simplicity, even though + # the usage of the properties on this object is split between the 'job' and 'steps' templates. + platform: {} + +steps: +# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.) +- script: | + set -x + df -h + + buildConfig=Release + # Check if AzDO substitutes in a build config from a variable, and use it if so. + if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then + buildConfig='$(_BuildConfig)' + fi + + officialBuildArgs= + if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then + officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)' + fi + + targetRidArgs= + if [ '${{ parameters.platform.targetRID }}' != '' ]; then + targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' + fi + + ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ + --configuration $buildConfig \ + --restore --build --pack --publish \ + $officialBuildArgs \ + $targetRidArgs \ + /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ + /p:ArcadeBuildFromSource=true + displayName: Build + +# Upload build logs for diagnosis. +- task: CopyFiles@2 + displayName: Prepare BuildLogs staging directory + inputs: + SourceFolder: '$(Build.SourcesDirectory)' + Contents: | + **/*.log + **/*.binlog + artifacts/source-build/self/prebuilt-report/** + TargetFolder: '$(Build.StagingDirectory)/BuildLogs' + CleanTargetFolder: true + continueOnError: true + condition: succeededOrFailed() + +- task: PublishPipelineArtifact@1 + displayName: Publish BuildLogs + inputs: + targetPath: '$(Build.StagingDirectory)/BuildLogs' + artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) + continueOnError: true + condition: succeededOrFailed() diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 37a3a3d6..9014e062 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -312,9 +312,15 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = return $global:_MSBuildExe } - $vsMinVersionReqdStr = '16.5' + # Minimum VS version to require. + $vsMinVersionReqdStr = '16.8' $vsMinVersionReqd = [Version]::new($vsMinVersionReqdStr) + # If the version of msbuild is going to be xcopied, + # use this version. Version matches a package here: + # https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=16.8.0-preview3&view=overview + $defaultXCopyMSBuildVersion = '16.8.0-preview3' + if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } $vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { $vsMinVersionReqdStr } $vsMinVersion = [Version]::new($vsMinVersionStr) @@ -349,23 +355,28 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = $xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild' $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0] } else { - #if vs version provided in global.json is incompatible then use the default version for xcopy msbuild download + #if vs version provided in global.json is incompatible (too low) then use the default version for xcopy msbuild download if($vsMinVersion -lt $vsMinVersionReqd){ - Write-Host "Using xcopy-msbuild version of $vsMinVersionReqdStr.0-alpha since VS version $vsMinVersionStr provided in global.json is not compatible" - $vsMajorVersion = $vsMinVersionReqd.Major - $vsMinorVersion = $vsMinVersionReqd.Minor + Write-Host "Using xcopy-msbuild version of $defaultXCopyMSBuildVersion since VS version $vsMinVersionStr provided in global.json is not compatible" + $xcopyMSBuildVersion = $defaultXCopyMSBuildVersion } else{ + # If the VS version IS compatible, look for an xcopy msbuild package + # with a version matching VS. + # Note: If this version does not exist, then an explicit version of xcopy msbuild + # can be specified in global.json. This will be required for pre-release versions of msbuild. $vsMajorVersion = $vsMinVersion.Major $vsMinorVersion = $vsMinVersion.Minor + $xcopyMSBuildVersion = "$vsMajorVersion.$vsMinorVersion.0" } - - $xcopyMSBuildVersion = "$vsMajorVersion.$vsMinorVersion.0-alpha" } $vsInstallDir = $null if ($xcopyMSBuildVersion.Trim() -ine "none") { $vsInstallDir = InitializeXCopyMSBuild $xcopyMSBuildVersion $install + if ($vsInstallDir -eq $null) { + throw "Could not xcopy msbuild. Please check that package 'RoslynTools.MSBuild @ $xcopyMSBuildVersion' exists on feed 'dotnet-eng'." + } } if ($vsInstallDir -eq $null) { throw 'Unable to find Visual Studio that has required version and components installed' @@ -529,13 +540,15 @@ function GetDefaultMSBuildEngine() { function GetNuGetPackageCachePath() { if ($env:NUGET_PACKAGES -eq $null) { - # Use local cache on CI to ensure deterministic build, + # Use local cache on CI to ensure deterministic build. + # Avoid using the http cache as workaround for https://github.com/NuGet/Home/issues/3116 # use global cache in dev builds to avoid cost of downloading packages. # For directory normalization, see also: https://github.com/NuGet/Home/issues/7968 if ($useGlobalNuGetCache) { $env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages\' } else { $env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages\' + $env:RESTORENOCACHE = $true } } diff --git a/eng/common/tools.sh b/eng/common/tools.sh index c722a058..b5d63cb1 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -312,12 +312,14 @@ function InitializeBuildTool { _InitializeBuildToolFramework="netcoreapp2.1" } +# Set RestoreNoCache as a workaround for https://github.com/NuGet/Home/issues/3116 function GetNuGetPackageCachePath { if [[ -z ${NUGET_PACKAGES:-} ]]; then if [[ "$use_global_nuget_cache" == true ]]; then export NUGET_PACKAGES="$HOME/.nuget/packages" else export NUGET_PACKAGES="$repo_root/.packages" + export RESTORENOCACHE=true fi fi diff --git a/global.json b/global.json index e1e835d4..42a3e013 100644 --- a/global.json +++ b/global.json @@ -1,8 +1,8 @@ { "tools": { - "dotnet": "5.0.100-preview.6.20310.4" + "dotnet": "5.0.100-rc.1.20452.10" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20459.8" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20474.4" } } diff --git a/src/SourceLink.Git.IntegrationTests/AzureDevOpsServerTests.cs b/src/SourceLink.Git.IntegrationTests/AzureDevOpsServerTests.cs index e49dbdb4..0f9fa83d 100644 --- a/src/SourceLink.Git.IntegrationTests/AzureDevOpsServerTests.cs +++ b/src/SourceLink.Git.IntegrationTests/AzureDevOpsServerTests.cs @@ -47,6 +47,7 @@ public void FullValidation_Https() }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://tfs.噸.local:8080/tfs/DefaultCollection/project/_apis/git/repositories/{repoName}/items?api-version=1.0&versionType=commit&version={commitSha}&path=/*", s_relativeSourceLinkJsonPath, @@ -104,6 +105,7 @@ public void FullValidation_Ssh() }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://tfs.噸.local/tfs/DefaultCollection/project/_apis/git/repositories/{repoName}/items?api-version=1.0&versionType=commit&version={commitSha}&path=/*", s_relativeSourceLinkJsonPath, diff --git a/src/SourceLink.Git.IntegrationTests/AzureReposAndGitHubTests.cs b/src/SourceLink.Git.IntegrationTests/AzureReposAndGitHubTests.cs index f61952ab..21d2b6b1 100644 --- a/src/SourceLink.Git.IntegrationTests/AzureReposAndGitHubTests.cs +++ b/src/SourceLink.Git.IntegrationTests/AzureReposAndGitHubTests.cs @@ -66,6 +66,7 @@ public void CustomTranslation() }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://raw.githubusercontent.com/test-org/{repoName}/{commitSha}/*", s_relativeSourceLinkJsonPath, @@ -122,6 +123,7 @@ public void Host_VisualStudio(string host) }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://test.{host}/test-org/_apis/git/repositories/{repoName}/items?api-version=1.0&versionType=commit&version={commitSha}&path=/*", s_relativeSourceLinkJsonPath, @@ -163,6 +165,7 @@ public void Host_DevAzureCom(string host) }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://{host}/test/test-org/_apis/git/repositories/{repoName}/items?api-version=1.0&versionType=commit&version={commitSha}&path=/*", s_relativeSourceLinkJsonPath, @@ -188,11 +191,12 @@ public void Host_Unknown() }, expressions: new[] { - "@(SourceRoot->'%(SourceLinkUrl)')", + "@(SourceRoot->'%(Identity):%(SourceLinkUrl)')", }, expectedResults: new[] { - "", + NuGetPackageFolders + ":", + EnsureTrailingDirectorySeparator(ProjectDir.Path) + ":", }, expectedWarnings: new[] { diff --git a/src/SourceLink.Git.IntegrationTests/AzureReposTests.cs b/src/SourceLink.Git.IntegrationTests/AzureReposTests.cs index f7b47f44..6a488d16 100644 --- a/src/SourceLink.Git.IntegrationTests/AzureReposTests.cs +++ b/src/SourceLink.Git.IntegrationTests/AzureReposTests.cs @@ -47,6 +47,7 @@ public void FullValidation_Https(string host) }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://test.{host}/test-org/_apis/git/repositories/{repoName}/items?api-version=1.0&versionType=commit&version={commitSha}&path=/*", s_relativeSourceLinkJsonPath, @@ -103,6 +104,7 @@ public void FullValidation_Ssh(string host) }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://test.{host}/test-org/_apis/git/repositories/{repoName}/items?api-version=1.0&versionType=commit&version={commitSha}&path=/*", s_relativeSourceLinkJsonPath, diff --git a/src/SourceLink.Git.IntegrationTests/BitbucketGitTests.cs b/src/SourceLink.Git.IntegrationTests/BitbucketGitTests.cs index 236499ad..efefa8e4 100644 --- a/src/SourceLink.Git.IntegrationTests/BitbucketGitTests.cs +++ b/src/SourceLink.Git.IntegrationTests/BitbucketGitTests.cs @@ -43,6 +43,7 @@ public void FullValidation_CloudHttps() }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://api.bitbucket.org/2.0/repositories/test-org/{repoName}/src/{commitSha}/*", s_relativeSourceLinkJsonPath, @@ -101,6 +102,7 @@ public void FullValidation_EnterpriseNewHttps() }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://bitbucket.domain.com/projects/test-org/repos/{repoName}/raw/*?at={commitSha}", s_relativeSourceLinkJsonPath, @@ -157,6 +159,7 @@ public void FullValidation_EnterpriseNewHttps_UrlWithPersonalToken() }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://bitbucket.domain.com/projects/test-org/repos/{repoName}/raw/*?at={commitSha}", s_relativeSourceLinkJsonPath, @@ -215,6 +218,7 @@ public void FullValidation_EnterpriseNewHttpsWithDefaultFlags() }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://bitbucket.domain.com/projects/test-org/repos/{repoName}/raw/*?at={commitSha}", s_relativeSourceLinkJsonPath, @@ -272,6 +276,7 @@ public void FullValidation_EnterpriseOldHttps() }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://bitbucket.domain.com/projects/test-org/repos/{repoName}/browse/*?at={commitSha}&raw", s_relativeSourceLinkJsonPath, @@ -329,6 +334,7 @@ public void FullValidation_CloudSsh() }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://api.噸.com/2.0/repositories/test-org/{repoName}/src/{commitSha}/*", s_relativeSourceLinkJsonPath, @@ -386,6 +392,7 @@ public void FullValidation_EnterpriseOldSsh() }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://噸.com/projects/test-org/repos/{repoName}/browse/*?at={commitSha}&raw", s_relativeSourceLinkJsonPath, @@ -443,6 +450,7 @@ public void FullValidation_EnterpriseNewSsh() }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://噸.com/projects/test-org/repos/{repoName}/raw/*?at={commitSha}", s_relativeSourceLinkJsonPath, diff --git a/src/SourceLink.Git.IntegrationTests/GitHubTests.cs b/src/SourceLink.Git.IntegrationTests/GitHubTests.cs index e9b84b6f..324d4322 100644 --- a/src/SourceLink.Git.IntegrationTests/GitHubTests.cs +++ b/src/SourceLink.Git.IntegrationTests/GitHubTests.cs @@ -38,7 +38,7 @@ public void EmptyRepository() }, expectedResults: new[] { - "" + NuGetPackageFolders }, expectedWarnings: new[] { @@ -99,6 +99,7 @@ public void MutlipleProjects() }, expectedResults: new[] { + NuGetPackageFolders, SourceRoot, $"https://raw.githubusercontent.com/test-org/test-repo2/{commitSha}/*", s_relativeSourceLinkJsonPath, @@ -240,6 +241,7 @@ public void FullValidation_Https() }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://raw.githubusercontent.com/test-org/{repoName}/{commitSha}/*", s_relativeSourceLinkJsonPath, @@ -295,6 +297,7 @@ public void FullValidation_Ssh() }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://raw.githubusercontent.com/test-org/{repoName}/{commitSha}/*", s_relativeSourceLinkJsonPath, diff --git a/src/SourceLink.Git.IntegrationTests/GitLabTests.cs b/src/SourceLink.Git.IntegrationTests/GitLabTests.cs index 16afc378..5185cc49 100644 --- a/src/SourceLink.Git.IntegrationTests/GitLabTests.cs +++ b/src/SourceLink.Git.IntegrationTests/GitLabTests.cs @@ -47,6 +47,7 @@ public void FullValidation_Https() }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://噸.com/test-org/{repoName}/raw/{commitSha}/*", s_relativeSourceLinkJsonPath, @@ -104,6 +105,7 @@ public void FullValidation_Ssh() }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://噸.com/test-org/{repoName}/raw/{commitSha}/*", s_relativeSourceLinkJsonPath, diff --git a/src/SourceLink.Git.IntegrationTests/GitWebTests.cs b/src/SourceLink.Git.IntegrationTests/GitWebTests.cs index b0958d59..22080050 100644 --- a/src/SourceLink.Git.IntegrationTests/GitWebTests.cs +++ b/src/SourceLink.Git.IntegrationTests/GitWebTests.cs @@ -50,6 +50,7 @@ public void FullValidation_Ssh() }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, $"https://噸.com/gitweb/?p={repoName};a=blob_plain;hb={commitSha};f=*", s_relativeSourceLinkJsonPath, diff --git a/src/SourceLink.Git.IntegrationTests/TargetTests.cs b/src/SourceLink.Git.IntegrationTests/TargetTests.cs index a17c657a..68b63dd0 100644 --- a/src/SourceLink.Git.IntegrationTests/TargetTests.cs +++ b/src/SourceLink.Git.IntegrationTests/TargetTests.cs @@ -52,6 +52,7 @@ public void GenerateSourceLinkFileTarget_EnableSourceLinkCondition() }, expectedResults: new[] { + NuGetPackageFolders, ProjectSourceRoot, "true", "true", @@ -86,7 +87,7 @@ public void DefaultValuesForEnableProperties_DesignTimeBuild() }, expectedResults: new[] { - "", + NuGetPackageFolders, "", "", "" @@ -121,7 +122,7 @@ public void DefaultValuesForEnableProperties_BuildingForLiveUnitTesting() }, expectedResults: new[] { - "", + NuGetPackageFolders, "", "", "" diff --git a/src/TestUtilities/DotNetSdk/DotNetSdkTestBase.cs b/src/TestUtilities/DotNetSdk/DotNetSdkTestBase.cs index 79d5bca4..a9936150 100644 --- a/src/TestUtilities/DotNetSdk/DotNetSdkTestBase.cs +++ b/src/TestUtilities/DotNetSdk/DotNetSdkTestBase.cs @@ -59,6 +59,7 @@ private static string GetLocalNuGetConfigContent(string packagesDir) => protected readonly TempDirectory ProjectDir; protected readonly TempDirectory ProjectObjDir; protected readonly TempDirectory NuGetCacheDir; + protected readonly string NuGetPackageFolders; protected readonly TempDirectory ProjectOutDir; protected readonly TempFile Project; protected readonly string SourceRoot; @@ -168,6 +169,7 @@ public DotNetSdkTestBase(params string[] packages) RootDir = Temp.CreateDirectory(); NuGetCacheDir = RootDir.CreateDirectory(".packages"); + NuGetPackageFolders = EnsureTrailingDirectorySeparator(NuGetCacheDir.Path); RootDir.CreateFile("Directory.Build.props").WriteAllText( $@" @@ -186,7 +188,8 @@ public DotNetSdkTestBase(params string[] packages) { { "MSBuildSDKsPath", sdksDir }, { "DOTNET_MSBUILD_SDK_RESOLVER_SDKS_DIR", sdksDir }, - { "NUGET_PACKAGES", NuGetCacheDir.Path } + { "NUGET_PACKAGES", NuGetCacheDir.Path }, + { "NuGetPackageFolders", NuGetPackageFolders } }; ProjectDir = RootDir.CreateDirectory(ProjectName); @@ -198,6 +201,9 @@ public DotNetSdkTestBase(params string[] packages) ProjectDir.CreateFile("TestClass.cs").WriteAllText(s_classSource); } + public static string EnsureTrailingDirectorySeparator(string path) + => (path.LastOrDefault() == Path.DirectorySeparatorChar) ? path : path + Path.DirectorySeparatorChar; + protected void VerifyValues( string customProps, string customTargets, From 0d4db50ebd6bb04c43e2271abe02f08436cb24aa Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 5 Oct 2020 12:36:51 +0000 Subject: [PATCH 097/161] Update dependencies from https://github.com/dotnet/arcade build 20200928.3 (#654) [master] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 12711821..14666a4e 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 61cde6e8fb9d5c9790867b279deb41783a780cd8 + c8a7611e15656f45b039d46ec4e09f9f2a7edf99 diff --git a/global.json b/global.json index 42a3e013..1c0bf2b0 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "5.0.100-rc.1.20452.10" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20474.4" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20478.3" } } From 217b2fe45196a92b8ecf1824aebbb3b5475bc0a5 Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Tue, 6 Oct 2020 16:25:30 -0500 Subject: [PATCH 098/161] Add basic source-build build config (#651) --- eng/SourceBuild.props | 15 +++++++++++++++ eng/SourceBuildPrebuiltBaseline.xml | 5 +++++ eng/Version.Details.xml | 5 +++++ 3 files changed, 25 insertions(+) create mode 100644 eng/SourceBuild.props create mode 100644 eng/SourceBuildPrebuiltBaseline.xml diff --git a/eng/SourceBuild.props b/eng/SourceBuild.props new file mode 100644 index 00000000..4cd76267 --- /dev/null +++ b/eng/SourceBuild.props @@ -0,0 +1,15 @@ + + + + sourcelink + true + + + + + $(InnerBuildArgs) /p:Pack=true + + + + diff --git a/eng/SourceBuildPrebuiltBaseline.xml b/eng/SourceBuildPrebuiltBaseline.xml new file mode 100644 index 00000000..c1b6dfbf --- /dev/null +++ b/eng/SourceBuildPrebuiltBaseline.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 14666a4e..dcc2c470 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -7,5 +7,10 @@ https://github.com/dotnet/arcade c8a7611e15656f45b039d46ec4e09f9f2a7edf99 + + https://github.com/dotnet/source-build-reference-packages + def2e2c6dc5064319250e2868a041a3dc07f9579 + + From d4dde65c825e33f7222e81a6f6b42c61a071e88e Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 12 Oct 2020 12:31:26 +0000 Subject: [PATCH 099/161] Update dependencies from https://github.com/dotnet/arcade build 20201006.7 (#655) [master] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- eng/common/templates/job/job.yml | 12 +++++++++++- eng/common/templates/steps/perf-send-to-helix.yml | 2 +- eng/common/templates/steps/send-to-helix.yml | 2 +- global.json | 2 +- 5 files changed, 16 insertions(+), 6 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index dcc2c470..48627fb0 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - c8a7611e15656f45b039d46ec4e09f9f2a7edf99 + ee39cd1573dbb8011f343e1037af51d4fc00a747 https://github.com/dotnet/source-build-reference-packages diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index e78ed9a1..8b81a7e5 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -204,7 +204,7 @@ jobs: - ${{ if eq(parameters.enablePublishTestResults, 'true') }}: - task: PublishTestResults@2 - displayName: Publish Test Results + displayName: Publish XUnit Test Results inputs: testResultsFormat: 'xUnit' testResultsFiles: '*.xml' @@ -213,6 +213,16 @@ jobs: mergeTestResults: ${{ parameters.mergeTestResults }} continueOnError: true condition: always() + - task: PublishTestResults@2 + displayName: Publish TRX Test Results + inputs: + testResultsFormat: 'VSTest' + testResultsFiles: '*.trx' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx + mergeTestResults: ${{ parameters.mergeTestResults }} + continueOnError: true + condition: always() - ${{ if and(eq(parameters.enablePublishBuildAssets, true), ne(parameters.enablePublishUsingPipelines, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - task: CopyFiles@2 diff --git a/eng/common/templates/steps/perf-send-to-helix.yml b/eng/common/templates/steps/perf-send-to-helix.yml index 8427de59..a468e92c 100644 --- a/eng/common/templates/steps/perf-send-to-helix.yml +++ b/eng/common/templates/steps/perf-send-to-helix.yml @@ -11,7 +11,7 @@ parameters: WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion - DotNetCliPackageType: '' # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json + DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml index 5eceb487..bb5f1a92 100644 --- a/eng/common/templates/steps/send-to-helix.yml +++ b/eng/common/templates/steps/send-to-helix.yml @@ -18,7 +18,7 @@ parameters: XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion - DotNetCliPackageType: '' # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases-index.json + DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases-index.json DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases-index.json EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." diff --git a/global.json b/global.json index 1c0bf2b0..aecb9328 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "5.0.100-rc.1.20452.10" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20478.3" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20506.7" } } From 0decb380f1e0f8267259b5ae5d0d1e5102e380a1 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2020 12:27:07 +0000 Subject: [PATCH 100/161] Update dependencies from https://github.com/dotnet/arcade build 20201015.7 (#660) [master] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 +- eng/common/cross/toolchain.cmake | 4 + eng/common/performance/performance-setup.sh | 1 + .../post-build/sourcelink-validation.ps1 | 15 +- .../templates/post-build/post-build.yml | 306 +++++++++--------- global.json | 4 +- 6 files changed, 172 insertions(+), 162 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 48627fb0..2d0e426e 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - ee39cd1573dbb8011f343e1037af51d4fc00a747 + d4de3ce701c2ba697d71cc70e2db69d7568913dc https://github.com/dotnet/source-build-reference-packages diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index 137736c0..8b437d8a 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -139,6 +139,10 @@ function(add_toolchain_linker_flag Flag) set("CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}" "${CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}} ${Flag}" PARENT_SCOPE) endfunction() +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/lib/${TOOLCHAIN}") + add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib/${TOOLCHAIN}") +endif() if(TARGET_ARCH_NAME STREQUAL "armel") if(DEFINED TIZEN_TOOLCHAIN) # For Tizen only diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index 99d1b7bc..9c0f6c90 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -201,6 +201,7 @@ if [[ "$internal" == true ]]; then fi if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "false" ]]; then + configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot" extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoMono" fi diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1 index c7e7ae67..1728b742 100644 --- a/eng/common/post-build/sourcelink-validation.ps1 +++ b/eng/common/post-build/sourcelink-validation.ps1 @@ -161,9 +161,12 @@ $ValidatePackage = { function CheckJobResult( $result, $packagePath, - [ref]$ValidationFailures) { - if ($jobResult.result -ne '0') { - Write-PipelineTelemetryError -Category 'SourceLink' -Message "$packagePath has broken SourceLink links." + [ref]$ValidationFailures, + [switch]$logErrors) { + if ($result -ne '0') { + if ($logError) { + Write-PipelineTelemetryError -Category 'SourceLink' -Message "$packagePath has broken SourceLink links." + } $ValidationFailures.Value++ } } @@ -228,16 +231,14 @@ function ValidateSourceLinkLinks { foreach ($Job in @(Get-Job -State 'Completed')) { $jobResult = Wait-Job -Id $Job.Id | Receive-Job - CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures) + CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures) -LogErrors Remove-Job -Id $Job.Id } } foreach ($Job in @(Get-Job)) { $jobResult = Wait-Job -Id $Job.Id | Receive-Job - if ($jobResult -ne '0') { - $ValidationFailures++ - } + CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures) Remove-Job -Id $Job.Id } if ($ValidationFailures -gt 0) { diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 0854e489..761fb1a2 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -67,176 +67,180 @@ parameters: VSMasterChannelId: 1012 stages: -- stage: Validate - dependsOn: ${{ parameters.validateDependsOn }} - displayName: Validate Build Assets - variables: - - template: common-variables.yml - jobs: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} +- ${{ if or(and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')), eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: + - stage: Validate + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Validate Build Assets + variables: + - template: common-variables.yml + jobs: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - ${{ if and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')) }}: + - job: + displayName: Post-build Checks + dependsOn: setupMaestroVars + variables: + - name: TargetChannels + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ] + pool: + vmImage: 'windows-2019' + steps: + - task: PowerShell@2 + displayName: Maestro Channels Consistency + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 + arguments: -PromoteToChannels "$(TargetChannels)" + -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview8ChannelId}},${{parameters.Net5RC1ChannelId}},${{parameters.Net5RC2ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}} - - ${{ if and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')) }}: - job: - displayName: Post-build Checks + displayName: NuGet Validation dependsOn: setupMaestroVars - variables: - - name: TargetChannels - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ] + condition: eq( ${{ parameters.enableNugetValidation }}, 'true') pool: vmImage: 'windows-2019' + variables: + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + - task: PowerShell@2 - displayName: Maestro Channels Consistency + displayName: Validate inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 - arguments: -PromoteToChannels "$(TargetChannels)" - -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview8ChannelId}},${{parameters.Net5RC1ChannelId}},${{parameters.Net5RC2ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}} - - - job: - displayName: NuGet Validation - dependsOn: setupMaestroVars - condition: eq( ${{ parameters.enableNugetValidation }}, 'true') - pool: - vmImage: 'windows-2019' - variables: - - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 - arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ - -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ - - - job: - displayName: Signing Validation - dependsOn: setupMaestroVars - condition: eq( ${{ parameters.enableSigningValidation }}, 'true') - variables: - - template: common-variables.yml - - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] - pool: - vmImage: 'windows-2019' - steps: - - ${{ if eq(parameters.useBuildManifest, true) }}: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ + -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ + + - job: + displayName: Signing Validation + dependsOn: setupMaestroVars + condition: eq( ${{ parameters.enableSigningValidation }}, 'true') + variables: + - template: common-variables.yml + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + pool: + vmImage: 'windows-2019' + steps: + - ${{ if eq(parameters.useBuildManifest, true) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download build manifest + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BuildManifests - task: DownloadBuildArtifacts@0 - displayName: Download build manifest + displayName: Download Package Artifacts inputs: buildType: specific buildVersionToDownload: specific project: $(AzDOProjectName) pipeline: $(AzDOPipelineId) buildId: $(AzDOBuildId) - artifactName: BuildManifests - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - - # This is necessary whenever we want to publish/restore to an AzDO private feed - # Since sdk-task.ps1 tries to restore packages we need to do this authentication here - # otherwise it'll complain about accessing a private feed. - - task: NuGetAuthenticate@0 - displayName: 'Authenticate to AzDO Feeds' - - - task: PowerShell@2 - displayName: Enable cross-org publishing - inputs: - filePath: eng\common\enable-cross-org-publishing.ps1 - arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) - - # Signing validation will optionally work with the buildmanifest file which is downloaded from - # Azure DevOps above. - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task SigningValidation -restore -msbuildEngine vs - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' - /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' - ${{ parameters.signingValidationAdditionalParameters }} - - - template: ../steps/publish-logs.yml - parameters: - StageLabel: 'Validation' - JobLabel: 'Signing' - - - job: - displayName: SourceLink Validation - dependsOn: setupMaestroVars - condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') - variables: - - template: common-variables.yml - - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: BlobArtifacts - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -ExtractPath $(Agent.BuildDirectory)/Extract/ - -GHRepoName $(Build.Repository.Name) - -GHCommit $(Build.SourceVersion) - -SourcelinkCliVersion $(SourceLinkCLIVersion) - continueOnError: true - - - template: /eng/common/templates/job/execute-sdl.yml - parameters: - enable: ${{ parameters.SDLValidationParameters.enable }} + artifactName: PackageArtifacts + + # This is necessary whenever we want to publish/restore to an AzDO private feed + # Since sdk-task.ps1 tries to restore packages we need to do this authentication here + # otherwise it'll complain about accessing a private feed. + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + + - task: PowerShell@2 + displayName: Enable cross-org publishing + inputs: + filePath: eng\common\enable-cross-org-publishing.ps1 + arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) + + # Signing validation will optionally work with the buildmanifest file which is downloaded from + # Azure DevOps above. + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task SigningValidation -restore -msbuildEngine vs + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' + /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' + ${{ parameters.signingValidationAdditionalParameters }} + + - template: ../steps/publish-logs.yml + parameters: + StageLabel: 'Validation' + JobLabel: 'Signing' + + - job: + displayName: SourceLink Validation dependsOn: setupMaestroVars - additionalParameters: ${{ parameters.SDLValidationParameters.params }} - continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} - artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }} - downloadArtifacts: ${{ parameters.SDLValidationParameters.downloadArtifacts }} + condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') + variables: + - template: common-variables.yml + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BlobArtifacts + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) + -GHCommit $(Build.SourceVersion) + -SourcelinkCliVersion $(SourceLinkCLIVersion) + continueOnError: true + + - template: /eng/common/templates/job/execute-sdl.yml + parameters: + enable: ${{ parameters.SDLValidationParameters.enable }} + dependsOn: setupMaestroVars + additionalParameters: ${{ parameters.SDLValidationParameters.params }} + continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} + artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }} + downloadArtifacts: ${{ parameters.SDLValidationParameters.downloadArtifacts }} - ${{ if or(ge(parameters.publishingInfraVersion, 3), eq(parameters.inline, 'false')) }}: - stage: publish_using_darc - dependsOn: Validate + ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: + dependsOn: Validate + ${{ if and(ne(parameters.enableNugetValidation, 'true'), ne(parameters.enableSigningValidation, 'true'), ne(parameters.enableSourceLinkValidation, 'true'), ne(parameters.SDLValidationParameters.enable, 'true')) }}: + dependsOn: ${{ parameters.validateDependsOn }} displayName: Publish using Darc variables: - template: common-variables.yml diff --git a/global.json b/global.json index aecb9328..ef0f5708 100644 --- a/global.json +++ b/global.json @@ -1,8 +1,8 @@ { "tools": { - "dotnet": "5.0.100-rc.1.20452.10" + "dotnet": "5.0.100-rc.2.20479.15" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20506.7" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.20515.7" } } From 67c1d9bced36779dbe8c11cf2d5afd13e7b06785 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 26 Oct 2020 12:26:31 +0000 Subject: [PATCH 101/161] Update dependencies from https://github.com/dotnet/arcade build 20201022.2 (#662) [master] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- eng/common/cross/build-rootfs.sh | 1 + eng/common/performance/crossgen_perf.proj | 27 ++++++++++++++++++----- eng/common/templates/job/job.yml | 8 ++++--- global.json | 2 +- 5 files changed, 31 insertions(+), 11 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 2d0e426e..58c89f15 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - d4de3ce701c2ba697d71cc70e2db69d7568913dc + c262e0121edb7df890528bb61589d4b6e440a1f6 https://github.com/dotnet/source-build-reference-packages diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index ffdff385..a841c64f 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -329,6 +329,7 @@ elif [[ -n $__CodeName ]]; then chroot $__RootfsDir apt-get -f -y install chroot $__RootfsDir apt-get -y install $__UbuntuPackages chroot $__RootfsDir symlinks -cr /usr + chroot $__RootfsDir apt clean if [ $__SkipUnmount == 0 ]; then umount $__RootfsDir/* || true diff --git a/eng/common/performance/crossgen_perf.proj b/eng/common/performance/crossgen_perf.proj index 42649203..3b8dfc2b 100644 --- a/eng/common/performance/crossgen_perf.proj +++ b/eng/common/performance/crossgen_perf.proj @@ -27,11 +27,18 @@ - - - - - + + + + + + + + + + + + @@ -51,6 +58,13 @@ + + + $(WorkItemDirectory) + $(Python) $(Crossgen2Directory)test.py crossgen2 --core-root $(CoreRoot) --single %(Identity) --singlethreaded True + + + @@ -60,6 +74,9 @@ 4:00 + + 4:00 + $(WorkItemDirectory) $(Python) $(Crossgen2Directory)test.py crossgen2 --core-root $(CoreRoot) --composite $(Crossgen2Directory)framework-r2r.dll.rsp diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 8b81a7e5..8e4ce2c2 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -27,6 +27,7 @@ parameters: useBuildManifest: false mergeTestResults: false testRunTitle: '' + testResultsFormat: '' name: '' preSteps: [] runAsPublic: false @@ -131,8 +132,8 @@ jobs: - task: RichCodeNavIndexer@0 displayName: RichCodeNav Upload inputs: - languages: 'csharp' - environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'prod') }} + languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }} + environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'production') }} richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin continueOnError: true @@ -202,7 +203,7 @@ jobs: continueOnError: true condition: always() - - ${{ if eq(parameters.enablePublishTestResults, 'true') }}: + - ${{ if or(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, 'xunit')) }}: - task: PublishTestResults@2 displayName: Publish XUnit Test Results inputs: @@ -213,6 +214,7 @@ jobs: mergeTestResults: ${{ parameters.mergeTestResults }} continueOnError: true condition: always() + - ${{ if or(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, 'vstest')) }}: - task: PublishTestResults@2 displayName: Publish TRX Test Results inputs: diff --git a/global.json b/global.json index ef0f5708..22258585 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "5.0.100-rc.2.20479.15" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.20515.7" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.20522.2" } } From d32ca9dc3838e5e290d201e91e96a11a7b8bd777 Mon Sep 17 00:00:00 2001 From: tmat Date: Tue, 27 Oct 2020 11:19:58 -0700 Subject: [PATCH 102/161] Add Microsoft.SourceLink.Tools source package implementing SourceLinkMap --- SourceLink.sln | 18 ++ eng/Versions.props | 1 + ...icrosoft.SourceLink.Tools.UnitTests.csproj | 12 + .../SourceLinkMapTests.cs | 194 +++++++++++++++ .../Microsoft.SourceLink.Tools.Package.csproj | 24 ++ .../Microsoft.SourceLink.Tools.projitems | 14 ++ .../Microsoft.SourceLink.Tools.shproj | 13 + src/SourceLink.Tools/SourceLinkMap.cs | 229 ++++++++++++++++++ 8 files changed, 505 insertions(+) create mode 100644 src/SourceLink.Tools.UnitTests/Microsoft.SourceLink.Tools.UnitTests.csproj create mode 100644 src/SourceLink.Tools.UnitTests/SourceLinkMapTests.cs create mode 100644 src/SourceLink.Tools/Microsoft.SourceLink.Tools.Package.csproj create mode 100644 src/SourceLink.Tools/Microsoft.SourceLink.Tools.projitems create mode 100644 src/SourceLink.Tools/Microsoft.SourceLink.Tools.shproj create mode 100644 src/SourceLink.Tools/SourceLinkMap.cs diff --git a/SourceLink.sln b/SourceLink.sln index b185660e..39b64cbd 100644 --- a/SourceLink.sln +++ b/SourceLink.sln @@ -58,7 +58,17 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.SourceLink.GitWeb EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.SourceLink.GitWeb.UnitTests", "src\SourceLink.GitWeb.UnitTests\Microsoft.SourceLink.GitWeb.UnitTests.csproj", "{50503A43-08C0-493B-B8CC-F368983644C1}" EndProject +Project("{D954291E-2A0B-460D-934E-DC6B0785DB48}") = "Microsoft.SourceLink.Tools", "src\SourceLink.Tools\Microsoft.SourceLink.Tools.shproj", "{5DF76CC2-5F0E-45A6-AD56-6BBBCCBC1A78}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.SourceLink.Tools.Package", "src\SourceLink.Tools\Microsoft.SourceLink.Tools.Package.csproj", "{141E6850-B424-47AD-884A-CED362027382}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.SourceLink.Tools.UnitTests", "src\SourceLink.Tools.UnitTests\Microsoft.SourceLink.Tools.UnitTests.csproj", "{99D113A9-24EC-471D-9F74-D2AC2F16220B}" +EndProject Global + GlobalSection(SharedMSBuildProjectFiles) = preSolution + src\SourceLink.Tools\Microsoft.SourceLink.Tools.projitems*{5df76cc2-5f0e-45a6-ad56-6bbbccbc1a78}*SharedItemsImports = 13 + src\SourceLink.Tools\Microsoft.SourceLink.Tools.projitems*{99d113a9-24ec-471d-9f74-d2ac2f16220b}*SharedItemsImports = 5 + EndGlobalSection GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU Release|Any CPU = Release|Any CPU @@ -148,6 +158,14 @@ Global {50503A43-08C0-493B-B8CC-F368983644C1}.Debug|Any CPU.Build.0 = Debug|Any CPU {50503A43-08C0-493B-B8CC-F368983644C1}.Release|Any CPU.ActiveCfg = Release|Any CPU {50503A43-08C0-493B-B8CC-F368983644C1}.Release|Any CPU.Build.0 = Release|Any CPU + {141E6850-B424-47AD-884A-CED362027382}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {141E6850-B424-47AD-884A-CED362027382}.Debug|Any CPU.Build.0 = Debug|Any CPU + {141E6850-B424-47AD-884A-CED362027382}.Release|Any CPU.ActiveCfg = Release|Any CPU + {141E6850-B424-47AD-884A-CED362027382}.Release|Any CPU.Build.0 = Release|Any CPU + {99D113A9-24EC-471D-9F74-D2AC2F16220B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {99D113A9-24EC-471D-9F74-D2AC2F16220B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {99D113A9-24EC-471D-9F74-D2AC2F16220B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {99D113A9-24EC-471D-9F74-D2AC2F16220B}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/eng/Versions.props b/eng/Versions.props index 1955a904..ec61a773 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -21,6 +21,7 @@ 2.1.0 4.9.2 4.5.0 + 4.7.2 0.26.0-preview-0070 diff --git a/src/SourceLink.Tools.UnitTests/Microsoft.SourceLink.Tools.UnitTests.csproj b/src/SourceLink.Tools.UnitTests/Microsoft.SourceLink.Tools.UnitTests.csproj new file mode 100644 index 00000000..194ea0a7 --- /dev/null +++ b/src/SourceLink.Tools.UnitTests/Microsoft.SourceLink.Tools.UnitTests.csproj @@ -0,0 +1,12 @@ + + + net472;netcoreapp3.1 + + + + + + + + + diff --git a/src/SourceLink.Tools.UnitTests/SourceLinkMapTests.cs b/src/SourceLink.Tools.UnitTests/SourceLinkMapTests.cs new file mode 100644 index 00000000..e143fae1 --- /dev/null +++ b/src/SourceLink.Tools.UnitTests/SourceLinkMapTests.cs @@ -0,0 +1,194 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.Json; +using TestUtilities; +using Xunit; + +namespace Microsoft.SourceLink.Tools.UnitTests +{ + public class SourceLinkMapTests + { + private IEnumerable Inspect(SourceLinkMap map) + => map.Entries.Select(e => $"('{e.FilePath.Path}', {(e.FilePath.IsPrefix ? "*" : "")}) -> ('{e.Uri.Prefix}', '{e.Uri.Suffix}')"); + + [Theory] + [InlineData(@"{}")] + [InlineData(@"{""xxx"":{}}")] + [InlineData(@"{""documents"":{}}")] + public void Empty(string json) + { + var map = SourceLinkMap.Parse(json); + Assert.Empty(map.Entries); + } + + [Fact] + public void Extra() + { + var map = SourceLinkMap.Parse(@" +{ + ""documents"" : + { + ""C:\\a*"": ""http://server/1/a*"", + }, + ""extra"": {} +}"); + AssertEx.Equal(new[] { "('C:\\a', *) -> ('http://server/1/a', '')" }, Inspect(map)); + } + + [Fact] + public void Entries() + { + var map = SourceLinkMap.Parse(@" +{ + ""documents"" : + { + ""C:\\a*"": ""http://server/[*]"", + ""C:\\a*"": ""http://a/"", + ""C:\\a"": ""http://a/"", + ""C:\\a*"": ""http://*a"", + ""C:\\b"": ""http://b"", + } +}"); + AssertEx.Equal(new[] + { + @"('C:\a', *) -> ('http://server/[', ']')", + @"('C:\a', *) -> ('http://a/', '')", + @"('C:\a', ) -> ('http://a/', '')", + @"('C:\a', *) -> ('http://', 'a')", + @"('C:\b', ) -> ('http://b', '')" + }, Inspect(map)); + + Assert.True(map.TryGetUrl(@"C:\a", out var url)); + Assert.Equal("http://server/[]", url); + + Assert.True(map.TryGetUrl(@"C:\a\b\c\d\e", out url)); + Assert.Equal("http://server/[/b/c/d/e]", url); + + Assert.True(map.TryGetUrl(@"C:\b", out url)); + Assert.Equal("http://b", url); + + Assert.False(map.TryGetUrl(@"C:\b\c", out _)); + } + + [Fact] + public void Order1() + { + var map = SourceLinkMap.Parse(@" +{ + ""documents"" : + { + ""C:\\a\\b*"": ""2:*"", + ""C:\\a\\b\\c*"": ""1:*"", + ""C:\\a*"": ""3:*"", + } +}"); + AssertEx.Equal(new[] + { + @"('C:\a\b\c', *) -> ('1:', '')", + @"('C:\a\b', *) -> ('2:', '')", + @"('C:\a', *) -> ('3:', '')" + }, Inspect(map)); + + string? url; + Assert.True(map.TryGetUrl(@"C:\a\b\c\d\e", out url)); + Assert.Equal("1:/d/e", url); + + Assert.True(map.TryGetUrl(@"C:\a\b\", out url)); + Assert.Equal("2:/", url); + + Assert.True(map.TryGetUrl(@"C:\a\x", out url)); + Assert.Equal("3:/x", url); + + Assert.False(map.TryGetUrl(@"D:\x", out _)); + } + + [Fact] + public void Order2() + { + var map = SourceLinkMap.Parse(@" +{ + ""documents"" : + { + ""C:\\aaa\\bbb*"": ""1:*"", + ""C:\\aaa\\bb*"": ""2:*"", + } +}"); + AssertEx.Equal(new[] + { + @"('C:\aaa\bbb', *) -> ('1:', '')", + @"('C:\aaa\bb', *) -> ('2:', '')", + }, Inspect(map)); + + string? url; + Assert.True(map.TryGetUrl(@"C:\aaa\bbbb", out url)); + Assert.Equal("1:b", url); + + Assert.True(map.TryGetUrl(@"C:\aaa\bbb", out url)); + Assert.Equal("1:", url); + + Assert.True(map.TryGetUrl(@"C:\aaa\bb", out url)); + Assert.Equal("2:", url); + + Assert.False(map.TryGetUrl(@"C:\aaa\b", out _)); + } + + [Fact] + public void TryGetUrl_Star() + { + var map = SourceLinkMap.Parse(@"{""documents"":{}}"); + Assert.False(map.TryGetUrl("path*", out _)); + } + + [Fact] + public void TryGetUrl_InvalidArgument() + { + var map = SourceLinkMap.Parse(@"{""documents"":{}}"); + Assert.Throws(() => map.TryGetUrl(null!, out _)); + } + + [Theory] + [InlineData(@"{")] + [InlineData(@"{""documents"" : { ""x"": ""y"" // comments not allowed +} }")] + [InlineData(@"{""documents"" : { 1: ""y"" } }")] + public void BadJson_Key(string json) + { + Assert.ThrowsAny(() => SourceLinkMap.Parse(json)); + } + + [Theory] + [InlineData(@"1")] + [InlineData(@"{""documents"": 1}")] + [InlineData(@"{""documents"":{""1"": 1}}")] + [InlineData(@"{""documents"":{""1"": null}}")] + [InlineData(@"{""documents"":{""1"": {}}}")] + [InlineData(@"{""documents"":{""1"": []}}")] + public void InvalidJsonTypes(string json) + { + Assert.Throws(() => SourceLinkMap.Parse(json)); + } + + [Theory] + [InlineData(@"{""documents"":{"""": ""x""}}")] + [InlineData(@"{""documents"":{""1**"": ""x""}}")] + [InlineData(@"{""documents"":{""*1*"": ""x""}}")] + [InlineData(@"{""documents"":{""1*"": ""**x""}}")] + [InlineData(@"{""documents"":{""*1"": ""x*""}}")] + [InlineData(@"{""documents"":{""1"": ""x*""}}")] + public void InvalidWildcards(string json) + { + Assert.Throws(() => SourceLinkMap.Parse(json)); + } + + [Fact] + public void Parse_InvalidArgument() + { + Assert.Throws(() => SourceLinkMap.Parse(null!)); + } + } +} diff --git a/src/SourceLink.Tools/Microsoft.SourceLink.Tools.Package.csproj b/src/SourceLink.Tools/Microsoft.SourceLink.Tools.Package.csproj new file mode 100644 index 00000000..578cfef0 --- /dev/null +++ b/src/SourceLink.Tools/Microsoft.SourceLink.Tools.Package.csproj @@ -0,0 +1,24 @@ + + + + + netcoreapp3.1;net472 + false + none + false + + + true + true + Microsoft.SourceLink.Tools + false + + Package containing sources of tools for reading Source Link. + + + $(NoWarn);NU5128 + + + + + \ No newline at end of file diff --git a/src/SourceLink.Tools/Microsoft.SourceLink.Tools.projitems b/src/SourceLink.Tools/Microsoft.SourceLink.Tools.projitems new file mode 100644 index 00000000..e52c86e4 --- /dev/null +++ b/src/SourceLink.Tools/Microsoft.SourceLink.Tools.projitems @@ -0,0 +1,14 @@ + + + + $(MSBuildAllProjects);$(MSBuildThisFileFullPath) + true + 5df76cc2-5f0e-45a6-ad56-6bbbccbc1a78 + + + Microsoft.SourceLink.Tools + + + + + \ No newline at end of file diff --git a/src/SourceLink.Tools/Microsoft.SourceLink.Tools.shproj b/src/SourceLink.Tools/Microsoft.SourceLink.Tools.shproj new file mode 100644 index 00000000..9e5b2a14 --- /dev/null +++ b/src/SourceLink.Tools/Microsoft.SourceLink.Tools.shproj @@ -0,0 +1,13 @@ + + + + 5df76cc2-5f0e-45a6-ad56-6bbbccbc1a78 + 14.0 + + + + + + + + diff --git a/src/SourceLink.Tools/SourceLinkMap.cs b/src/SourceLink.Tools/SourceLinkMap.cs new file mode 100644 index 00000000..776c75c6 --- /dev/null +++ b/src/SourceLink.Tools/SourceLinkMap.cs @@ -0,0 +1,229 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text.Json; + +#if NETCOREAPP +using System.Diagnostics.CodeAnalysis; +#endif + +namespace Microsoft.SourceLink.Tools +{ + /// + /// Source Link URL map. Maps file paths matching Source Link patterns to URLs. + /// + internal readonly struct SourceLinkMap + { + private readonly ReadOnlyCollection _entries; + + private SourceLinkMap(ReadOnlyCollection mappings) + { + _entries = mappings; + } + + public readonly struct Entry + { + public readonly FilePathPattern FilePath; + public readonly UriPattern Uri; + + public Entry(FilePathPattern filePath, UriPattern uri) + { + FilePath = filePath; + Uri = uri; + } + + public void Deconstruct(out FilePathPattern filePath, out UriPattern uri) + { + filePath = FilePath; + uri = Uri; + } + } + + public readonly struct FilePathPattern + { + public readonly string Path; + public readonly bool IsPrefix; + + public FilePathPattern(string path, bool isPrefix) + { + Path = path; + IsPrefix = isPrefix; + } + } + + public readonly struct UriPattern + { + public readonly string Prefix; + public readonly string Suffix; + + public UriPattern(string prefix, string suffix) + { + Prefix = prefix; + Suffix = suffix; + } + } + + public IReadOnlyList Entries => _entries; + + /// + /// Parses Source Link JSON string. + /// + /// is null. + /// The JSON does not follow Source Link specification. + /// is not valid JSON string. + public static SourceLinkMap Parse(string json) + { + if (json is null) + { + throw new ArgumentNullException(nameof(json)); + } + + var list = new List(); + + var root = JsonDocument.Parse(json, new JsonDocumentOptions() { AllowTrailingCommas = true }).RootElement; + if (root.ValueKind != JsonValueKind.Object) + { + throw new InvalidDataException(); + } + + foreach (var rootEntry in root.EnumerateObject()) + { + if (!rootEntry.NameEquals("documents")) + { + // potential future extensibility + continue; + } + + if (rootEntry.Value.ValueKind != JsonValueKind.Object) + { + throw new InvalidDataException(); + } + + foreach (var documentsEntry in rootEntry.Value.EnumerateObject()) + { + if (documentsEntry.Value.ValueKind != JsonValueKind.String || + !TryParseEntry(documentsEntry.Name, documentsEntry.Value.GetString(), out var entry)) + { + throw new InvalidDataException(); + } + + list.Add(entry); + } + } + + // Sort the map by decreasing file path length. This ensures that the most specific paths will checked before the least specific + // and that absolute paths will be checked before a wildcard path with a matching base + list.Sort((left, right) => -left.FilePath.Path.Length.CompareTo(right.FilePath.Path.Length)); + + return new SourceLinkMap(new ReadOnlyCollection(list)); + } + + private static bool TryParseEntry(string key, string value, out Entry entry) + { + entry = default; + + // VALIDATION RULES + // 1. The only acceptable wildcard is one and only one '*', which if present will be replaced by a relative path + // 2. If the filepath does not contain a *, the uri cannot contain a * and if the filepath contains a * the uri must contain a * + // 3. If the filepath contains a *, it must be the final character + // 4. If the uri contains a *, it may be anywhere in the uri + if (key.Length == 0) + { + return false; + } + + int filePathStar = key.IndexOf('*'); + if (filePathStar == key.Length - 1) + { + key = key.Substring(0, filePathStar); + } + else if (filePathStar >= 0) + { + return false; + } + + string uriPrefix, uriSuffix; + int uriStar = value.IndexOf('*'); + if (uriStar >= 0) + { + if (filePathStar < 0) + { + return false; + } + + uriPrefix = value.Substring(0, uriStar); + uriSuffix = value.Substring(uriStar + 1); + + if (uriSuffix.IndexOf('*') >= 0) + { + return false; + } + } + else + { + uriPrefix = value; + uriSuffix = ""; + } + + entry = new Entry( + new FilePathPattern(key, isPrefix: filePathStar >= 0), + new UriPattern(uriPrefix, uriSuffix)); + + return true; + } + + /// + /// Maps specified to the corresponding URL. + /// + /// is null. + public bool TryGetUrl( + string path, +#if NETCOREAPP + [NotNullWhen(true)] +#endif + out string? url) + { + if (path == null) + { + throw new ArgumentNullException(nameof(path)); + } + + if (path.IndexOf('*') >= 0) + { + url = null; + return false; + } + + // Note: the mapping function is case-insensitive. + + foreach (var (file, mappedUrl) in _entries) + { + if (file.IsPrefix) + { + if (path.StartsWith(file.Path, StringComparison.OrdinalIgnoreCase)) + { + var escapedPath = string.Join("/", path.Substring(file.Path.Length).Split(new[] { '/', '\\' }).Select(Uri.EscapeDataString)); + url = mappedUrl.Prefix + escapedPath + mappedUrl.Suffix; + return true; + } + } + else if (string.Equals(path, file.Path, StringComparison.OrdinalIgnoreCase)) + { + Debug.Assert(mappedUrl.Suffix.Length == 0); + url = mappedUrl.Prefix; + return true; + } + } + + url = null; + return false; + } + } +} \ No newline at end of file From eaf4d96095919f0c40121b268346c8a431db62bc Mon Sep 17 00:00:00 2001 From: tmat Date: Tue, 27 Oct 2020 11:38:53 -0700 Subject: [PATCH 103/161] Fix packaging --- src/Directory.Build.targets | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Directory.Build.targets b/src/Directory.Build.targets index d6017228..a4b4d2c1 100644 --- a/src/Directory.Build.targets +++ b/src/Directory.Build.targets @@ -17,7 +17,11 @@ false - + + $(_ProjectDefinedPackageId) $(AssemblyName) From 563830a5971728008de24b26f0ea86ed1c95fa5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Matou=C5=A1ek?= Date: Tue, 27 Oct 2020 13:40:26 -0700 Subject: [PATCH 104/161] Retarget to net5.0/net472 (#664) --- .../Microsoft.Build.StandardCI.csproj | 2 +- .../Microsoft.Build.Tasks.Git.UnitTests.csproj | 2 +- .../GitDataReader/GitIgnore.cs | 2 +- .../Microsoft.Build.Tasks.Git.csproj | 2 +- .../build/Microsoft.Build.Tasks.Git.props | 4 ++-- .../build/Microsoft.Build.Tasks.Tfvc.props | 2 +- ...eLink.AzureDevOpsServer.Git.UnitTests.csproj | 2 +- .../TranslateRepositoryUrlsTests.cs | 2 +- ...soft.SourceLink.AzureDevOpsServer.Git.csproj | 2 +- ...oft.SourceLink.AzureDevOpsServer.Git.targets | 4 ++-- ...t.SourceLink.AzureRepos.Git.UnitTests.csproj | 2 +- .../TranslateRepositoryUrlsTests.cs | 2 +- .../Microsoft.SourceLink.AzureRepos.Git.csproj | 2 +- .../Microsoft.SourceLink.AzureRepos.Git.targets | 4 ++-- ...Microsoft.SourceLink.AzureRepos.Tfvc.targets | 4 ++-- ...ft.SourceLink.Bitbucket.Git.UnitTests.csproj | 2 +- .../TranslateRepositoryUrlsTests.cs | 2 +- .../Microsoft.SourceLink.Bitbucket.Git.csproj | 2 +- .../Microsoft.SourceLink.Bitbucket.Git.targets | 4 ++-- ...Microsoft.SourceLink.Common.UnitTests.csproj | 2 +- .../TranslateRepositoryUrlsTests.cs | 2 +- .../Microsoft.SourceLink.Common.csproj | 2 +- .../build/Microsoft.SourceLink.Common.props | 4 ++-- ...osoft.SourceLink.Git.IntegrationTests.csproj | 2 +- .../Utilities/TestUtilities.cs | 17 ++++++++--------- ...Microsoft.SourceLink.GitHub.UnitTests.csproj | 2 +- .../TranslateRepositoryUrlsTests.cs | 2 +- .../Microsoft.SourceLink.GitHub.csproj | 2 +- .../build/Microsoft.SourceLink.GitHub.targets | 4 ++-- ...Microsoft.SourceLink.GitLab.UnitTests.csproj | 2 +- .../TranslateRepositoryUrlsTests.cs | 2 +- .../Microsoft.SourceLink.GitLab.csproj | 2 +- .../build/Microsoft.SourceLink.GitLab.targets | 4 ++-- ...Microsoft.SourceLink.GitWeb.UnitTests.csproj | 2 +- .../TranslateRepositoryUrlsTests.cs | 2 +- .../Microsoft.SourceLink.GitWeb.csproj | 2 +- .../build/Microsoft.SourceLink.GitWeb.targets | 4 ++-- .../Microsoft.SourceLink.Tools.UnitTests.csproj | 2 +- .../Microsoft.SourceLink.Tools.Package.csproj | 2 +- src/SourceLink.Tools/SourceLinkMap.cs | 4 ++-- src/TestUtilities/ProcessUtilities.cs | 8 ++++---- src/TestUtilities/TestUtilities.csproj | 2 +- 42 files changed, 62 insertions(+), 63 deletions(-) diff --git a/src/Microsoft.Build.StandardCI/Microsoft.Build.StandardCI.csproj b/src/Microsoft.Build.StandardCI/Microsoft.Build.StandardCI.csproj index 79031b5b..6f3f0405 100644 --- a/src/Microsoft.Build.StandardCI/Microsoft.Build.StandardCI.csproj +++ b/src/Microsoft.Build.StandardCI/Microsoft.Build.StandardCI.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp3.1 + net472;net5.0 true true true diff --git a/src/Microsoft.Build.Tasks.Git.UnitTests/Microsoft.Build.Tasks.Git.UnitTests.csproj b/src/Microsoft.Build.Tasks.Git.UnitTests/Microsoft.Build.Tasks.Git.UnitTests.csproj index 038b1028..fdbe4a08 100644 --- a/src/Microsoft.Build.Tasks.Git.UnitTests/Microsoft.Build.Tasks.Git.UnitTests.csproj +++ b/src/Microsoft.Build.Tasks.Git.UnitTests/Microsoft.Build.Tasks.Git.UnitTests.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp3.1 + net472;net5.0 diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitIgnore.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitIgnore.cs index c4e10052..47f75dd7 100644 --- a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitIgnore.cs +++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitIgnore.cs @@ -108,7 +108,7 @@ public Matcher CreateMatcher() StreamReader reader; try { - reader = File.OpenText(path); + reader = File.OpenText(path!); } catch (Exception e) when (e is FileNotFoundException || e is DirectoryNotFoundException) { diff --git a/src/Microsoft.Build.Tasks.Git/Microsoft.Build.Tasks.Git.csproj b/src/Microsoft.Build.Tasks.Git/Microsoft.Build.Tasks.Git.csproj index c8cc26b7..71b45dbb 100644 --- a/src/Microsoft.Build.Tasks.Git/Microsoft.Build.Tasks.Git.csproj +++ b/src/Microsoft.Build.Tasks.Git/Microsoft.Build.Tasks.Git.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp3.1 + net472;net5.0 true diff --git a/src/Microsoft.Build.Tasks.Git/build/Microsoft.Build.Tasks.Git.props b/src/Microsoft.Build.Tasks.Git/build/Microsoft.Build.Tasks.Git.props index 8204380b..337242ea 100644 --- a/src/Microsoft.Build.Tasks.Git/build/Microsoft.Build.Tasks.Git.props +++ b/src/Microsoft.Build.Tasks.Git/build/Microsoft.Build.Tasks.Git.props @@ -1,7 +1,7 @@ - $(MSBuildThisFileDirectory)..\tools\net461\Microsoft.Build.Tasks.Git.dll - $(MSBuildThisFileDirectory)..\tools\netcoreapp3.1\Microsoft.Build.Tasks.Git.dll + $(MSBuildThisFileDirectory)..\tools\net472\Microsoft.Build.Tasks.Git.dll + $(MSBuildThisFileDirectory)..\tools\net5.0\Microsoft.Build.Tasks.Git.dll diff --git a/src/Microsoft.Build.Tasks.Tfvc/build/Microsoft.Build.Tasks.Tfvc.props b/src/Microsoft.Build.Tasks.Tfvc/build/Microsoft.Build.Tasks.Tfvc.props index 5b41cd83..aee1e765 100644 --- a/src/Microsoft.Build.Tasks.Tfvc/build/Microsoft.Build.Tasks.Tfvc.props +++ b/src/Microsoft.Build.Tasks.Tfvc/build/Microsoft.Build.Tasks.Tfvc.props @@ -1,6 +1,6 @@ - $(MSBuildThisFileDirectory)..\tools\net461\Microsoft.Build.Tasks.Tfvc.dll + $(MSBuildThisFileDirectory)..\tools\net472\Microsoft.Build.Tasks.Tfvc.dll \ No newline at end of file diff --git a/src/SourceLink.AzureDevOpsServer.Git.UnitTests/Microsoft.SourceLink.AzureDevOpsServer.Git.UnitTests.csproj b/src/SourceLink.AzureDevOpsServer.Git.UnitTests/Microsoft.SourceLink.AzureDevOpsServer.Git.UnitTests.csproj index 8c8d3235..9ff7a02d 100644 --- a/src/SourceLink.AzureDevOpsServer.Git.UnitTests/Microsoft.SourceLink.AzureDevOpsServer.Git.UnitTests.csproj +++ b/src/SourceLink.AzureDevOpsServer.Git.UnitTests/Microsoft.SourceLink.AzureDevOpsServer.Git.UnitTests.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp3.1 + net472;net5.0 diff --git a/src/SourceLink.AzureDevOpsServer.Git.UnitTests/TranslateRepositoryUrlsTests.cs b/src/SourceLink.AzureDevOpsServer.Git.UnitTests/TranslateRepositoryUrlsTests.cs index 1dc56e41..81ef6a32 100644 --- a/src/SourceLink.AzureDevOpsServer.Git.UnitTests/TranslateRepositoryUrlsTests.cs +++ b/src/SourceLink.AzureDevOpsServer.Git.UnitTests/TranslateRepositoryUrlsTests.cs @@ -47,7 +47,7 @@ public void Translate() "ssh://account@mytfs.com/tfs/project/team/_ssh/repo", "ssh://account@contoso2.com:22/tfs/project/team/_ssh/repo", "ssh://account@contoso.com:22/tfs/project/team/ZZZ/repo" - }, task.TranslatedSourceRoots.Select(r => r.GetMetadata("ScmRepositoryUrl"))); + }, task.TranslatedSourceRoots?.Select(r => r.GetMetadata("ScmRepositoryUrl"))); Assert.True(result); } diff --git a/src/SourceLink.AzureDevOpsServer.Git/Microsoft.SourceLink.AzureDevOpsServer.Git.csproj b/src/SourceLink.AzureDevOpsServer.Git/Microsoft.SourceLink.AzureDevOpsServer.Git.csproj index 0d4851b3..b905b8e5 100644 --- a/src/SourceLink.AzureDevOpsServer.Git/Microsoft.SourceLink.AzureDevOpsServer.Git.csproj +++ b/src/SourceLink.AzureDevOpsServer.Git/Microsoft.SourceLink.AzureDevOpsServer.Git.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp3.1 + net472;net5.0 true diff --git a/src/SourceLink.AzureDevOpsServer.Git/build/Microsoft.SourceLink.AzureDevOpsServer.Git.targets b/src/SourceLink.AzureDevOpsServer.Git/build/Microsoft.SourceLink.AzureDevOpsServer.Git.targets index 94a6ca8b..1c2b313b 100644 --- a/src/SourceLink.AzureDevOpsServer.Git/build/Microsoft.SourceLink.AzureDevOpsServer.Git.targets +++ b/src/SourceLink.AzureDevOpsServer.Git/build/Microsoft.SourceLink.AzureDevOpsServer.Git.targets @@ -1,8 +1,8 @@ - <_SourceLinkAzureDevOpsServerGitAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net461\Microsoft.SourceLink.AzureDevOpsServer.Git.dll - <_SourceLinkAzureDevOpsServerGitAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\netcoreapp3.1\Microsoft.SourceLink.AzureDevOpsServer.Git.dll + <_SourceLinkAzureDevOpsServerGitAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net472\Microsoft.SourceLink.AzureDevOpsServer.Git.dll + <_SourceLinkAzureDevOpsServerGitAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\net5.0\Microsoft.SourceLink.AzureDevOpsServer.Git.dll diff --git a/src/SourceLink.AzureRepos.Git.UnitTests/Microsoft.SourceLink.AzureRepos.Git.UnitTests.csproj b/src/SourceLink.AzureRepos.Git.UnitTests/Microsoft.SourceLink.AzureRepos.Git.UnitTests.csproj index 504f0e01..180dd936 100644 --- a/src/SourceLink.AzureRepos.Git.UnitTests/Microsoft.SourceLink.AzureRepos.Git.UnitTests.csproj +++ b/src/SourceLink.AzureRepos.Git.UnitTests/Microsoft.SourceLink.AzureRepos.Git.UnitTests.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp3.1 + net472;net5.0 diff --git a/src/SourceLink.AzureRepos.Git.UnitTests/TranslateRepositoryUrlsTests.cs b/src/SourceLink.AzureRepos.Git.UnitTests/TranslateRepositoryUrlsTests.cs index 3e09d943..adbb8c0a 100644 --- a/src/SourceLink.AzureRepos.Git.UnitTests/TranslateRepositoryUrlsTests.cs +++ b/src/SourceLink.AzureRepos.Git.UnitTests/TranslateRepositoryUrlsTests.cs @@ -56,7 +56,7 @@ public void Translate() "ssh://account@vs-ssh.contoso.com:22/project/team/_ssh/repo", "ssh://account@vs-ssh.contoso2.com:22/project/team/_ssh/repo", "ssh://account@vs-ssh.contoso.com:22/project/team/ZZZ/repo" - }, task.TranslatedSourceRoots.Select(r => r.GetMetadata("ScmRepositoryUrl"))); + }, task.TranslatedSourceRoots?.Select(r => r.GetMetadata("ScmRepositoryUrl"))); Assert.True(result); } diff --git a/src/SourceLink.AzureRepos.Git/Microsoft.SourceLink.AzureRepos.Git.csproj b/src/SourceLink.AzureRepos.Git/Microsoft.SourceLink.AzureRepos.Git.csproj index 816dcfc9..8a5a0bf3 100644 --- a/src/SourceLink.AzureRepos.Git/Microsoft.SourceLink.AzureRepos.Git.csproj +++ b/src/SourceLink.AzureRepos.Git/Microsoft.SourceLink.AzureRepos.Git.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp3.1 + net472;net5.0 true diff --git a/src/SourceLink.AzureRepos.Git/build/Microsoft.SourceLink.AzureRepos.Git.targets b/src/SourceLink.AzureRepos.Git/build/Microsoft.SourceLink.AzureRepos.Git.targets index e955685e..d49e459e 100644 --- a/src/SourceLink.AzureRepos.Git/build/Microsoft.SourceLink.AzureRepos.Git.targets +++ b/src/SourceLink.AzureRepos.Git/build/Microsoft.SourceLink.AzureRepos.Git.targets @@ -1,8 +1,8 @@ - <_SourceLinkAzureReposGitAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net461\Microsoft.SourceLink.AzureRepos.Git.dll - <_SourceLinkAzureReposGitAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\netcoreapp3.1\Microsoft.SourceLink.AzureRepos.Git.dll + <_SourceLinkAzureReposGitAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net472\Microsoft.SourceLink.AzureRepos.Git.dll + <_SourceLinkAzureReposGitAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\net5.0\Microsoft.SourceLink.AzureRepos.Git.dll diff --git a/src/SourceLink.AzureRepos.Tfvc/build/Microsoft.SourceLink.AzureRepos.Tfvc.targets b/src/SourceLink.AzureRepos.Tfvc/build/Microsoft.SourceLink.AzureRepos.Tfvc.targets index 3a26e1a5..f4045fb8 100644 --- a/src/SourceLink.AzureRepos.Tfvc/build/Microsoft.SourceLink.AzureRepos.Tfvc.targets +++ b/src/SourceLink.AzureRepos.Tfvc/build/Microsoft.SourceLink.AzureRepos.Tfvc.targets @@ -1,8 +1,8 @@ - <_SourceLinkAzureReposTfvcAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net461\Microsoft.SourceLink.AzureRepos.Tfvc.dll - <_SourceLinkAzureReposTfvcAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\netcoreapp3.1\Microsoft.SourceLink.AzureRepos.Tfvc.dll + <_SourceLinkAzureReposTfvcAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net472\Microsoft.SourceLink.AzureRepos.Tfvc.dll + <_SourceLinkAzureReposTfvcAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\net5.0\Microsoft.SourceLink.AzureRepos.Tfvc.dll diff --git a/src/SourceLink.Bitbucket.Git.UnitTests/Microsoft.SourceLink.Bitbucket.Git.UnitTests.csproj b/src/SourceLink.Bitbucket.Git.UnitTests/Microsoft.SourceLink.Bitbucket.Git.UnitTests.csproj index bbe4ecba..a0c5edf4 100644 --- a/src/SourceLink.Bitbucket.Git.UnitTests/Microsoft.SourceLink.Bitbucket.Git.UnitTests.csproj +++ b/src/SourceLink.Bitbucket.Git.UnitTests/Microsoft.SourceLink.Bitbucket.Git.UnitTests.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp3.1 + net472;net5.0 diff --git a/src/SourceLink.Bitbucket.Git.UnitTests/TranslateRepositoryUrlsTests.cs b/src/SourceLink.Bitbucket.Git.UnitTests/TranslateRepositoryUrlsTests.cs index 4d0f737b..68af468d 100644 --- a/src/SourceLink.Bitbucket.Git.UnitTests/TranslateRepositoryUrlsTests.cs +++ b/src/SourceLink.Bitbucket.Git.UnitTests/TranslateRepositoryUrlsTests.cs @@ -42,7 +42,7 @@ public void Translate() "ssh://bitbucket1.org/a/b", "https://bitbucket1.org/a/b", "ssh://bitbucket2.org/a/b" - }, task.TranslatedSourceRoots.Select(r => r.GetMetadata("ScmRepositoryUrl"))); + }, task.TranslatedSourceRoots?.Select(r => r.GetMetadata("ScmRepositoryUrl"))); Assert.True(result); } diff --git a/src/SourceLink.Bitbucket.Git/Microsoft.SourceLink.Bitbucket.Git.csproj b/src/SourceLink.Bitbucket.Git/Microsoft.SourceLink.Bitbucket.Git.csproj index 2f00828c..387b65c0 100644 --- a/src/SourceLink.Bitbucket.Git/Microsoft.SourceLink.Bitbucket.Git.csproj +++ b/src/SourceLink.Bitbucket.Git/Microsoft.SourceLink.Bitbucket.Git.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp3.1 + net472;net5.0 true diff --git a/src/SourceLink.Bitbucket.Git/build/Microsoft.SourceLink.Bitbucket.Git.targets b/src/SourceLink.Bitbucket.Git/build/Microsoft.SourceLink.Bitbucket.Git.targets index 12fc0b8d..0cb1491d 100644 --- a/src/SourceLink.Bitbucket.Git/build/Microsoft.SourceLink.Bitbucket.Git.targets +++ b/src/SourceLink.Bitbucket.Git/build/Microsoft.SourceLink.Bitbucket.Git.targets @@ -1,8 +1,8 @@ - <_SourceLinkBitbucketAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net461\Microsoft.SourceLink.Bitbucket.Git.dll - <_SourceLinkBitbucketAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\netcoreapp3.1\Microsoft.SourceLink.Bitbucket.Git.dll + <_SourceLinkBitbucketAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net472\Microsoft.SourceLink.Bitbucket.Git.dll + <_SourceLinkBitbucketAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\net5.0\Microsoft.SourceLink.Bitbucket.Git.dll diff --git a/src/SourceLink.Common.UnitTests/Microsoft.SourceLink.Common.UnitTests.csproj b/src/SourceLink.Common.UnitTests/Microsoft.SourceLink.Common.UnitTests.csproj index 54a8a2fc..4ca53283 100644 --- a/src/SourceLink.Common.UnitTests/Microsoft.SourceLink.Common.UnitTests.csproj +++ b/src/SourceLink.Common.UnitTests/Microsoft.SourceLink.Common.UnitTests.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp3.1 + net472;net5.0 diff --git a/src/SourceLink.Common.UnitTests/TranslateRepositoryUrlsTests.cs b/src/SourceLink.Common.UnitTests/TranslateRepositoryUrlsTests.cs index 39c63ddc..9c77a292 100644 --- a/src/SourceLink.Common.UnitTests/TranslateRepositoryUrlsTests.cs +++ b/src/SourceLink.Common.UnitTests/TranslateRepositoryUrlsTests.cs @@ -49,7 +49,7 @@ public void Translate() "ftp://account@contoso.com:123/a/b?x=y", "ssh://account@contoso.com:123/a/b?x=y", "ssh://account@contoso2.com:123/a/b?x=y" - }, task.TranslatedSourceRoots.Select(r => r.GetMetadata("ScmRepositoryUrl"))); + }, task.TranslatedSourceRoots?.Select(r => r.GetMetadata("ScmRepositoryUrl"))); Assert.True(result); } diff --git a/src/SourceLink.Common/Microsoft.SourceLink.Common.csproj b/src/SourceLink.Common/Microsoft.SourceLink.Common.csproj index eeb641d3..b2205638 100644 --- a/src/SourceLink.Common/Microsoft.SourceLink.Common.csproj +++ b/src/SourceLink.Common/Microsoft.SourceLink.Common.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp3.1 + net472;net5.0 true diff --git a/src/SourceLink.Common/build/Microsoft.SourceLink.Common.props b/src/SourceLink.Common/build/Microsoft.SourceLink.Common.props index 4d3ac740..ac63a8ae 100644 --- a/src/SourceLink.Common/build/Microsoft.SourceLink.Common.props +++ b/src/SourceLink.Common/build/Microsoft.SourceLink.Common.props @@ -1,8 +1,8 @@ - <_MicrosoftSourceLinkCommonAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net461\Microsoft.SourceLink.Common.dll - <_MicrosoftSourceLinkCommonAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\netcoreapp3.1\Microsoft.SourceLink.Common.dll + <_MicrosoftSourceLinkCommonAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net472\Microsoft.SourceLink.Common.dll + <_MicrosoftSourceLinkCommonAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\net5.0\Microsoft.SourceLink.Common.dll diff --git a/src/SourceLink.Git.IntegrationTests/Microsoft.SourceLink.Git.IntegrationTests.csproj b/src/SourceLink.Git.IntegrationTests/Microsoft.SourceLink.Git.IntegrationTests.csproj index 60e8ea90..12a748c5 100644 --- a/src/SourceLink.Git.IntegrationTests/Microsoft.SourceLink.Git.IntegrationTests.csproj +++ b/src/SourceLink.Git.IntegrationTests/Microsoft.SourceLink.Git.IntegrationTests.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp3.1 + net472;net5.0 diff --git a/src/SourceLink.Git.IntegrationTests/Utilities/TestUtilities.cs b/src/SourceLink.Git.IntegrationTests/Utilities/TestUtilities.cs index c26b8307..0ca5030c 100644 --- a/src/SourceLink.Git.IntegrationTests/Utilities/TestUtilities.cs +++ b/src/SourceLink.Git.IntegrationTests/Utilities/TestUtilities.cs @@ -20,15 +20,14 @@ public static void ValidateAssemblyInformationalVersion(string assembylPath, str public static void ValidateNuSpecRepository(string nuspecPath, string type, string commit, string url) { - using (var archive = new ZipArchive(File.OpenRead(nuspecPath))) - using (var nuspecStream = archive.GetEntry("test.nuspec").Open()) - { - var nuspec = XDocument.Load(nuspecStream); - var repositoryNode = nuspec.Descendants(XName.Get("repository", "http://schemas.microsoft.com/packaging/2012/06/nuspec.xsd")); - AssertEx.AreEqual(type, repositoryNode.Attributes("type").Single().Value); - AssertEx.AreEqual(commit, repositoryNode.Attributes("commit").Single().Value); - AssertEx.AreEqual(url, repositoryNode.Attributes("url").Single().Value); - } + using var archive = new ZipArchive(File.OpenRead(nuspecPath)); + using var nuspecStream = archive.GetEntry("test.nuspec")!.Open(); + + var nuspec = XDocument.Load(nuspecStream); + var repositoryNode = nuspec.Descendants(XName.Get("repository", "http://schemas.microsoft.com/packaging/2012/06/nuspec.xsd")); + AssertEx.AreEqual(type, repositoryNode.Attributes("type").Single().Value); + AssertEx.AreEqual(commit, repositoryNode.Attributes("commit").Single().Value); + AssertEx.AreEqual(url, repositoryNode.Attributes("url").Single().Value); } } } diff --git a/src/SourceLink.GitHub.UnitTests/Microsoft.SourceLink.GitHub.UnitTests.csproj b/src/SourceLink.GitHub.UnitTests/Microsoft.SourceLink.GitHub.UnitTests.csproj index 5dfd5d1c..d229ac5d 100644 --- a/src/SourceLink.GitHub.UnitTests/Microsoft.SourceLink.GitHub.UnitTests.csproj +++ b/src/SourceLink.GitHub.UnitTests/Microsoft.SourceLink.GitHub.UnitTests.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp3.1 + net472;net5.0 diff --git a/src/SourceLink.GitHub.UnitTests/TranslateRepositoryUrlsTests.cs b/src/SourceLink.GitHub.UnitTests/TranslateRepositoryUrlsTests.cs index 9c99a608..3310cbd3 100644 --- a/src/SourceLink.GitHub.UnitTests/TranslateRepositoryUrlsTests.cs +++ b/src/SourceLink.GitHub.UnitTests/TranslateRepositoryUrlsTests.cs @@ -42,7 +42,7 @@ public void Translate() "ssh://github1.com/a/b", "https://github1.com/a/b", "ssh://github2.com/a/b" - }, task.TranslatedSourceRoots.Select(r => r.GetMetadata("ScmRepositoryUrl"))); + }, task.TranslatedSourceRoots?.Select(r => r.GetMetadata("ScmRepositoryUrl"))); Assert.True(result); } diff --git a/src/SourceLink.GitHub/Microsoft.SourceLink.GitHub.csproj b/src/SourceLink.GitHub/Microsoft.SourceLink.GitHub.csproj index 56e25605..45985c8e 100644 --- a/src/SourceLink.GitHub/Microsoft.SourceLink.GitHub.csproj +++ b/src/SourceLink.GitHub/Microsoft.SourceLink.GitHub.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp3.1 + net472;net5.0 true diff --git a/src/SourceLink.GitHub/build/Microsoft.SourceLink.GitHub.targets b/src/SourceLink.GitHub/build/Microsoft.SourceLink.GitHub.targets index 3300c412..2641835e 100644 --- a/src/SourceLink.GitHub/build/Microsoft.SourceLink.GitHub.targets +++ b/src/SourceLink.GitHub/build/Microsoft.SourceLink.GitHub.targets @@ -1,8 +1,8 @@ - <_SourceLinkGitHubAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net461\Microsoft.SourceLink.GitHub.dll - <_SourceLinkGitHubAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\netcoreapp3.1\Microsoft.SourceLink.GitHub.dll + <_SourceLinkGitHubAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net472\Microsoft.SourceLink.GitHub.dll + <_SourceLinkGitHubAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\net5.0\Microsoft.SourceLink.GitHub.dll diff --git a/src/SourceLink.GitLab.UnitTests/Microsoft.SourceLink.GitLab.UnitTests.csproj b/src/SourceLink.GitLab.UnitTests/Microsoft.SourceLink.GitLab.UnitTests.csproj index c74ab311..c31a3718 100644 --- a/src/SourceLink.GitLab.UnitTests/Microsoft.SourceLink.GitLab.UnitTests.csproj +++ b/src/SourceLink.GitLab.UnitTests/Microsoft.SourceLink.GitLab.UnitTests.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp3.1 + net472;net5.0 diff --git a/src/SourceLink.GitLab.UnitTests/TranslateRepositoryUrlsTests.cs b/src/SourceLink.GitLab.UnitTests/TranslateRepositoryUrlsTests.cs index 1d2db5b5..e536cf20 100644 --- a/src/SourceLink.GitLab.UnitTests/TranslateRepositoryUrlsTests.cs +++ b/src/SourceLink.GitLab.UnitTests/TranslateRepositoryUrlsTests.cs @@ -42,7 +42,7 @@ public void Translate() "ssh://gitlab1.com/a/b", "https://gitlab1.com/a/b", "ssh://gitlab2.com/a/b" - }, task.TranslatedSourceRoots.Select(r => r.GetMetadata("ScmRepositoryUrl"))); + }, task.TranslatedSourceRoots?.Select(r => r.GetMetadata("ScmRepositoryUrl"))); Assert.True(result); } diff --git a/src/SourceLink.GitLab/Microsoft.SourceLink.GitLab.csproj b/src/SourceLink.GitLab/Microsoft.SourceLink.GitLab.csproj index 440baba4..18300cef 100644 --- a/src/SourceLink.GitLab/Microsoft.SourceLink.GitLab.csproj +++ b/src/SourceLink.GitLab/Microsoft.SourceLink.GitLab.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp3.1 + net472;net5.0 true diff --git a/src/SourceLink.GitLab/build/Microsoft.SourceLink.GitLab.targets b/src/SourceLink.GitLab/build/Microsoft.SourceLink.GitLab.targets index 608000b3..ffc5140d 100644 --- a/src/SourceLink.GitLab/build/Microsoft.SourceLink.GitLab.targets +++ b/src/SourceLink.GitLab/build/Microsoft.SourceLink.GitLab.targets @@ -1,8 +1,8 @@ - <_SourceLinkGitLabAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net461\Microsoft.SourceLink.GitLab.dll - <_SourceLinkGitLabAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\netcoreapp3.1\Microsoft.SourceLink.GitLab.dll + <_SourceLinkGitLabAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net472\Microsoft.SourceLink.GitLab.dll + <_SourceLinkGitLabAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\net5.0\Microsoft.SourceLink.GitLab.dll diff --git a/src/SourceLink.GitWeb.UnitTests/Microsoft.SourceLink.GitWeb.UnitTests.csproj b/src/SourceLink.GitWeb.UnitTests/Microsoft.SourceLink.GitWeb.UnitTests.csproj index 411ec39f..5a9bcb86 100644 --- a/src/SourceLink.GitWeb.UnitTests/Microsoft.SourceLink.GitWeb.UnitTests.csproj +++ b/src/SourceLink.GitWeb.UnitTests/Microsoft.SourceLink.GitWeb.UnitTests.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp3.1 + net472;net5.0 diff --git a/src/SourceLink.GitWeb.UnitTests/TranslateRepositoryUrlsTests.cs b/src/SourceLink.GitWeb.UnitTests/TranslateRepositoryUrlsTests.cs index 255fdb2f..91ca1976 100644 --- a/src/SourceLink.GitWeb.UnitTests/TranslateRepositoryUrlsTests.cs +++ b/src/SourceLink.GitWeb.UnitTests/TranslateRepositoryUrlsTests.cs @@ -43,7 +43,7 @@ public void Translate() "ssh://git@src.intranet.company1.com/root_dir_name/sub_dirs/reponame.git", "ssh://git@src.intranet.company1.com/root_dir_name/sub_dirs/reponame.git", "ssh://git@src.intranet.company2.com/root_dir_name/sub_dirs/reponame.git", - }, task.TranslatedSourceRoots.Select(r => r.GetMetadata("ScmRepositoryUrl"))); + }, task.TranslatedSourceRoots?.Select(r => r.GetMetadata("ScmRepositoryUrl"))); Assert.True(result); } diff --git a/src/SourceLink.GitWeb/Microsoft.SourceLink.GitWeb.csproj b/src/SourceLink.GitWeb/Microsoft.SourceLink.GitWeb.csproj index 3fd7502a..1991db75 100644 --- a/src/SourceLink.GitWeb/Microsoft.SourceLink.GitWeb.csproj +++ b/src/SourceLink.GitWeb/Microsoft.SourceLink.GitWeb.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp3.1 + net472;net5.0 true diff --git a/src/SourceLink.GitWeb/build/Microsoft.SourceLink.GitWeb.targets b/src/SourceLink.GitWeb/build/Microsoft.SourceLink.GitWeb.targets index 065585ee..9f21d62b 100644 --- a/src/SourceLink.GitWeb/build/Microsoft.SourceLink.GitWeb.targets +++ b/src/SourceLink.GitWeb/build/Microsoft.SourceLink.GitWeb.targets @@ -1,8 +1,8 @@ - <_SourceLinkGitWebAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net461\Microsoft.SourceLink.GitWeb.dll - <_SourceLinkGitWebAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\netcoreapp3.1\Microsoft.SourceLink.GitWeb.dll + <_SourceLinkGitWebAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net472\Microsoft.SourceLink.GitWeb.dll + <_SourceLinkGitWebAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\net5.0\Microsoft.SourceLink.GitWeb.dll diff --git a/src/SourceLink.Tools.UnitTests/Microsoft.SourceLink.Tools.UnitTests.csproj b/src/SourceLink.Tools.UnitTests/Microsoft.SourceLink.Tools.UnitTests.csproj index 194ea0a7..8d211b84 100644 --- a/src/SourceLink.Tools.UnitTests/Microsoft.SourceLink.Tools.UnitTests.csproj +++ b/src/SourceLink.Tools.UnitTests/Microsoft.SourceLink.Tools.UnitTests.csproj @@ -1,6 +1,6 @@  - net472;netcoreapp3.1 + net472;net5.0 diff --git a/src/SourceLink.Tools/Microsoft.SourceLink.Tools.Package.csproj b/src/SourceLink.Tools/Microsoft.SourceLink.Tools.Package.csproj index 578cfef0..053f1245 100644 --- a/src/SourceLink.Tools/Microsoft.SourceLink.Tools.Package.csproj +++ b/src/SourceLink.Tools/Microsoft.SourceLink.Tools.Package.csproj @@ -2,7 +2,7 @@ - netcoreapp3.1;net472 + net5.0;net472 false none false diff --git a/src/SourceLink.Tools/SourceLinkMap.cs b/src/SourceLink.Tools/SourceLinkMap.cs index 776c75c6..91d74ac4 100644 --- a/src/SourceLink.Tools/SourceLinkMap.cs +++ b/src/SourceLink.Tools/SourceLinkMap.cs @@ -109,7 +109,7 @@ public static SourceLinkMap Parse(string json) foreach (var documentsEntry in rootEntry.Value.EnumerateObject()) { if (documentsEntry.Value.ValueKind != JsonValueKind.String || - !TryParseEntry(documentsEntry.Name, documentsEntry.Value.GetString(), out var entry)) + !TryParseEntry(documentsEntry.Name, documentsEntry.Value.GetString()!, out var entry)) { throw new InvalidDataException(); } @@ -182,7 +182,7 @@ private static bool TryParseEntry(string key, string value, out Entry entry) /// /// Maps specified to the corresponding URL. /// - /// is null. + /// is null. public bool TryGetUrl( string path, #if NETCOREAPP diff --git a/src/TestUtilities/ProcessUtilities.cs b/src/TestUtilities/ProcessUtilities.cs index 83406fb8..445b54cb 100644 --- a/src/TestUtilities/ProcessUtilities.cs +++ b/src/TestUtilities/ProcessUtilities.cs @@ -16,7 +16,7 @@ public static class ProcessUtilities public static ProcessResult Run( string fileName, string arguments, - string? workingDirectory = null, + string workingDirectory = "", IEnumerable>? additionalEnvironmentVars = null, string? stdInput = null) { @@ -88,7 +88,7 @@ public static ProcessResult Run( /// Launch a process, and return Process object. The process continues to run asynchronously. /// You cannot capture the output. /// - public static Process StartProcess(string fileName, string arguments, string? workingDirectory = null) + public static Process StartProcess(string fileName, string arguments, string workingDirectory = "") { if (fileName == null) { @@ -129,12 +129,12 @@ public static string RunAndGetOutput(string exeFileName, string? arguments = nul startInfo.WorkingDirectory = startFolder; } - using (var process = System.Diagnostics.Process.Start(startInfo)) + using (var process = Process.Start(startInfo)) { // Do not wait for the child process to exit before reading to the end of its // redirected stream. Read the output stream first and then wait. Doing otherwise // might cause a deadlock. - result = process.StandardOutput.ReadToEnd(); + result = process!.StandardOutput.ReadToEnd(); process.WaitForExit(); Assert.True(expectedRetCode == process.ExitCode, $"Unexpected exit code: {process.ExitCode} (expecting {expectedRetCode}). Process output: {result}"); } diff --git a/src/TestUtilities/TestUtilities.csproj b/src/TestUtilities/TestUtilities.csproj index 22b4d7e3..f5588696 100644 --- a/src/TestUtilities/TestUtilities.csproj +++ b/src/TestUtilities/TestUtilities.csproj @@ -1,6 +1,6 @@  - net461;netcoreapp3.1 + net472;net5.0 false true From a5792fc4015f9a83e4a1a9857e65789d52605b0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Matou=C5=A1ek?= Date: Tue, 27 Oct 2020 15:19:05 -0700 Subject: [PATCH 105/161] Rename TryGetUrl to TryGetUri (#665) --- .../SourceLinkMapTests.cs | 28 +++++++++---------- src/SourceLink.Tools/SourceLinkMap.cs | 16 +++++------ 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/SourceLink.Tools.UnitTests/SourceLinkMapTests.cs b/src/SourceLink.Tools.UnitTests/SourceLinkMapTests.cs index e143fae1..db134dfa 100644 --- a/src/SourceLink.Tools.UnitTests/SourceLinkMapTests.cs +++ b/src/SourceLink.Tools.UnitTests/SourceLinkMapTests.cs @@ -63,16 +63,16 @@ public void Entries() @"('C:\b', ) -> ('http://b', '')" }, Inspect(map)); - Assert.True(map.TryGetUrl(@"C:\a", out var url)); + Assert.True(map.TryGetUri(@"C:\a", out var url)); Assert.Equal("http://server/[]", url); - Assert.True(map.TryGetUrl(@"C:\a\b\c\d\e", out url)); + Assert.True(map.TryGetUri(@"C:\a\b\c\d\e", out url)); Assert.Equal("http://server/[/b/c/d/e]", url); - Assert.True(map.TryGetUrl(@"C:\b", out url)); + Assert.True(map.TryGetUri(@"C:\b", out url)); Assert.Equal("http://b", url); - Assert.False(map.TryGetUrl(@"C:\b\c", out _)); + Assert.False(map.TryGetUri(@"C:\b\c", out _)); } [Fact] @@ -95,16 +95,16 @@ public void Order1() }, Inspect(map)); string? url; - Assert.True(map.TryGetUrl(@"C:\a\b\c\d\e", out url)); + Assert.True(map.TryGetUri(@"C:\a\b\c\d\e", out url)); Assert.Equal("1:/d/e", url); - Assert.True(map.TryGetUrl(@"C:\a\b\", out url)); + Assert.True(map.TryGetUri(@"C:\a\b\", out url)); Assert.Equal("2:/", url); - Assert.True(map.TryGetUrl(@"C:\a\x", out url)); + Assert.True(map.TryGetUri(@"C:\a\x", out url)); Assert.Equal("3:/x", url); - Assert.False(map.TryGetUrl(@"D:\x", out _)); + Assert.False(map.TryGetUri(@"D:\x", out _)); } [Fact] @@ -125,30 +125,30 @@ public void Order2() }, Inspect(map)); string? url; - Assert.True(map.TryGetUrl(@"C:\aaa\bbbb", out url)); + Assert.True(map.TryGetUri(@"C:\aaa\bbbb", out url)); Assert.Equal("1:b", url); - Assert.True(map.TryGetUrl(@"C:\aaa\bbb", out url)); + Assert.True(map.TryGetUri(@"C:\aaa\bbb", out url)); Assert.Equal("1:", url); - Assert.True(map.TryGetUrl(@"C:\aaa\bb", out url)); + Assert.True(map.TryGetUri(@"C:\aaa\bb", out url)); Assert.Equal("2:", url); - Assert.False(map.TryGetUrl(@"C:\aaa\b", out _)); + Assert.False(map.TryGetUri(@"C:\aaa\b", out _)); } [Fact] public void TryGetUrl_Star() { var map = SourceLinkMap.Parse(@"{""documents"":{}}"); - Assert.False(map.TryGetUrl("path*", out _)); + Assert.False(map.TryGetUri("path*", out _)); } [Fact] public void TryGetUrl_InvalidArgument() { var map = SourceLinkMap.Parse(@"{""documents"":{}}"); - Assert.Throws(() => map.TryGetUrl(null!, out _)); + Assert.Throws(() => map.TryGetUri(null!, out _)); } [Theory] diff --git a/src/SourceLink.Tools/SourceLinkMap.cs b/src/SourceLink.Tools/SourceLinkMap.cs index 91d74ac4..2aff6b81 100644 --- a/src/SourceLink.Tools/SourceLinkMap.cs +++ b/src/SourceLink.Tools/SourceLinkMap.cs @@ -183,12 +183,12 @@ private static bool TryParseEntry(string key, string value, out Entry entry) /// Maps specified to the corresponding URL. /// /// is null. - public bool TryGetUrl( + public bool TryGetUri( string path, #if NETCOREAPP [NotNullWhen(true)] #endif - out string? url) + out string? uri) { if (path == null) { @@ -197,32 +197,32 @@ public bool TryGetUrl( if (path.IndexOf('*') >= 0) { - url = null; + uri = null; return false; } // Note: the mapping function is case-insensitive. - foreach (var (file, mappedUrl) in _entries) + foreach (var (file, mappedUri) in _entries) { if (file.IsPrefix) { if (path.StartsWith(file.Path, StringComparison.OrdinalIgnoreCase)) { var escapedPath = string.Join("/", path.Substring(file.Path.Length).Split(new[] { '/', '\\' }).Select(Uri.EscapeDataString)); - url = mappedUrl.Prefix + escapedPath + mappedUrl.Suffix; + uri = mappedUri.Prefix + escapedPath + mappedUri.Suffix; return true; } } else if (string.Equals(path, file.Path, StringComparison.OrdinalIgnoreCase)) { - Debug.Assert(mappedUrl.Suffix.Length == 0); - url = mappedUrl.Prefix; + Debug.Assert(mappedUri.Suffix.Length == 0); + uri = mappedUri.Prefix; return true; } } - url = null; + uri = null; return false; } } From 2909054971c449957ac9c9b1621246b012b42b2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Matou=C5=A1ek?= Date: Tue, 27 Oct 2020 15:57:54 -0700 Subject: [PATCH 106/161] Update dependencies (#666) --- eng/Versions.props | 19 ++++++++----------- .../Microsoft.Build.Tasks.Tfvc.csproj | 2 +- ...icrosoft.SourceLink.AzureRepos.Tfvc.csproj | 2 +- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/eng/Versions.props b/eng/Versions.props index ec61a773..5af13c12 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -5,21 +5,18 @@ 1.1.0 beta true + true true + 2.4.1 - - 15.7.179 - 15.7.179 - 15.112.1 - 2.1.0 - 4.9.2 + 16.7.0 + 16.7.0 + 16.153.0 + 3.1.6 + 5.7.0 + 5.7.0 4.5.0 4.7.2 diff --git a/src/Microsoft.Build.Tasks.Tfvc/Microsoft.Build.Tasks.Tfvc.csproj b/src/Microsoft.Build.Tasks.Tfvc/Microsoft.Build.Tasks.Tfvc.csproj index 567d9f56..2c6b3458 100644 --- a/src/Microsoft.Build.Tasks.Tfvc/Microsoft.Build.Tasks.Tfvc.csproj +++ b/src/Microsoft.Build.Tasks.Tfvc/Microsoft.Build.Tasks.Tfvc.csproj @@ -1,6 +1,6 @@  - net46 + net472 true true true diff --git a/src/SourceLink.AzureRepos.Tfvc/Microsoft.SourceLink.AzureRepos.Tfvc.csproj b/src/SourceLink.AzureRepos.Tfvc/Microsoft.SourceLink.AzureRepos.Tfvc.csproj index 4c2d6cc8..91af51fe 100644 --- a/src/SourceLink.AzureRepos.Tfvc/Microsoft.SourceLink.AzureRepos.Tfvc.csproj +++ b/src/SourceLink.AzureRepos.Tfvc/Microsoft.SourceLink.AzureRepos.Tfvc.csproj @@ -1,6 +1,6 @@  - net46 + net472 true true true From 295d6afc63f85a02dca13c1fbcfd4f92520ce710 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 2 Nov 2020 13:27:17 +0000 Subject: [PATCH 107/161] Update dependencies from https://github.com/dotnet/arcade build 20201029.1 (#667) [master] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 +- eng/Versions.props | 2 - eng/common/build.ps1 | 2 - eng/common/build.sh | 9 +- eng/common/cross/build-rootfs.sh | 7 + eng/common/cross/s390x/sources.list.bionic | 11 + eng/common/cross/toolchain.cmake | 7 +- .../dotnet-install-scripts/dotnet-install.ps1 | 774 ----------- .../dotnet-install-scripts/dotnet-install.sh | 1133 ----------------- eng/common/performance/crossgen_perf.proj | 24 + eng/common/post-build/symbols-validation.ps1 | 76 +- eng/common/templates/job/job.yml | 4 +- eng/common/tools.ps1 | 63 +- eng/common/tools.sh | 37 +- global.json | 2 +- 15 files changed, 143 insertions(+), 2012 deletions(-) create mode 100644 eng/common/cross/s390x/sources.list.bionic delete mode 100644 eng/common/dotnet-install-scripts/dotnet-install.ps1 delete mode 100644 eng/common/dotnet-install-scripts/dotnet-install.sh diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 58c89f15..f89d3ac7 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - c262e0121edb7df890528bb61589d4b6e440a1f6 + 19e7e769f7ca2ece42221f7ff951e7ec705498ec https://github.com/dotnet/source-build-reference-packages diff --git a/eng/Versions.props b/eng/Versions.props index 5af13c12..4ea51a89 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -5,11 +5,9 @@ 1.1.0 beta true - true true - 2.4.1 16.7.0 16.7.0 diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 1fd7f686..94a91c08 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -7,7 +7,6 @@ Param( [string] $msbuildEngine = $null, [bool] $warnAsError = $true, [bool] $nodeReuse = $true, - [bool] $useDefaultDotnetInstall = $false, [switch][Alias('r')]$restore, [switch] $deployDeps, [switch][Alias('b')]$build, @@ -66,7 +65,6 @@ function Print-Usage() { Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build" Write-Host " -warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." - Write-Host " -useDefaultDotnetInstall Use dotnet-install.* scripts from public location as opposed to from eng common folder" Write-Host "" Write-Host "Command line arguments not listed above are passed thru to msbuild." diff --git a/eng/common/build.sh b/eng/common/build.sh index 19849adb..252b6360 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -36,8 +36,6 @@ usage() echo " --prepareMachine Prepare machine for CI run, clean up processes after build" echo " --nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')" echo " --warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" - echo " --useDefaultDotnetInstall Use dotnet-install.* scripts from public location as opposed to from eng common folder" - echo "" echo "Command line arguments not listed above are passed thru to msbuild." echo "Arguments can also be passed in with a single hyphen." @@ -80,7 +78,6 @@ prepare_machine=false verbosity='minimal' runtime_source_feed='' runtime_source_feed_key='' -use_default_dotnet_install=false properties='' while [[ $# > 0 ]]; do @@ -159,14 +156,10 @@ while [[ $# > 0 ]]; do runtime_source_feed=$2 shift ;; - -runtimesourcefeedkey) + -runtimesourcefeedkey) runtime_source_feed_key=$2 shift ;; - -usedefaultdotnetinstall) - use_default_dotnet_install=$2 - shift - ;; *) properties="$properties $1" ;; diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index a841c64f..575eee9f 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -106,6 +106,13 @@ while :; do __UbuntuRepo="http://ftp.debian.org/debian/" __CodeName=jessie ;; + s390x) + __BuildArch=s390x + __UbuntuArch=s390x + __UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/" + __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libunwind8-dev//') + unset __LLDB_Package + ;; x86) __BuildArch=x86 __UbuntuArch=i386 diff --git a/eng/common/cross/s390x/sources.list.bionic b/eng/common/cross/s390x/sources.list.bionic new file mode 100644 index 00000000..21095574 --- /dev/null +++ b/eng/common/cross/s390x/sources.list.bionic @@ -0,0 +1,11 @@ +deb http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted +deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted + +deb http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse +deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index 8b437d8a..fc11001a 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -36,6 +36,9 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm64") if("$ENV{__DistroRid}" MATCHES "tizen.*") set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu/9.2.0") endif() +elseif(TARGET_ARCH_NAME STREQUAL "s390x") + set(CMAKE_SYSTEM_PROCESSOR s390x) + set(TOOLCHAIN "s390x-linux-gnu") elseif(TARGET_ARCH_NAME STREQUAL "x86") set(CMAKE_SYSTEM_PROCESSOR i686) set(TOOLCHAIN "i686-linux-gnu") @@ -46,7 +49,7 @@ elseif (ILLUMOS) set(CMAKE_SYSTEM_PROCESSOR "x86_64") set(TOOLCHAIN "x86_64-illumos") else() - message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only armel, arm, arm64 and x86 are supported!") + message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only armel, arm, arm64, s390x and x86 are supported!") endif() if(DEFINED ENV{TOOLCHAIN}) @@ -171,7 +174,7 @@ endif() # Specify compile options -if((TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64)$" AND NOT "$ENV{__DistroRid}" MATCHES "android.*") OR ILLUMOS) +if((TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64|s390x)$" AND NOT "$ENV{__DistroRid}" MATCHES "android.*") OR ILLUMOS) set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN}) diff --git a/eng/common/dotnet-install-scripts/dotnet-install.ps1 b/eng/common/dotnet-install-scripts/dotnet-install.ps1 deleted file mode 100644 index f63b533f..00000000 --- a/eng/common/dotnet-install-scripts/dotnet-install.ps1 +++ /dev/null @@ -1,774 +0,0 @@ -# -# Copyright (c) .NET Foundation and contributors. All rights reserved. -# Licensed under the MIT license. See LICENSE file in the project root for full license information. -# - -# Copied from https://dot.net/v1/dotnet-install.ps1 on 8/26/2020 - -<# -.SYNOPSIS - Installs dotnet cli -.DESCRIPTION - Installs dotnet cli. If dotnet installation already exists in the given directory - it will update it only if the requested version differs from the one already installed. -.PARAMETER Channel - Default: LTS - Download from the Channel specified. Possible values: - - Current - most current release - - LTS - most current supported release - - 2-part version in a format A.B - represents a specific release - examples: 2.0, 1.0 - - Branch name - examples: release/2.0.0, Master - Note: The version parameter overrides the channel parameter. -.PARAMETER Version - Default: latest - Represents a build version on specific channel. Possible values: - - latest - most latest build on specific channel - - coherent - most latest coherent build on specific channel - coherent applies only to SDK downloads - - 3-part version in a format A.B.C - represents specific version of build - examples: 2.0.0-preview2-006120, 1.1.0 -.PARAMETER InstallDir - Default: %LocalAppData%\Microsoft\dotnet - Path to where to install dotnet. Note that binaries will be placed directly in a given directory. -.PARAMETER Architecture - Default: - this value represents currently running OS architecture - Architecture of dotnet binaries to be installed. - Possible values are: , amd64, x64, x86, arm64, arm -.PARAMETER SharedRuntime - This parameter is obsolete and may be removed in a future version of this script. - The recommended alternative is '-Runtime dotnet'. - Installs just the shared runtime bits, not the entire SDK. -.PARAMETER Runtime - Installs just a shared runtime, not the entire SDK. - Possible values: - - dotnet - the Microsoft.NETCore.App shared runtime - - aspnetcore - the Microsoft.AspNetCore.App shared runtime - - windowsdesktop - the Microsoft.WindowsDesktop.App shared runtime -.PARAMETER DryRun - If set it will not perform installation but instead display what command line to use to consistently install - currently requested version of dotnet cli. In example if you specify version 'latest' it will display a link - with specific version so that this command can be used deterministicly in a build script. - It also displays binaries location if you prefer to install or download it yourself. -.PARAMETER NoPath - By default this script will set environment variable PATH for the current process to the binaries folder inside installation folder. - If set it will display binaries location but not set any environment variable. -.PARAMETER Verbose - Displays diagnostics information. -.PARAMETER AzureFeed - Default: https://dotnetcli.azureedge.net/dotnet - This parameter typically is not changed by the user. - It allows changing the URL for the Azure feed used by this installer. -.PARAMETER UncachedFeed - This parameter typically is not changed by the user. - It allows changing the URL for the Uncached feed used by this installer. -.PARAMETER FeedCredential - Used as a query string to append to the Azure feed. - It allows changing the URL to use non-public blob storage accounts. -.PARAMETER ProxyAddress - If set, the installer will use the proxy when making web requests -.PARAMETER ProxyUseDefaultCredentials - Default: false - Use default credentials, when using proxy address. -.PARAMETER ProxyBypassList - If set with ProxyAddress, will provide the list of comma separated urls that will bypass the proxy -.PARAMETER SkipNonVersionedFiles - Default: false - Skips installing non-versioned files if they already exist, such as dotnet.exe. -.PARAMETER NoCdn - Disable downloading from the Azure CDN, and use the uncached feed directly. -.PARAMETER JSonFile - Determines the SDK version from a user specified global.json file - Note: global.json must have a value for 'SDK:Version' -#> -[cmdletbinding()] -param( - [string]$Channel="LTS", - [string]$Version="Latest", - [string]$JSonFile, - [string]$InstallDir="", - [string]$Architecture="", - [ValidateSet("dotnet", "aspnetcore", "windowsdesktop", IgnoreCase = $false)] - [string]$Runtime, - [Obsolete("This parameter may be removed in a future version of this script. The recommended alternative is '-Runtime dotnet'.")] - [switch]$SharedRuntime, - [switch]$DryRun, - [switch]$NoPath, - [string]$AzureFeed="https://dotnetcli.azureedge.net/dotnet", - [string]$UncachedFeed="https://dotnetcli.blob.core.windows.net/dotnet", - [string]$FeedCredential, - [string]$ProxyAddress, - [switch]$ProxyUseDefaultCredentials, - [string[]]$ProxyBypassList=@(), - [switch]$SkipNonVersionedFiles, - [switch]$NoCdn -) - -Set-StrictMode -Version Latest -$ErrorActionPreference="Stop" -$ProgressPreference="SilentlyContinue" - -if ($NoCdn) { - $AzureFeed = $UncachedFeed -} - -$BinFolderRelativePath="" - -if ($SharedRuntime -and (-not $Runtime)) { - $Runtime = "dotnet" -} - -# example path with regex: shared/1.0.0-beta-12345/somepath -$VersionRegEx="/\d+\.\d+[^/]+/" -$OverrideNonVersionedFiles = !$SkipNonVersionedFiles - -function Say($str) { - try - { - Write-Host "dotnet-install: $str" - } - catch - { - # Some platforms cannot utilize Write-Host (Azure Functions, for instance). Fall back to Write-Output - Write-Output "dotnet-install: $str" - } -} - -function Say-Verbose($str) { - try - { - Write-Verbose "dotnet-install: $str" - } - catch - { - # Some platforms cannot utilize Write-Verbose (Azure Functions, for instance). Fall back to Write-Output - Write-Output "dotnet-install: $str" - } -} - -function Say-Invocation($Invocation) { - $command = $Invocation.MyCommand; - $args = (($Invocation.BoundParameters.Keys | foreach { "-$_ `"$($Invocation.BoundParameters[$_])`"" }) -join " ") - Say-Verbose "$command $args" -} - -function Invoke-With-Retry([ScriptBlock]$ScriptBlock, [int]$MaxAttempts = 3, [int]$SecondsBetweenAttempts = 1) { - $Attempts = 0 - - while ($true) { - try { - return $ScriptBlock.Invoke() - } - catch { - $Attempts++ - if ($Attempts -lt $MaxAttempts) { - Start-Sleep $SecondsBetweenAttempts - } - else { - throw - } - } - } -} - -function Get-Machine-Architecture() { - Say-Invocation $MyInvocation - - # On PS x86, PROCESSOR_ARCHITECTURE reports x86 even on x64 systems. - # To get the correct architecture, we need to use PROCESSOR_ARCHITEW6432. - # PS x64 doesn't define this, so we fall back to PROCESSOR_ARCHITECTURE. - # Possible values: amd64, x64, x86, arm64, arm - - if( $ENV:PROCESSOR_ARCHITEW6432 -ne $null ) - { - return $ENV:PROCESSOR_ARCHITEW6432 - } - - return $ENV:PROCESSOR_ARCHITECTURE -} - -function Get-CLIArchitecture-From-Architecture([string]$Architecture) { - Say-Invocation $MyInvocation - - switch ($Architecture.ToLower()) { - { $_ -eq "" } { return Get-CLIArchitecture-From-Architecture $(Get-Machine-Architecture) } - { ($_ -eq "amd64") -or ($_ -eq "x64") } { return "x64" } - { $_ -eq "x86" } { return "x86" } - { $_ -eq "arm" } { return "arm" } - { $_ -eq "arm64" } { return "arm64" } - default { throw "Architecture not supported. If you think this is a bug, report it at https://github.com/dotnet/sdk/issues" } - } -} - -# The version text returned from the feeds is a 1-line or 2-line string: -# For the SDK and the dotnet runtime (2 lines): -# Line 1: # commit_hash -# Line 2: # 4-part version -# For the aspnetcore runtime (1 line): -# Line 1: # 4-part version -function Get-Version-Info-From-Version-Text([string]$VersionText) { - Say-Invocation $MyInvocation - - $Data = -split $VersionText - - $VersionInfo = @{ - CommitHash = $(if ($Data.Count -gt 1) { $Data[0] }) - Version = $Data[-1] # last line is always the version number. - } - return $VersionInfo -} - -function Load-Assembly([string] $Assembly) { - try { - Add-Type -Assembly $Assembly | Out-Null - } - catch { - # On Nano Server, Powershell Core Edition is used. Add-Type is unable to resolve base class assemblies because they are not GAC'd. - # Loading the base class assemblies is not unnecessary as the types will automatically get resolved. - } -} - -function GetHTTPResponse([Uri] $Uri) -{ - Invoke-With-Retry( - { - - $HttpClient = $null - - try { - # HttpClient is used vs Invoke-WebRequest in order to support Nano Server which doesn't support the Invoke-WebRequest cmdlet. - Load-Assembly -Assembly System.Net.Http - - if(-not $ProxyAddress) { - try { - # Despite no proxy being explicitly specified, we may still be behind a default proxy - $DefaultProxy = [System.Net.WebRequest]::DefaultWebProxy; - if($DefaultProxy -and (-not $DefaultProxy.IsBypassed($Uri))) { - $ProxyAddress = $DefaultProxy.GetProxy($Uri).OriginalString - $ProxyUseDefaultCredentials = $true - } - } catch { - # Eat the exception and move forward as the above code is an attempt - # at resolving the DefaultProxy that may not have been a problem. - $ProxyAddress = $null - Say-Verbose("Exception ignored: $_.Exception.Message - moving forward...") - } - } - - if($ProxyAddress) { - $HttpClientHandler = New-Object System.Net.Http.HttpClientHandler - $HttpClientHandler.Proxy = New-Object System.Net.WebProxy -Property @{ - Address=$ProxyAddress; - UseDefaultCredentials=$ProxyUseDefaultCredentials; - BypassList = $ProxyBypassList; - } - $HttpClient = New-Object System.Net.Http.HttpClient -ArgumentList $HttpClientHandler - } - else { - - $HttpClient = New-Object System.Net.Http.HttpClient - } - # Default timeout for HttpClient is 100s. For a 50 MB download this assumes 500 KB/s average, any less will time out - # 20 minutes allows it to work over much slower connections. - $HttpClient.Timeout = New-TimeSpan -Minutes 20 - $Response = $HttpClient.GetAsync("${Uri}${FeedCredential}").Result - if (($Response -eq $null) -or (-not ($Response.IsSuccessStatusCode))) { - # The feed credential is potentially sensitive info. Do not log FeedCredential to console output. - $ErrorMsg = "Failed to download $Uri." - if ($Response -ne $null) { - $ErrorMsg += " $Response" - } - - throw $ErrorMsg - } - - return $Response - } - finally { - if ($HttpClient -ne $null) { - $HttpClient.Dispose() - } - } - }) -} - -function Get-Latest-Version-Info([string]$AzureFeed, [string]$Channel, [bool]$Coherent) { - Say-Invocation $MyInvocation - - $VersionFileUrl = $null - if ($Runtime -eq "dotnet") { - $VersionFileUrl = "$UncachedFeed/Runtime/$Channel/latest.version" - } - elseif ($Runtime -eq "aspnetcore") { - $VersionFileUrl = "$UncachedFeed/aspnetcore/Runtime/$Channel/latest.version" - } - # Currently, the WindowsDesktop runtime is manufactured with the .Net core runtime - elseif ($Runtime -eq "windowsdesktop") { - $VersionFileUrl = "$UncachedFeed/Runtime/$Channel/latest.version" - } - elseif (-not $Runtime) { - if ($Coherent) { - $VersionFileUrl = "$UncachedFeed/Sdk/$Channel/latest.coherent.version" - } - else { - $VersionFileUrl = "$UncachedFeed/Sdk/$Channel/latest.version" - } - } - else { - throw "Invalid value for `$Runtime" - } - try { - $Response = GetHTTPResponse -Uri $VersionFileUrl - } - catch { - throw "Could not resolve version information." - } - $StringContent = $Response.Content.ReadAsStringAsync().Result - - switch ($Response.Content.Headers.ContentType) { - { ($_ -eq "application/octet-stream") } { $VersionText = $StringContent } - { ($_ -eq "text/plain") } { $VersionText = $StringContent } - { ($_ -eq "text/plain; charset=UTF-8") } { $VersionText = $StringContent } - default { throw "``$Response.Content.Headers.ContentType`` is an unknown .version file content type." } - } - - $VersionInfo = Get-Version-Info-From-Version-Text $VersionText - - return $VersionInfo -} - -function Parse-Jsonfile-For-Version([string]$JSonFile) { - Say-Invocation $MyInvocation - - If (-Not (Test-Path $JSonFile)) { - throw "Unable to find '$JSonFile'" - } - try { - $JSonContent = Get-Content($JSonFile) -Raw | ConvertFrom-Json | Select-Object -expand "sdk" -ErrorAction SilentlyContinue - } - catch { - throw "Json file unreadable: '$JSonFile'" - } - if ($JSonContent) { - try { - $JSonContent.PSObject.Properties | ForEach-Object { - $PropertyName = $_.Name - if ($PropertyName -eq "version") { - $Version = $_.Value - Say-Verbose "Version = $Version" - } - } - } - catch { - throw "Unable to parse the SDK node in '$JSonFile'" - } - } - else { - throw "Unable to find the SDK node in '$JSonFile'" - } - If ($Version -eq $null) { - throw "Unable to find the SDK:version node in '$JSonFile'" - } - return $Version -} - -function Get-Specific-Version-From-Version([string]$AzureFeed, [string]$Channel, [string]$Version, [string]$JSonFile) { - Say-Invocation $MyInvocation - - if (-not $JSonFile) { - switch ($Version.ToLower()) { - { $_ -eq "latest" } { - $LatestVersionInfo = Get-Latest-Version-Info -AzureFeed $AzureFeed -Channel $Channel -Coherent $False - return $LatestVersionInfo.Version - } - { $_ -eq "coherent" } { - $LatestVersionInfo = Get-Latest-Version-Info -AzureFeed $AzureFeed -Channel $Channel -Coherent $True - return $LatestVersionInfo.Version - } - default { return $Version } - } - } - else { - return Parse-Jsonfile-For-Version $JSonFile - } -} - -function Get-Download-Link([string]$AzureFeed, [string]$SpecificVersion, [string]$CLIArchitecture) { - Say-Invocation $MyInvocation - - # If anything fails in this lookup it will default to $SpecificVersion - $SpecificProductVersion = Get-Product-Version -AzureFeed $AzureFeed -SpecificVersion $SpecificVersion - - if ($Runtime -eq "dotnet") { - $PayloadURL = "$AzureFeed/Runtime/$SpecificVersion/dotnet-runtime-$SpecificProductVersion-win-$CLIArchitecture.zip" - } - elseif ($Runtime -eq "aspnetcore") { - $PayloadURL = "$AzureFeed/aspnetcore/Runtime/$SpecificVersion/aspnetcore-runtime-$SpecificProductVersion-win-$CLIArchitecture.zip" - } - elseif ($Runtime -eq "windowsdesktop") { - $PayloadURL = "$AzureFeed/Runtime/$SpecificVersion/windowsdesktop-runtime-$SpecificProductVersion-win-$CLIArchitecture.zip" - } - elseif (-not $Runtime) { - $PayloadURL = "$AzureFeed/Sdk/$SpecificVersion/dotnet-sdk-$SpecificProductVersion-win-$CLIArchitecture.zip" - } - else { - throw "Invalid value for `$Runtime" - } - - Say-Verbose "Constructed primary named payload URL: $PayloadURL" - - return $PayloadURL, $SpecificProductVersion -} - -function Get-LegacyDownload-Link([string]$AzureFeed, [string]$SpecificVersion, [string]$CLIArchitecture) { - Say-Invocation $MyInvocation - - if (-not $Runtime) { - $PayloadURL = "$AzureFeed/Sdk/$SpecificVersion/dotnet-dev-win-$CLIArchitecture.$SpecificVersion.zip" - } - elseif ($Runtime -eq "dotnet") { - $PayloadURL = "$AzureFeed/Runtime/$SpecificVersion/dotnet-win-$CLIArchitecture.$SpecificVersion.zip" - } - else { - return $null - } - - Say-Verbose "Constructed legacy named payload URL: $PayloadURL" - - return $PayloadURL -} - -function Get-Product-Version([string]$AzureFeed, [string]$SpecificVersion) { - Say-Invocation $MyInvocation - - if ($Runtime -eq "dotnet") { - $ProductVersionTxtURL = "$AzureFeed/Runtime/$SpecificVersion/productVersion.txt" - } - elseif ($Runtime -eq "aspnetcore") { - $ProductVersionTxtURL = "$AzureFeed/aspnetcore/Runtime/$SpecificVersion/productVersion.txt" - } - elseif ($Runtime -eq "windowsdesktop") { - $ProductVersionTxtURL = "$AzureFeed/Runtime/$SpecificVersion/productVersion.txt" - } - elseif (-not $Runtime) { - $ProductVersionTxtURL = "$AzureFeed/Sdk/$SpecificVersion/productVersion.txt" - } - else { - throw "Invalid value specified for `$Runtime" - } - - Say-Verbose "Checking for existence of $ProductVersionTxtURL" - - try { - $productVersionResponse = GetHTTPResponse($productVersionTxtUrl) - - if ($productVersionResponse.StatusCode -eq 200) { - $productVersion = $productVersionResponse.Content.ReadAsStringAsync().Result.Trim() - if ($productVersion -ne $SpecificVersion) - { - Say "Using alternate version $productVersion found in $ProductVersionTxtURL" - } - - return $productVersion - } - else { - Say-Verbose "Got StatusCode $($productVersionResponse.StatusCode) trying to get productVersion.txt at $productVersionTxtUrl, so using default value of $SpecificVersion" - $productVersion = $SpecificVersion - } - } catch { - Say-Verbose "Could not read productVersion.txt at $productVersionTxtUrl, so using default value of $SpecificVersion" - $productVersion = $SpecificVersion - } - - return $productVersion -} - -function Get-User-Share-Path() { - Say-Invocation $MyInvocation - - $InstallRoot = $env:DOTNET_INSTALL_DIR - if (!$InstallRoot) { - $InstallRoot = "$env:LocalAppData\Microsoft\dotnet" - } - return $InstallRoot -} - -function Resolve-Installation-Path([string]$InstallDir) { - Say-Invocation $MyInvocation - - if ($InstallDir -eq "") { - return Get-User-Share-Path - } - return $InstallDir -} - -function Is-Dotnet-Package-Installed([string]$InstallRoot, [string]$RelativePathToPackage, [string]$SpecificVersion) { - Say-Invocation $MyInvocation - - $DotnetPackagePath = Join-Path -Path $InstallRoot -ChildPath $RelativePathToPackage | Join-Path -ChildPath $SpecificVersion - Say-Verbose "Is-Dotnet-Package-Installed: DotnetPackagePath=$DotnetPackagePath" - return Test-Path $DotnetPackagePath -PathType Container -} - -function Get-Absolute-Path([string]$RelativeOrAbsolutePath) { - # Too much spam - # Say-Invocation $MyInvocation - - return $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($RelativeOrAbsolutePath) -} - -function Get-Path-Prefix-With-Version($path) { - $match = [regex]::match($path, $VersionRegEx) - if ($match.Success) { - return $entry.FullName.Substring(0, $match.Index + $match.Length) - } - - return $null -} - -function Get-List-Of-Directories-And-Versions-To-Unpack-From-Dotnet-Package([System.IO.Compression.ZipArchive]$Zip, [string]$OutPath) { - Say-Invocation $MyInvocation - - $ret = @() - foreach ($entry in $Zip.Entries) { - $dir = Get-Path-Prefix-With-Version $entry.FullName - if ($dir -ne $null) { - $path = Get-Absolute-Path $(Join-Path -Path $OutPath -ChildPath $dir) - if (-Not (Test-Path $path -PathType Container)) { - $ret += $dir - } - } - } - - $ret = $ret | Sort-Object | Get-Unique - - $values = ($ret | foreach { "$_" }) -join ";" - Say-Verbose "Directories to unpack: $values" - - return $ret -} - -# Example zip content and extraction algorithm: -# Rule: files if extracted are always being extracted to the same relative path locally -# .\ -# a.exe # file does not exist locally, extract -# b.dll # file exists locally, override only if $OverrideFiles set -# aaa\ # same rules as for files -# ... -# abc\1.0.0\ # directory contains version and exists locally -# ... # do not extract content under versioned part -# abc\asd\ # same rules as for files -# ... -# def\ghi\1.0.1\ # directory contains version and does not exist locally -# ... # extract content -function Extract-Dotnet-Package([string]$ZipPath, [string]$OutPath) { - Say-Invocation $MyInvocation - - Load-Assembly -Assembly System.IO.Compression.FileSystem - Set-Variable -Name Zip - try { - $Zip = [System.IO.Compression.ZipFile]::OpenRead($ZipPath) - - $DirectoriesToUnpack = Get-List-Of-Directories-And-Versions-To-Unpack-From-Dotnet-Package -Zip $Zip -OutPath $OutPath - - foreach ($entry in $Zip.Entries) { - $PathWithVersion = Get-Path-Prefix-With-Version $entry.FullName - if (($PathWithVersion -eq $null) -Or ($DirectoriesToUnpack -contains $PathWithVersion)) { - $DestinationPath = Get-Absolute-Path $(Join-Path -Path $OutPath -ChildPath $entry.FullName) - $DestinationDir = Split-Path -Parent $DestinationPath - $OverrideFiles=$OverrideNonVersionedFiles -Or (-Not (Test-Path $DestinationPath)) - if ((-Not $DestinationPath.EndsWith("\")) -And $OverrideFiles) { - New-Item -ItemType Directory -Force -Path $DestinationDir | Out-Null - [System.IO.Compression.ZipFileExtensions]::ExtractToFile($entry, $DestinationPath, $OverrideNonVersionedFiles) - } - } - } - } - finally { - if ($Zip -ne $null) { - $Zip.Dispose() - } - } -} - -function DownloadFile($Source, [string]$OutPath) { - if ($Source -notlike "http*") { - # Using System.IO.Path.GetFullPath to get the current directory - # does not work in this context - $pwd gives the current directory - if (![System.IO.Path]::IsPathRooted($Source)) { - $Source = $(Join-Path -Path $pwd -ChildPath $Source) - } - $Source = Get-Absolute-Path $Source - Say "Copying file from $Source to $OutPath" - Copy-Item $Source $OutPath - return - } - - $Stream = $null - - try { - $Response = GetHTTPResponse -Uri $Source - $Stream = $Response.Content.ReadAsStreamAsync().Result - $File = [System.IO.File]::Create($OutPath) - $Stream.CopyTo($File) - $File.Close() - } - finally { - if ($Stream -ne $null) { - $Stream.Dispose() - } - } -} - -function Prepend-Sdk-InstallRoot-To-Path([string]$InstallRoot, [string]$BinFolderRelativePath) { - $BinPath = Get-Absolute-Path $(Join-Path -Path $InstallRoot -ChildPath $BinFolderRelativePath) - if (-Not $NoPath) { - $SuffixedBinPath = "$BinPath;" - if (-Not $env:path.Contains($SuffixedBinPath)) { - Say "Adding to current process PATH: `"$BinPath`". Note: This change will not be visible if PowerShell was run as a child process." - $env:path = $SuffixedBinPath + $env:path - } else { - Say-Verbose "Current process PATH already contains `"$BinPath`"" - } - } - else { - Say "Binaries of dotnet can be found in $BinPath" - } -} - -$CLIArchitecture = Get-CLIArchitecture-From-Architecture $Architecture -$SpecificVersion = Get-Specific-Version-From-Version -AzureFeed $AzureFeed -Channel $Channel -Version $Version -JSonFile $JSonFile -$DownloadLink, $EffectiveVersion = Get-Download-Link -AzureFeed $AzureFeed -SpecificVersion $SpecificVersion -CLIArchitecture $CLIArchitecture -$LegacyDownloadLink = Get-LegacyDownload-Link -AzureFeed $AzureFeed -SpecificVersion $SpecificVersion -CLIArchitecture $CLIArchitecture - -$InstallRoot = Resolve-Installation-Path $InstallDir -Say-Verbose "InstallRoot: $InstallRoot" -$ScriptName = $MyInvocation.MyCommand.Name - -if ($DryRun) { - Say "Payload URLs:" - Say "Primary named payload URL: $DownloadLink" - if ($LegacyDownloadLink) { - Say "Legacy named payload URL: $LegacyDownloadLink" - } - $RepeatableCommand = ".\$ScriptName -Version `"$SpecificVersion`" -InstallDir `"$InstallRoot`" -Architecture `"$CLIArchitecture`"" - if ($Runtime -eq "dotnet") { - $RepeatableCommand+=" -Runtime `"dotnet`"" - } - elseif ($Runtime -eq "aspnetcore") { - $RepeatableCommand+=" -Runtime `"aspnetcore`"" - } - foreach ($key in $MyInvocation.BoundParameters.Keys) { - if (-not (@("Architecture","Channel","DryRun","InstallDir","Runtime","SharedRuntime","Version") -contains $key)) { - $RepeatableCommand+=" -$key `"$($MyInvocation.BoundParameters[$key])`"" - } - } - Say "Repeatable invocation: $RepeatableCommand" - exit 0 -} - -if ($Runtime -eq "dotnet") { - $assetName = ".NET Core Runtime" - $dotnetPackageRelativePath = "shared\Microsoft.NETCore.App" -} -elseif ($Runtime -eq "aspnetcore") { - $assetName = "ASP.NET Core Runtime" - $dotnetPackageRelativePath = "shared\Microsoft.AspNetCore.App" -} -elseif ($Runtime -eq "windowsdesktop") { - $assetName = ".NET Core Windows Desktop Runtime" - $dotnetPackageRelativePath = "shared\Microsoft.WindowsDesktop.App" -} -elseif (-not $Runtime) { - $assetName = ".NET Core SDK" - $dotnetPackageRelativePath = "sdk" -} -else { - throw "Invalid value for `$Runtime" -} - -if ($SpecificVersion -ne $EffectiveVersion) -{ - Say "Performing installation checks for effective version: $EffectiveVersion" - $SpecificVersion = $EffectiveVersion -} - -# Check if the SDK version is already installed. -$isAssetInstalled = Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $SpecificVersion -if ($isAssetInstalled) { - Say "$assetName version $SpecificVersion is already installed." - Prepend-Sdk-InstallRoot-To-Path -InstallRoot $InstallRoot -BinFolderRelativePath $BinFolderRelativePath - exit 0 -} - -New-Item -ItemType Directory -Force -Path $InstallRoot | Out-Null - -$installDrive = $((Get-Item $InstallRoot).PSDrive.Name); -$diskInfo = Get-PSDrive -Name $installDrive -if ($diskInfo.Free / 1MB -le 100) { - Say "There is not enough disk space on drive ${installDrive}:" - exit 0 -} - -$ZipPath = [System.IO.Path]::combine([System.IO.Path]::GetTempPath(), [System.IO.Path]::GetRandomFileName()) -Say-Verbose "Zip path: $ZipPath" - -$DownloadFailed = $false -Say "Downloading link: $DownloadLink" -try { - DownloadFile -Source $DownloadLink -OutPath $ZipPath -} -catch { - Say "Cannot download: $DownloadLink" - if ($LegacyDownloadLink) { - $DownloadLink = $LegacyDownloadLink - $ZipPath = [System.IO.Path]::combine([System.IO.Path]::GetTempPath(), [System.IO.Path]::GetRandomFileName()) - Say-Verbose "Legacy zip path: $ZipPath" - Say "Downloading legacy link: $DownloadLink" - try { - DownloadFile -Source $DownloadLink -OutPath $ZipPath - } - catch { - Say "Cannot download: $DownloadLink" - $DownloadFailed = $true - } - } - else { - $DownloadFailed = $true - } -} - -if ($DownloadFailed) { - throw "Could not find/download: `"$assetName`" with version = $SpecificVersion`nRefer to: https://aka.ms/dotnet-os-lifecycle for information on .NET Core support" -} - -Say "Extracting zip from $DownloadLink" -Extract-Dotnet-Package -ZipPath $ZipPath -OutPath $InstallRoot - -# Check if the SDK version is installed; if not, fail the installation. -$isAssetInstalled = $false - -# if the version contains "RTM" or "servicing"; check if a 'release-type' SDK version is installed. -if ($SpecificVersion -Match "rtm" -or $SpecificVersion -Match "servicing") { - $ReleaseVersion = $SpecificVersion.Split("-")[0] - Say-Verbose "Checking installation: version = $ReleaseVersion" - $isAssetInstalled = Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $ReleaseVersion -} - -# Check if the SDK version is installed. -if (!$isAssetInstalled) { - Say-Verbose "Checking installation: version = $SpecificVersion" - $isAssetInstalled = Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $SpecificVersion -} - -if (!$isAssetInstalled) { - throw "`"$assetName`" with version = $SpecificVersion failed to install with an unknown error." -} - -Remove-Item $ZipPath - -Prepend-Sdk-InstallRoot-To-Path -InstallRoot $InstallRoot -BinFolderRelativePath $BinFolderRelativePath - -Say "Installation finished" -exit 0 \ No newline at end of file diff --git a/eng/common/dotnet-install-scripts/dotnet-install.sh b/eng/common/dotnet-install-scripts/dotnet-install.sh deleted file mode 100644 index 92161141..00000000 --- a/eng/common/dotnet-install-scripts/dotnet-install.sh +++ /dev/null @@ -1,1133 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) .NET Foundation and contributors. All rights reserved. -# Licensed under the MIT license. See LICENSE file in the project root for full license information. -# - -# Stop script on NZEC -set -e -# Stop script if unbound variable found (use ${var:-} if intentional) -set -u -# By default cmd1 | cmd2 returns exit code of cmd2 regardless of cmd1 success -# This is causing it to fail -set -o pipefail - -# Use in the the functions: eval $invocation -invocation='say_verbose "Calling: ${yellow:-}${FUNCNAME[0]} ${green:-}$*${normal:-}"' - -# standard output may be used as a return value in the functions -# we need a way to write text on the screen in the functions so that -# it won't interfere with the return value. -# Exposing stream 3 as a pipe to standard output of the script itself -exec 3>&1 - -# Setup some colors to use. These need to work in fairly limited shells, like the Ubuntu Docker container where there are only 8 colors. -# See if stdout is a terminal -if [ -t 1 ] && command -v tput > /dev/null; then - # see if it supports colors - ncolors=$(tput colors) - if [ -n "$ncolors" ] && [ $ncolors -ge 8 ]; then - bold="$(tput bold || echo)" - normal="$(tput sgr0 || echo)" - black="$(tput setaf 0 || echo)" - red="$(tput setaf 1 || echo)" - green="$(tput setaf 2 || echo)" - yellow="$(tput setaf 3 || echo)" - blue="$(tput setaf 4 || echo)" - magenta="$(tput setaf 5 || echo)" - cyan="$(tput setaf 6 || echo)" - white="$(tput setaf 7 || echo)" - fi -fi - -say_warning() { - printf "%b\n" "${yellow:-}dotnet_install: Warning: $1${normal:-}" -} - -say_err() { - printf "%b\n" "${red:-}dotnet_install: Error: $1${normal:-}" >&2 -} - -say() { - # using stream 3 (defined in the beginning) to not interfere with stdout of functions - # which may be used as return value - printf "%b\n" "${cyan:-}dotnet-install:${normal:-} $1" >&3 -} - -say_verbose() { - if [ "$verbose" = true ]; then - say "$1" - fi -} - -# This platform list is finite - if the SDK/Runtime has supported Linux distribution-specific assets, -# then and only then should the Linux distribution appear in this list. -# Adding a Linux distribution to this list does not imply distribution-specific support. -get_legacy_os_name_from_platform() { - eval $invocation - - platform="$1" - case "$platform" in - "centos.7") - echo "centos" - return 0 - ;; - "debian.8") - echo "debian" - return 0 - ;; - "debian.9") - echo "debian.9" - return 0 - ;; - "fedora.23") - echo "fedora.23" - return 0 - ;; - "fedora.24") - echo "fedora.24" - return 0 - ;; - "fedora.27") - echo "fedora.27" - return 0 - ;; - "fedora.28") - echo "fedora.28" - return 0 - ;; - "opensuse.13.2") - echo "opensuse.13.2" - return 0 - ;; - "opensuse.42.1") - echo "opensuse.42.1" - return 0 - ;; - "opensuse.42.3") - echo "opensuse.42.3" - return 0 - ;; - "rhel.7"*) - echo "rhel" - return 0 - ;; - "ubuntu.14.04") - echo "ubuntu" - return 0 - ;; - "ubuntu.16.04") - echo "ubuntu.16.04" - return 0 - ;; - "ubuntu.16.10") - echo "ubuntu.16.10" - return 0 - ;; - "ubuntu.18.04") - echo "ubuntu.18.04" - return 0 - ;; - "alpine.3.4.3") - echo "alpine" - return 0 - ;; - esac - return 1 -} - -get_linux_platform_name() { - eval $invocation - - if [ -n "$runtime_id" ]; then - echo "${runtime_id%-*}" - return 0 - else - if [ -e /etc/os-release ]; then - . /etc/os-release - echo "$ID${VERSION_ID:+.${VERSION_ID}}" - return 0 - elif [ -e /etc/redhat-release ]; then - local redhatRelease=$(&1 || true) | grep -q musl -} - -get_current_os_name() { - eval $invocation - - local uname=$(uname) - if [ "$uname" = "Darwin" ]; then - echo "osx" - return 0 - elif [ "$uname" = "FreeBSD" ]; then - echo "freebsd" - return 0 - elif [ "$uname" = "Linux" ]; then - local linux_platform_name - linux_platform_name="$(get_linux_platform_name)" || { echo "linux" && return 0 ; } - - if [ "$linux_platform_name" = "rhel.6" ]; then - echo $linux_platform_name - return 0 - elif is_musl_based_distro; then - echo "linux-musl" - return 0 - else - echo "linux" - return 0 - fi - fi - - say_err "OS name could not be detected: UName = $uname" - return 1 -} - -get_legacy_os_name() { - eval $invocation - - local uname=$(uname) - if [ "$uname" = "Darwin" ]; then - echo "osx" - return 0 - elif [ -n "$runtime_id" ]; then - echo $(get_legacy_os_name_from_platform "${runtime_id%-*}" || echo "${runtime_id%-*}") - return 0 - else - if [ -e /etc/os-release ]; then - . /etc/os-release - os=$(get_legacy_os_name_from_platform "$ID${VERSION_ID:+.${VERSION_ID}}" || echo "") - if [ -n "$os" ]; then - echo "$os" - return 0 - fi - fi - fi - - say_verbose "Distribution specific OS name and version could not be detected: UName = $uname" - return 1 -} - -machine_has() { - eval $invocation - - hash "$1" > /dev/null 2>&1 - return $? -} - - -check_min_reqs() { - local hasMinimum=false - if machine_has "curl"; then - hasMinimum=true - elif machine_has "wget"; then - hasMinimum=true - fi - - if [ "$hasMinimum" = "false" ]; then - say_err "curl (recommended) or wget are required to download dotnet. Install missing prerequisite to proceed." - return 1 - fi - return 0 -} - -check_pre_reqs() { - eval $invocation - - if [ "${DOTNET_INSTALL_SKIP_PREREQS:-}" = "1" ]; then - return 0 - fi - - if [ "$(uname)" = "Linux" ]; then - if is_musl_based_distro; then - if ! command -v scanelf > /dev/null; then - say_warning "scanelf not found, please install pax-utils package." - return 0 - fi - LDCONFIG_COMMAND="scanelf --ldpath -BF '%f'" - [ -z "$($LDCONFIG_COMMAND 2>/dev/null | grep libintl)" ] && say_warning "Unable to locate libintl. Probable prerequisite missing; install libintl (or gettext)." - else - if [ ! -x "$(command -v ldconfig)" ]; then - say_verbose "ldconfig is not in PATH, trying /sbin/ldconfig." - LDCONFIG_COMMAND="/sbin/ldconfig" - else - LDCONFIG_COMMAND="ldconfig" - fi - local librarypath=${LD_LIBRARY_PATH:-} - LDCONFIG_COMMAND="$LDCONFIG_COMMAND -NXv ${librarypath//:/ }" - fi - - [ -z "$($LDCONFIG_COMMAND 2>/dev/null | grep zlib)" ] && say_warning "Unable to locate zlib. Probable prerequisite missing; install zlib." - [ -z "$($LDCONFIG_COMMAND 2>/dev/null | grep ssl)" ] && say_warning "Unable to locate libssl. Probable prerequisite missing; install libssl." - [ -z "$($LDCONFIG_COMMAND 2>/dev/null | grep libicu)" ] && say_warning "Unable to locate libicu. Probable prerequisite missing; install libicu." - [ -z "$($LDCONFIG_COMMAND 2>/dev/null | grep lttng)" ] && say_warning "Unable to locate liblttng. Probable prerequisite missing; install libcurl." - [ -z "$($LDCONFIG_COMMAND 2>/dev/null | grep libcurl)" ] && say_warning "Unable to locate libcurl. Probable prerequisite missing; install libcurl." - fi - - return 0 -} - -# args: -# input - $1 -to_lowercase() { - #eval $invocation - - echo "$1" | tr '[:upper:]' '[:lower:]' - return 0 -} - -# args: -# input - $1 -remove_trailing_slash() { - #eval $invocation - - local input="${1:-}" - echo "${input%/}" - return 0 -} - -# args: -# input - $1 -remove_beginning_slash() { - #eval $invocation - - local input="${1:-}" - echo "${input#/}" - return 0 -} - -# args: -# root_path - $1 -# child_path - $2 - this parameter can be empty -combine_paths() { - eval $invocation - - # TODO: Consider making it work with any number of paths. For now: - if [ ! -z "${3:-}" ]; then - say_err "combine_paths: Function takes two parameters." - return 1 - fi - - local root_path="$(remove_trailing_slash "$1")" - local child_path="$(remove_beginning_slash "${2:-}")" - say_verbose "combine_paths: root_path=$root_path" - say_verbose "combine_paths: child_path=$child_path" - echo "$root_path/$child_path" - return 0 -} - -get_machine_architecture() { - eval $invocation - - if command -v uname > /dev/null; then - CPUName=$(uname -m) - case $CPUName in - armv7l) - echo "arm" - return 0 - ;; - aarch64) - echo "arm64" - return 0 - ;; - esac - fi - - # Always default to 'x64' - echo "x64" - return 0 -} - -# args: -# architecture - $1 -get_normalized_architecture_from_architecture() { - eval $invocation - - local architecture="$(to_lowercase "$1")" - case "$architecture" in - \) - echo "$(get_normalized_architecture_from_architecture "$(get_machine_architecture)")" - return 0 - ;; - amd64|x64) - echo "x64" - return 0 - ;; - arm) - echo "arm" - return 0 - ;; - arm64) - echo "arm64" - return 0 - ;; - esac - - say_err "Architecture \`$architecture\` not supported. If you think this is a bug, report it at https://github.com/dotnet/sdk/issues" - return 1 -} - -# The version text returned from the feeds is a 1-line or 2-line string: -# For the SDK and the dotnet runtime (2 lines): -# Line 1: # commit_hash -# Line 2: # 4-part version -# For the aspnetcore runtime (1 line): -# Line 1: # 4-part version - -# args: -# version_text - stdin -get_version_from_version_info() { - eval $invocation - - cat | tail -n 1 | sed 's/\r$//' - return 0 -} - -# args: -# install_root - $1 -# relative_path_to_package - $2 -# specific_version - $3 -is_dotnet_package_installed() { - eval $invocation - - local install_root="$1" - local relative_path_to_package="$2" - local specific_version="${3//[$'\t\r\n']}" - - local dotnet_package_path="$(combine_paths "$(combine_paths "$install_root" "$relative_path_to_package")" "$specific_version")" - say_verbose "is_dotnet_package_installed: dotnet_package_path=$dotnet_package_path" - - if [ -d "$dotnet_package_path" ]; then - return 0 - else - return 1 - fi -} - -# args: -# azure_feed - $1 -# channel - $2 -# normalized_architecture - $3 -# coherent - $4 -get_latest_version_info() { - eval $invocation - - local azure_feed="$1" - local channel="$2" - local normalized_architecture="$3" - local coherent="$4" - - local version_file_url=null - if [[ "$runtime" == "dotnet" ]]; then - version_file_url="$uncached_feed/Runtime/$channel/latest.version" - elif [[ "$runtime" == "aspnetcore" ]]; then - version_file_url="$uncached_feed/aspnetcore/Runtime/$channel/latest.version" - elif [ -z "$runtime" ]; then - if [ "$coherent" = true ]; then - version_file_url="$uncached_feed/Sdk/$channel/latest.coherent.version" - else - version_file_url="$uncached_feed/Sdk/$channel/latest.version" - fi - else - say_err "Invalid value for \$runtime" - return 1 - fi - say_verbose "get_latest_version_info: latest url: $version_file_url" - - download "$version_file_url" - return $? -} - -# args: -# json_file - $1 -parse_jsonfile_for_version() { - eval $invocation - - local json_file="$1" - if [ ! -f "$json_file" ]; then - say_err "Unable to find \`$json_file\`" - return 1 - fi - - sdk_section=$(cat $json_file | awk '/"sdk"/,/}/') - if [ -z "$sdk_section" ]; then - say_err "Unable to parse the SDK node in \`$json_file\`" - return 1 - fi - - sdk_list=$(echo $sdk_section | awk -F"[{}]" '{print $2}') - sdk_list=${sdk_list//[\" ]/} - sdk_list=${sdk_list//,/$'\n'} - sdk_list="$(echo -e "${sdk_list}" | tr -d '[[:space:]]')" - - local version_info="" - while read -r line; do - IFS=: - while read -r key value; do - if [[ "$key" == "version" ]]; then - version_info=$value - fi - done <<< "$line" - done <<< "$sdk_list" - if [ -z "$version_info" ]; then - say_err "Unable to find the SDK:version node in \`$json_file\`" - return 1 - fi - - unset IFS; - echo "$version_info" - return 0 -} - -# args: -# azure_feed - $1 -# channel - $2 -# normalized_architecture - $3 -# version - $4 -# json_file - $5 -get_specific_version_from_version() { - eval $invocation - - local azure_feed="$1" - local channel="$2" - local normalized_architecture="$3" - local version="$(to_lowercase "$4")" - local json_file="$5" - - if [ -z "$json_file" ]; then - case "$version" in - latest) - local version_info - version_info="$(get_latest_version_info "$azure_feed" "$channel" "$normalized_architecture" false)" || return 1 - say_verbose "get_specific_version_from_version: version_info=$version_info" - echo "$version_info" | get_version_from_version_info - return 0 - ;; - coherent) - local version_info - version_info="$(get_latest_version_info "$azure_feed" "$channel" "$normalized_architecture" true)" || return 1 - say_verbose "get_specific_version_from_version: version_info=$version_info" - echo "$version_info" | get_version_from_version_info - return 0 - ;; - *) - echo "$version" - return 0 - ;; - esac - else - local version_info - version_info="$(parse_jsonfile_for_version "$json_file")" || return 1 - echo "$version_info" - return 0 - fi -} - -# args: -# azure_feed - $1 -# channel - $2 -# normalized_architecture - $3 -# specific_version - $4 -construct_download_link() { - eval $invocation - - local azure_feed="$1" - local channel="$2" - local normalized_architecture="$3" - local specific_version="${4//[$'\t\r\n']}" - local specific_product_version="$(get_specific_product_version "$1" "$4")" - - local osname - osname="$(get_current_os_name)" || return 1 - - local download_link=null - if [[ "$runtime" == "dotnet" ]]; then - download_link="$azure_feed/Runtime/$specific_version/dotnet-runtime-$specific_product_version-$osname-$normalized_architecture.tar.gz" - elif [[ "$runtime" == "aspnetcore" ]]; then - download_link="$azure_feed/aspnetcore/Runtime/$specific_version/aspnetcore-runtime-$specific_product_version-$osname-$normalized_architecture.tar.gz" - elif [ -z "$runtime" ]; then - download_link="$azure_feed/Sdk/$specific_version/dotnet-sdk-$specific_product_version-$osname-$normalized_architecture.tar.gz" - else - return 1 - fi - - echo "$download_link" - return 0 -} - -# args: -# azure_feed - $1 -# specific_version - $2 -get_specific_product_version() { - # If we find a 'productVersion.txt' at the root of any folder, we'll use its contents - # to resolve the version of what's in the folder, superseding the specified version. - eval $invocation - - local azure_feed="$1" - local specific_version="${2//[$'\t\r\n']}" - local specific_product_version=$specific_version - - local download_link=null - if [[ "$runtime" == "dotnet" ]]; then - download_link="$azure_feed/Runtime/$specific_version/productVersion.txt${feed_credential}" - elif [[ "$runtime" == "aspnetcore" ]]; then - download_link="$azure_feed/aspnetcore/Runtime/$specific_version/productVersion.txt${feed_credential}" - elif [ -z "$runtime" ]; then - download_link="$azure_feed/Sdk/$specific_version/productVersion.txt${feed_credential}" - else - return 1 - fi - - specific_product_version=$(curl -s --fail "$download_link") - if [ $? -ne 0 ] - then - specific_product_version=$(wget -qO- "$download_link") - if [ $? -ne 0 ] - then - specific_product_version=$specific_version - fi - fi - specific_product_version="${specific_product_version//[$'\t\r\n']}" - - echo "$specific_product_version" - return 0 -} - -# args: -# azure_feed - $1 -# channel - $2 -# normalized_architecture - $3 -# specific_version - $4 -construct_legacy_download_link() { - eval $invocation - - local azure_feed="$1" - local channel="$2" - local normalized_architecture="$3" - local specific_version="${4//[$'\t\r\n']}" - - local distro_specific_osname - distro_specific_osname="$(get_legacy_os_name)" || return 1 - - local legacy_download_link=null - if [[ "$runtime" == "dotnet" ]]; then - legacy_download_link="$azure_feed/Runtime/$specific_version/dotnet-$distro_specific_osname-$normalized_architecture.$specific_version.tar.gz" - elif [ -z "$runtime" ]; then - legacy_download_link="$azure_feed/Sdk/$specific_version/dotnet-dev-$distro_specific_osname-$normalized_architecture.$specific_version.tar.gz" - else - return 1 - fi - - echo "$legacy_download_link" - return 0 -} - -get_user_install_path() { - eval $invocation - - if [ ! -z "${DOTNET_INSTALL_DIR:-}" ]; then - echo "$DOTNET_INSTALL_DIR" - else - echo "$HOME/.dotnet" - fi - return 0 -} - -# args: -# install_dir - $1 -resolve_installation_path() { - eval $invocation - - local install_dir=$1 - if [ "$install_dir" = "" ]; then - local user_install_path="$(get_user_install_path)" - say_verbose "resolve_installation_path: user_install_path=$user_install_path" - echo "$user_install_path" - return 0 - fi - - echo "$install_dir" - return 0 -} - -# args: -# relative_or_absolute_path - $1 -get_absolute_path() { - eval $invocation - - local relative_or_absolute_path=$1 - echo "$(cd "$(dirname "$1")" && pwd -P)/$(basename "$1")" - return 0 -} - -# args: -# input_files - stdin -# root_path - $1 -# out_path - $2 -# override - $3 -copy_files_or_dirs_from_list() { - eval $invocation - - local root_path="$(remove_trailing_slash "$1")" - local out_path="$(remove_trailing_slash "$2")" - local override="$3" - local osname="$(get_current_os_name)" - local override_switch=$( - if [ "$override" = false ]; then - if [ "$osname" = "linux-musl" ]; then - printf -- "-u"; - else - printf -- "-n"; - fi - fi) - - cat | uniq | while read -r file_path; do - local path="$(remove_beginning_slash "${file_path#$root_path}")" - local target="$out_path/$path" - if [ "$override" = true ] || (! ([ -d "$target" ] || [ -e "$target" ])); then - mkdir -p "$out_path/$(dirname "$path")" - if [ -d "$target" ]; then - rm -rf "$target" - fi - cp -R $override_switch "$root_path/$path" "$target" - fi - done -} - -# args: -# zip_path - $1 -# out_path - $2 -extract_dotnet_package() { - eval $invocation - - local zip_path="$1" - local out_path="$2" - - local temp_out_path="$(mktemp -d "$temporary_file_template")" - - local failed=false - tar -xzf "$zip_path" -C "$temp_out_path" > /dev/null || failed=true - - local folders_with_version_regex='^.*/[0-9]+\.[0-9]+[^/]+/' - find "$temp_out_path" -type f | grep -Eo "$folders_with_version_regex" | sort | copy_files_or_dirs_from_list "$temp_out_path" "$out_path" false - find "$temp_out_path" -type f | grep -Ev "$folders_with_version_regex" | copy_files_or_dirs_from_list "$temp_out_path" "$out_path" "$override_non_versioned_files" - - rm -rf "$temp_out_path" - - if [ "$failed" = true ]; then - say_err "Extraction failed" - return 1 - fi -} - -# args: -# remote_path - $1 -# [out_path] - $2 - stdout if not provided -download() { - eval $invocation - - local remote_path="$1" - local out_path="${2:-}" - - if [[ "$remote_path" != "http"* ]]; then - cp "$remote_path" "$out_path" - return $? - fi - - local failed=false - if machine_has "curl"; then - downloadcurl "$remote_path" "$out_path" || failed=true - elif machine_has "wget"; then - downloadwget "$remote_path" "$out_path" || failed=true - else - failed=true - fi - if [ "$failed" = true ]; then - say_verbose "Download failed: $remote_path" - return 1 - fi - return 0 -} - -downloadcurl() { - eval $invocation - local remote_path="$1" - local out_path="${2:-}" - - # Append feed_credential as late as possible before calling curl to avoid logging feed_credential - remote_path="${remote_path}${feed_credential}" - - local curl_options="--retry 20 --retry-delay 2 --connect-timeout 15 -sSL -f --create-dirs " - local failed=false - if [ -z "$out_path" ]; then - curl $curl_options "$remote_path" || failed=true - else - curl $curl_options -o "$out_path" "$remote_path" || failed=true - fi - if [ "$failed" = true ]; then - say_verbose "Curl download failed" - return 1 - fi - return 0 -} - -downloadwget() { - eval $invocation - local remote_path="$1" - local out_path="${2:-}" - - # Append feed_credential as late as possible before calling wget to avoid logging feed_credential - remote_path="${remote_path}${feed_credential}" - local wget_options="--tries 20 --waitretry 2 --connect-timeout 15 " - local failed=false - if [ -z "$out_path" ]; then - wget -q $wget_options -O - "$remote_path" || failed=true - else - wget $wget_options -O "$out_path" "$remote_path" || failed=true - fi - if [ "$failed" = true ]; then - say_verbose "Wget download failed" - return 1 - fi - return 0 -} - -calculate_vars() { - eval $invocation - valid_legacy_download_link=true - - normalized_architecture="$(get_normalized_architecture_from_architecture "$architecture")" - say_verbose "normalized_architecture=$normalized_architecture" - - specific_version="$(get_specific_version_from_version "$azure_feed" "$channel" "$normalized_architecture" "$version" "$json_file")" - specific_product_version="$(get_specific_product_version "$azure_feed" "$specific_version")" - say_verbose "specific_version=$specific_version" - if [ -z "$specific_version" ]; then - say_err "Could not resolve version information." - return 1 - fi - - download_link="$(construct_download_link "$azure_feed" "$channel" "$normalized_architecture" "$specific_version")" - say_verbose "Constructed primary named payload URL: $download_link" - - legacy_download_link="$(construct_legacy_download_link "$azure_feed" "$channel" "$normalized_architecture" "$specific_version")" || valid_legacy_download_link=false - - if [ "$valid_legacy_download_link" = true ]; then - say_verbose "Constructed legacy named payload URL: $legacy_download_link" - else - say_verbose "Cound not construct a legacy_download_link; omitting..." - fi - - install_root="$(resolve_installation_path "$install_dir")" - say_verbose "InstallRoot: $install_root" -} - -install_dotnet() { - eval $invocation - local download_failed=false - local asset_name='' - local asset_relative_path='' - - if [[ "$runtime" == "dotnet" ]]; then - asset_relative_path="shared/Microsoft.NETCore.App" - asset_name=".NET Core Runtime" - elif [[ "$runtime" == "aspnetcore" ]]; then - asset_relative_path="shared/Microsoft.AspNetCore.App" - asset_name="ASP.NET Core Runtime" - elif [ -z "$runtime" ]; then - asset_relative_path="sdk" - asset_name=".NET Core SDK" - else - say_err "Invalid value for \$runtime" - return 1 - fi - - # Check if the SDK version is already installed. - if is_dotnet_package_installed "$install_root" "$asset_relative_path" "$specific_version"; then - say "$asset_name version $specific_version is already installed." - return 0 - fi - - mkdir -p "$install_root" - zip_path="$(mktemp "$temporary_file_template")" - say_verbose "Zip path: $zip_path" - - say "Downloading link: $download_link" - - # Failures are normal in the non-legacy case for ultimately legacy downloads. - # Do not output to stderr, since output to stderr is considered an error. - download "$download_link" "$zip_path" 2>&1 || download_failed=true - - # if the download fails, download the legacy_download_link - if [ "$download_failed" = true ]; then - say "Cannot download: $download_link" - - if [ "$valid_legacy_download_link" = true ]; then - download_failed=false - download_link="$legacy_download_link" - zip_path="$(mktemp "$temporary_file_template")" - say_verbose "Legacy zip path: $zip_path" - say "Downloading legacy link: $download_link" - download "$download_link" "$zip_path" 2>&1 || download_failed=true - - if [ "$download_failed" = true ]; then - say "Cannot download: $download_link" - fi - fi - fi - - if [ "$download_failed" = true ]; then - say_err "Could not find/download: \`$asset_name\` with version = $specific_version" - say_err "Refer to: https://aka.ms/dotnet-os-lifecycle for information on .NET Core support" - return 1 - fi - - say "Extracting zip from $download_link" - extract_dotnet_package "$zip_path" "$install_root" - - # Check if the SDK version is installed; if not, fail the installation. - # if the version contains "RTM" or "servicing"; check if a 'release-type' SDK version is installed. - if [[ $specific_version == *"rtm"* || $specific_version == *"servicing"* ]]; then - IFS='-' - read -ra verArr <<< "$specific_version" - release_version="${verArr[0]}" - unset IFS; - say_verbose "Checking installation: version = $release_version" - if is_dotnet_package_installed "$install_root" "$asset_relative_path" "$release_version"; then - return 0 - fi - fi - - # Check if the standard SDK version is installed. - say_verbose "Checking installation: version = $specific_product_version" - if is_dotnet_package_installed "$install_root" "$asset_relative_path" "$specific_product_version"; then - return 0 - fi - - say_err "\`$asset_name\` with version = $specific_product_version failed to install with an unknown error." - return 1 -} - -args=("$@") - -local_version_file_relative_path="/.version" -bin_folder_relative_path="" -temporary_file_template="${TMPDIR:-/tmp}/dotnet.XXXXXXXXX" - -channel="LTS" -version="Latest" -json_file="" -install_dir="" -architecture="" -dry_run=false -no_path=false -no_cdn=false -azure_feed="https://dotnetcli.azureedge.net/dotnet" -uncached_feed="https://dotnetcli.blob.core.windows.net/dotnet" -feed_credential="" -verbose=false -runtime="" -runtime_id="" -override_non_versioned_files=true -non_dynamic_parameters="" - -while [ $# -ne 0 ] -do - name="$1" - case "$name" in - -c|--channel|-[Cc]hannel) - shift - channel="$1" - ;; - -v|--version|-[Vv]ersion) - shift - version="$1" - ;; - -i|--install-dir|-[Ii]nstall[Dd]ir) - shift - install_dir="$1" - ;; - --arch|--architecture|-[Aa]rch|-[Aa]rchitecture) - shift - architecture="$1" - ;; - --shared-runtime|-[Ss]hared[Rr]untime) - say_warning "The --shared-runtime flag is obsolete and may be removed in a future version of this script. The recommended usage is to specify '--runtime dotnet'." - if [ -z "$runtime" ]; then - runtime="dotnet" - fi - ;; - --runtime|-[Rr]untime) - shift - runtime="$1" - if [[ "$runtime" != "dotnet" ]] && [[ "$runtime" != "aspnetcore" ]]; then - say_err "Unsupported value for --runtime: '$1'. Valid values are 'dotnet' and 'aspnetcore'." - if [[ "$runtime" == "windowsdesktop" ]]; then - say_err "WindowsDesktop archives are manufactured for Windows platforms only." - fi - exit 1 - fi - ;; - --dry-run|-[Dd]ry[Rr]un) - dry_run=true - ;; - --no-path|-[Nn]o[Pp]ath) - no_path=true - non_dynamic_parameters+=" $name" - ;; - --verbose|-[Vv]erbose) - verbose=true - non_dynamic_parameters+=" $name" - ;; - --no-cdn|-[Nn]o[Cc]dn) - no_cdn=true - non_dynamic_parameters+=" $name" - ;; - --azure-feed|-[Aa]zure[Ff]eed) - shift - azure_feed="$1" - non_dynamic_parameters+=" $name "\""$1"\""" - ;; - --uncached-feed|-[Uu]ncached[Ff]eed) - shift - uncached_feed="$1" - non_dynamic_parameters+=" $name "\""$1"\""" - ;; - --feed-credential|-[Ff]eed[Cc]redential) - shift - feed_credential="$1" - non_dynamic_parameters+=" $name "\""$1"\""" - ;; - --runtime-id|-[Rr]untime[Ii]d) - shift - runtime_id="$1" - non_dynamic_parameters+=" $name "\""$1"\""" - ;; - --jsonfile|-[Jj][Ss]on[Ff]ile) - shift - json_file="$1" - ;; - --skip-non-versioned-files|-[Ss]kip[Nn]on[Vv]ersioned[Ff]iles) - override_non_versioned_files=false - non_dynamic_parameters+=" $name" - ;; - -?|--?|-h|--help|-[Hh]elp) - script_name="$(basename "$0")" - echo ".NET Tools Installer" - echo "Usage: $script_name [-c|--channel ] [-v|--version ] [-p|--prefix ]" - echo " $script_name -h|-?|--help" - echo "" - echo "$script_name is a simple command line interface for obtaining dotnet cli." - echo "" - echo "Options:" - echo " -c,--channel Download from the channel specified, Defaults to \`$channel\`." - echo " -Channel" - echo " Possible values:" - echo " - Current - most current release" - echo " - LTS - most current supported release" - echo " - 2-part version in a format A.B - represents a specific release" - echo " examples: 2.0; 1.0" - echo " - Branch name" - echo " examples: release/2.0.0; Master" - echo " Note: The version parameter overrides the channel parameter." - echo " -v,--version Use specific VERSION, Defaults to \`$version\`." - echo " -Version" - echo " Possible values:" - echo " - latest - most latest build on specific channel" - echo " - coherent - most latest coherent build on specific channel" - echo " coherent applies only to SDK downloads" - echo " - 3-part version in a format A.B.C - represents specific version of build" - echo " examples: 2.0.0-preview2-006120; 1.1.0" - echo " -i,--install-dir Install under specified location (see Install Location below)" - echo " -InstallDir" - echo " --architecture Architecture of dotnet binaries to be installed, Defaults to \`$architecture\`." - echo " --arch,-Architecture,-Arch" - echo " Possible values: x64, arm, and arm64" - echo " --runtime Installs a shared runtime only, without the SDK." - echo " -Runtime" - echo " Possible values:" - echo " - dotnet - the Microsoft.NETCore.App shared runtime" - echo " - aspnetcore - the Microsoft.AspNetCore.App shared runtime" - echo " --dry-run,-DryRun Do not perform installation. Display download link." - echo " --no-path, -NoPath Do not set PATH for the current process." - echo " --verbose,-Verbose Display diagnostics information." - echo " --azure-feed,-AzureFeed Azure feed location. Defaults to $azure_feed, This parameter typically is not changed by the user." - echo " --uncached-feed,-UncachedFeed Uncached feed location. This parameter typically is not changed by the user." - echo " --feed-credential,-FeedCredential Azure feed shared access token. This parameter typically is not specified." - echo " --skip-non-versioned-files Skips non-versioned files if they already exist, such as the dotnet executable." - echo " -SkipNonVersionedFiles" - echo " --no-cdn,-NoCdn Disable downloading from the Azure CDN, and use the uncached feed directly." - echo " --jsonfile Determines the SDK version from a user specified global.json file." - echo " Note: global.json must have a value for 'SDK:Version'" - echo " --runtime-id Installs the .NET Tools for the given platform (use linux-x64 for portable linux)." - echo " -RuntimeId" - echo " -?,--?,-h,--help,-Help Shows this help message" - echo "" - echo "Obsolete parameters:" - echo " --shared-runtime The recommended alternative is '--runtime dotnet'." - echo " This parameter is obsolete and may be removed in a future version of this script." - echo " Installs just the shared runtime bits, not the entire SDK." - echo "" - echo "Install Location:" - echo " Location is chosen in following order:" - echo " - --install-dir option" - echo " - Environmental variable DOTNET_INSTALL_DIR" - echo " - $HOME/.dotnet" - exit 0 - ;; - *) - say_err "Unknown argument \`$name\`" - exit 1 - ;; - esac - - shift -done - -if [ "$no_cdn" = true ]; then - azure_feed="$uncached_feed" -fi - -check_min_reqs -calculate_vars -script_name=$(basename "$0") - -if [ "$dry_run" = true ]; then - say "Payload URLs:" - say "Primary named payload URL: $download_link" - if [ "$valid_legacy_download_link" = true ]; then - say "Legacy named payload URL: $legacy_download_link" - fi - repeatable_command="./$script_name --version "\""$specific_version"\"" --install-dir "\""$install_root"\"" --architecture "\""$normalized_architecture"\""" - if [[ "$runtime" == "dotnet" ]]; then - repeatable_command+=" --runtime "\""dotnet"\""" - elif [[ "$runtime" == "aspnetcore" ]]; then - repeatable_command+=" --runtime "\""aspnetcore"\""" - fi - repeatable_command+="$non_dynamic_parameters" - say "Repeatable invocation: $repeatable_command" - exit 0 -fi - -check_pre_reqs -install_dotnet - -bin_path="$(get_absolute_path "$(combine_paths "$install_root" "$bin_folder_relative_path")")" -if [ "$no_path" = false ]; then - say "Adding to current process PATH: \`$bin_path\`. Note: This change will be visible only when sourcing script." - export PATH="$bin_path":"$PATH" -else - say "Binaries of dotnet can be found in $bin_path" -fi - -say "Installation finished successfully." diff --git a/eng/common/performance/crossgen_perf.proj b/eng/common/performance/crossgen_perf.proj index 3b8dfc2b..79a54865 100644 --- a/eng/common/performance/crossgen_perf.proj +++ b/eng/common/performance/crossgen_perf.proj @@ -65,6 +65,24 @@ + + + $(WorkItemDirectory) + $(Python) pre.py crossgen --core-root $(CoreRoot) --single %(Identity) + $(Python) test.py sod --scenario-name "Crossgen %(Identity) Size" --dirs ./crossgen/ + $(Python) post.py + + + + + + $(WorkItemDirectory) + $(Python) $(Crossgen2Directory)pre.py crossgen2 --core-root $(CoreRoot) --single %(Identity) + $(Python) test.py sod --scenario-name "Crossgen2 %(Identity) Size" --dirs ./crossgen/ + $(Python) post.py + + + @@ -82,5 +100,11 @@ $(Python) $(Crossgen2Directory)test.py crossgen2 --core-root $(CoreRoot) --composite $(Crossgen2Directory)framework-r2r.dll.rsp 1:00 + + 4:00 + + + 4:00 + \ No newline at end of file diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 index fcc6019b..99bf28cd 100644 --- a/eng/common/post-build/symbols-validation.ps1 +++ b/eng/common/post-build/symbols-validation.ps1 @@ -9,9 +9,16 @@ param( # Maximum number of jobs to run in parallel $MaxParallelJobs = 6 +# Max number of retries +$MaxRetry = 5 + # Wait time between check for system load $SecondsBetweenLoadChecks = 10 +# Set error codes +Set-Variable -Name "ERROR_BADEXTRACT" -Option Constant -Value -1 +Set-Variable -Name "ERROR_FILEDOESNOTEXIST" -Option Constant -Value -2 + $CountMissingSymbols = { param( [string] $PackagePath # Path to a NuGet package @@ -21,10 +28,15 @@ $CountMissingSymbols = { Add-Type -AssemblyName System.IO.Compression.FileSystem + Write-Host "Validating $PackagePath " + # Ensure input file exist if (!(Test-Path $PackagePath)) { Write-PipelineTaskError "Input file does not exist: $PackagePath" - return -2 + return [pscustomobject]@{ + result = $using:ERROR_FILEDOESNOTEXIST + packagePath = $PackagePath + } } # Extensions for which we'll look for symbols @@ -45,7 +57,7 @@ $CountMissingSymbols = { Write-Host "Something went wrong extracting $PackagePath" Write-Host $_ return [pscustomobject]@{ - result = -1 + result = $using:ERROR_BADEXTRACT packagePath = $PackagePath } } @@ -91,26 +103,37 @@ $CountMissingSymbols = { $dotnetSymbolExe = "$env:USERPROFILE\.dotnet\tools" $dotnetSymbolExe = Resolve-Path "$dotnetSymbolExe\dotnet-symbol.exe" - & $dotnetSymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null + $totalRetries = 0 - if (Test-Path $PdbPath) { - return 'PDB' - } - elseif (Test-Path $NGenPdb) { - return 'NGen PDB' - } - elseif (Test-Path $SODbg) { - return 'DBG for SO' - } - elseif (Test-Path $DylibDwarf) { - return 'Dwarf for Dylib' - } - elseif (Test-Path $SymbolPath) { - return 'Module' - } - else { - return $null + while ($totalRetries -lt $using:MaxRetry) { + # Save the output and get diagnostic output + $output = & $dotnetSymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath --diagnostics | Out-String + + if (Test-Path $PdbPath) { + return 'PDB' + } + elseif (Test-Path $NGenPdb) { + return 'NGen PDB' + } + elseif (Test-Path $SODbg) { + return 'DBG for SO' + } + elseif (Test-Path $DylibDwarf) { + return 'Dwarf for Dylib' + } + elseif (Test-Path $SymbolPath) { + return 'Module' + } + elseif ($output.Contains("503 Service Unavailable")) { + # If we got a 503 error, we should retry. + $totalRetries++ + } + else { + return $null + } } + + return $null } $SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault $FileName '--microsoft-symbol-server' $SymbolsPath @@ -155,14 +178,21 @@ function CheckJobResult( $packagePath, [ref]$DupedSymbols, [ref]$TotalFailures) { - if ($result -eq '-1') { + if ($result -eq $ERROR_BADEXTRACT) { Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$packagePath has duplicated symbol files" $DupedSymbols.Value++ } - elseif ($jobResult.result -ne '0') { + elseif ($result -eq $ERROR_FILEDOESNOTEXIST) { + Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$packagePath does not exist" + $TotalFailures.Value++ + } + elseif ($result -gt '0') { Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $result modules in the package $packagePath" $TotalFailures.Value++ } + else { + Write-Host "All symbols verified for package $packagePath" + } } function CheckSymbolsAvailable { @@ -192,8 +222,6 @@ function CheckSymbolsAvailable { return } - Write-Host "Validating $FileName " - Start-Job -ScriptBlock $CountMissingSymbols -ArgumentList $FullName | Out-Null $NumJobs = @(Get-Job -State 'Running').Count diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 8e4ce2c2..06048c27 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -203,7 +203,7 @@ jobs: continueOnError: true condition: always() - - ${{ if or(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, 'xunit')) }}: + - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}: - task: PublishTestResults@2 displayName: Publish XUnit Test Results inputs: @@ -214,7 +214,7 @@ jobs: mergeTestResults: ${{ parameters.mergeTestResults }} continueOnError: true condition: always() - - ${{ if or(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, 'vstest')) }}: + - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'vstest')) }}: - task: PublishTestResults@2 displayName: Publish TRX Test Results inputs: diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 9014e062..fc09059f 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -57,15 +57,11 @@ set-strictmode -version 2.0 $ErrorActionPreference = 'Stop' [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 -# If specified, provides an alternate path for getting .NET Core SDKs and Runtimes. This script will still try public sources first. +# If specifies, provides an alternate path for getting .NET Core SDKs and Runtimes. This script will still try public sources first. [string]$runtimeSourceFeed = if (Test-Path variable:runtimeSourceFeed) { $runtimeSourceFeed } else { $null } # Base-64 encoded SAS token that has permission to storage container described by $runtimeSourceFeed [string]$runtimeSourceFeedKey = if (Test-Path variable:runtimeSourceFeedKey) { $runtimeSourceFeedKey } else { $null } -# If false, use copy of dotnet-install from /eng/common/dotnet-install-scripts (for custom behaviors). -# otherwise will fetch from public location. -[bool]$useDefaultDotnetInstall = if (Test-Path variable:useDefaultDotnetInstall) { $useDefaultDotnetInstall } else { $false } - function Create-Directory ([string[]] $path) { New-Item -Path $path -Force -ItemType 'Directory' | Out-Null } @@ -197,46 +193,37 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { function GetDotNetInstallScript([string] $dotnetRoot) { $installScript = Join-Path $dotnetRoot 'dotnet-install.ps1' if (!(Test-Path $installScript)) { - create-directory $dotnetroot - - if ($useDefaultDotnetInstall) - { - $progresspreference = 'silentlycontinue' # don't display the console progress ui - it's a huge perf hit + Create-Directory $dotnetRoot + $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit - $maxretries = 5 - $retries = 1 + $maxRetries = 5 + $retries = 1 - $uri = "https://dot.net/$dotnetinstallscriptversion/dotnet-install.ps1" + $uri = "https://dot.net/$dotnetInstallScriptVersion/dotnet-install.ps1" - while($true) { - try { - write-host "get $uri" - invoke-webrequest $uri -outfile $installscript - break - } - catch { - write-host "failed to download '$uri'" - write-error $_.exception.message -erroraction continue - } + while($true) { + try { + Write-Host "GET $uri" + Invoke-WebRequest $uri -OutFile $installScript + break + } + catch { + Write-Host "Failed to download '$uri'" + Write-Error $_.Exception.Message -ErrorAction Continue + } - if (++$retries -le $maxretries) { - $delayinseconds = [math]::pow(2, $retries) - 1 # exponential backoff - write-host "retrying. waiting for $delayinseconds seconds before next attempt ($retries of $maxretries)." - start-sleep -seconds $delayinseconds - } - else { - throw "unable to download file in $maxretries attempts." - } + if (++$retries -le $maxRetries) { + $delayInSeconds = [math]::Pow(2, $retries) - 1 # Exponential backoff + Write-Host "Retrying. Waiting for $delayInSeconds seconds before next attempt ($retries of $maxRetries)." + Start-Sleep -Seconds $delayInSeconds } - } - else - { - # Use a special version of the script from eng/common that understands the existence of a "productVersion.txt" in a dotnet path. - # See https://github.com/dotnet/arcade/issues/6047 for details - $engCommonCopy = Resolve-Path (Join-Path $PSScriptRoot 'dotnet-install-scripts\dotnet-install.ps1') - Copy-Item $engCommonCopy -Destination $installScript -Force + else { + throw "Unable to download file in $maxRetries attempts." + } + } } + return $installScript } diff --git a/eng/common/tools.sh b/eng/common/tools.sh index b5d63cb1..98186e78 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -68,10 +68,6 @@ fi runtime_source_feed=${runtime_source_feed:-''} runtime_source_feed_key=${runtime_source_feed_key:-''} -# Determines if dotnet-install.sh comes from the eng/common folder or the internet -# (default = public version) -use_default_dotnet_install=${use_default_dotnet_install:-false} - # Resolve any symlinks in the given path. function ResolvePath { local path=$1 @@ -271,30 +267,23 @@ function GetDotNetInstallScript { if [[ ! -a "$install_script" ]]; then mkdir -p "$root" - if [[ "$use_default_dotnet_install" == true ]]; then - echo "Downloading '$install_script_url'" + echo "Downloading '$install_script_url'" - # Use curl if available, otherwise use wget - if command -v curl > /dev/null; then - with_retries curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || { - local exit_code=$? - Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." - ExitWithExitCode $exit_code - } - else - with_retries wget -v -O "$install_script" "$install_script_url" || { - local exit_code=$? - Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." - ExitWithExitCode $exit_code - } - fi + # Use curl if available, otherwise use wget + if command -v curl > /dev/null; then + with_retries curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || { + local exit_code=$? + Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." + ExitWithExitCode $exit_code + } else - # Use a special version of the script from eng/common that understands the existence of a "productVersion.txt" in a dotnet path. - # See https://github.com/dotnet/arcade/issues/6047 for details - cp $repo_root/eng/common/dotnet-install-scripts/dotnet-install.sh $install_script + with_retries wget -v -O "$install_script" "$install_script_url" || { + local exit_code=$? + Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." + ExitWithExitCode $exit_code + } fi fi - # return value _GetDotNetInstallScript="$install_script" } diff --git a/global.json b/global.json index 22258585..82157391 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "5.0.100-rc.2.20479.15" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.20522.2" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.20529.1" } } From 2ee055ab910e774da0c224ee9b78b131dc04d7b6 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 9 Nov 2020 13:21:58 +0000 Subject: [PATCH 108/161] Update dependencies from https://github.com/dotnet/arcade build 20201102.5 (#671) [master] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index f89d3ac7..f951a3dd 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 19e7e769f7ca2ece42221f7ff951e7ec705498ec + 72b28b7e023d4c3fffa0a0b9748a7d4e8cc799be https://github.com/dotnet/source-build-reference-packages diff --git a/global.json b/global.json index 82157391..cf1538a0 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "5.0.100-rc.2.20479.15" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.20529.1" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.20552.5" } } From 13fe3d56a6a996edb9f1fd2b04a05a9dac582d48 Mon Sep 17 00:00:00 2001 From: Steve Ognibene Date: Sat, 14 Nov 2020 21:06:18 -0500 Subject: [PATCH 109/161] fix link which is 404ing --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 199245ff..4b235a94 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Source Link -Source Link is a language- and source-control agnostic system for providing first-class source debugging experiences for binaries. The goal of the project is to enable anyone building [NuGet libraries to provide source debugging](https://github.com/dotnet/designs/blob/master/accepted/diagnostics/debugging-with-symbols-and-sources.md) for their users with almost no effort. Microsoft libraries, such as .NET Core and Roslyn have enabled Source Link. Source Link is supported by Microsoft. +Source Link is a language- and source-control agnostic system for providing first-class source debugging experiences for binaries. The goal of the project is to enable anyone building [NuGet libraries to provide source debugging](https://github.com/dotnet/designs/blob/main/accepted/2020/diagnostics/debugging-with-symbols-and-sources.md) for their users with almost no effort. Microsoft libraries, such as .NET Core and Roslyn have enabled Source Link. Source Link is supported by Microsoft. Source Link is a [set of packages](https://www.nuget.org/packages?q=Microsoft.SourceLink) and a [specification](https://github.com/dotnet/designs/blob/master/accepted/2020/diagnostics/source-link.md#source-link-file-specification) for describing source control metadata that can be embedded in symbols, binaries and packages. From d3394b77de882fa0ef7357bd522fe7323ba503a2 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 16 Nov 2020 13:32:08 +0000 Subject: [PATCH 110/161] Update dependencies from https://github.com/dotnet/arcade build 20201111.5 (#673) [master] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 +- eng/common/performance/crossgen_perf.proj | 10 +-- eng/common/performance/performance-setup.ps1 | 4 +- eng/common/performance/performance-setup.sh | 4 +- eng/common/post-build/publish-using-darc.ps1 | 6 ++ eng/common/templates/job/job.yml | 10 --- .../templates/post-build/post-build.yml | 67 ++----------------- global.json | 4 +- 8 files changed, 25 insertions(+), 84 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index f951a3dd..d8328ad0 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 72b28b7e023d4c3fffa0a0b9748a7d4e8cc799be + e65d307045c183af6e48b087571f6b7a05e39f01 https://github.com/dotnet/source-build-reference-packages diff --git a/eng/common/performance/crossgen_perf.proj b/eng/common/performance/crossgen_perf.proj index 79a54865..cf09e405 100644 --- a/eng/common/performance/crossgen_perf.proj +++ b/eng/common/performance/crossgen_perf.proj @@ -68,9 +68,9 @@ $(WorkItemDirectory) - $(Python) pre.py crossgen --core-root $(CoreRoot) --single %(Identity) - $(Python) test.py sod --scenario-name "Crossgen %(Identity) Size" --dirs ./crossgen/ - $(Python) post.py + $(Python) $(CrossgenDirectory)pre.py crossgen --core-root $(CoreRoot) --single %(Identity) + $(Python) $(CrossgenDirectory)test.py sod --scenario-name "Crossgen %(Identity) Size" --dirs ./crossgen/ + $(Python) $(CrossgenDirectory)post.py @@ -78,8 +78,8 @@ $(WorkItemDirectory) $(Python) $(Crossgen2Directory)pre.py crossgen2 --core-root $(CoreRoot) --single %(Identity) - $(Python) test.py sod --scenario-name "Crossgen2 %(Identity) Size" --dirs ./crossgen/ - $(Python) post.py + $(Python) $(Crossgen2Directory)test.py sod --scenario-name "Crossgen2 %(Identity) Size" --dirs ./crossgen/ + $(Python) $(Crossgen2Directory)post.py diff --git a/eng/common/performance/performance-setup.ps1 b/eng/common/performance/performance-setup.ps1 index 656c0bd9..0edb2ae2 100644 --- a/eng/common/performance/performance-setup.ps1 +++ b/eng/common/performance/performance-setup.ps1 @@ -82,7 +82,9 @@ $SetupArguments = "--repository https://github.com/$Repository --branch $Branch #This grabs the LKG version number of dotnet and passes it to our scripts $VersionJSON = Get-Content global.json | ConvertFrom-Json $DotNetVersion = $VersionJSON.tools.dotnet -$SetupArguments = "--dotnet-versions $DotNetVersion $SetupArguments" +# TODO: Change this back to parsing when we have a good story for dealing with TFM changes or when the LKG in runtime gets updated to include net6.0 +# $SetupArguments = "--dotnet-versions $DotNetVersion $SetupArguments" +$SetupArguments = "--dotnet-versions 6.0.100-alpha.1.20553.6 $SetupArguments" if ($RunFromPerformanceRepo) { diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index 9c0f6c90..315815a9 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -223,7 +223,9 @@ if [[ "$use_latest_dotnet" = false ]]; then # Get the tools section from the global.json. # This grabs the LKG version number of dotnet and passes it to our scripts dotnet_version=`cat global.json | python3 -c 'import json,sys;obj=json.load(sys.stdin);print(obj["tools"]["dotnet"])'` - setup_arguments="--dotnet-versions $dotnet_version $setup_arguments" + # TODO: Change this back to parsing when we have a good story for dealing with TFM changes or when the LKG in runtime gets updated to include net6.0 + # setup_arguments="--dotnet-versions $dotnet_version $setup_arguments" + setup_arguments="--dotnet-versions 6.0.100-alpha.1.20553.6 $setup_arguments" fi if [[ "$run_from_perf_repo" = true ]]; then diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1 index 650b13b0..31cf2767 100644 --- a/eng/common/post-build/publish-using-darc.ps1 +++ b/eng/common/post-build/publish-using-darc.ps1 @@ -10,6 +10,7 @@ param( [Parameter(Mandatory=$false)][string] $EnableNugetValidation, [Parameter(Mandatory=$false)][string] $PublishInstallersAndChecksums, [Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters, + [Parameter(Mandatory=$false)][string] $SymbolPublishingAdditionalParameters, [Parameter(Mandatory=$false)][string] $SigningValidationAdditionalParameters ) @@ -25,6 +26,11 @@ try { $optionalParams.Add($ArtifactsPublishingAdditionalParameters) | Out-Null } + if ("" -ne $SymbolPublishingAdditionalParameters) { + $optionalParams.Add("symbol-publishing-parameters") | Out-Null + $optionalParams.Add($SymbolPublishingAdditionalParameters) | Out-Null + } + if ("false" -eq $WaitPublishingFinish) { $optionalParams.Add("--no-wait") | Out-Null } diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 06048c27..86696793 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -24,7 +24,6 @@ parameters: enablePublishBuildAssets: false enablePublishTestResults: false enablePublishUsingPipelines: false - useBuildManifest: false mergeTestResults: false testRunTitle: '' testResultsFormat: '' @@ -243,12 +242,3 @@ jobs: ArtifactName: AssetManifests continueOnError: ${{ parameters.continueOnError }} condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) - - - ${{ if eq(parameters.useBuildManifest, true) }}: - - task: PublishBuildArtifacts@1 - displayName: Publish Build Manifest - inputs: - PathToPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/manifest.props' - PublishLocation: Container - ArtifactName: BuildManifests - continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 761fb1a2..41f2d96a 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -32,7 +32,6 @@ parameters: symbolPublishingAdditionalParameters: '' artifactsPublishingAdditionalParameters: '' signingValidationAdditionalParameters: '' - useBuildManifest: false # Which stages should finish execution before post-build stages start validateDependsOn: @@ -54,9 +53,6 @@ parameters: NETCoreExperimentalChannelId: 562 NetEngServicesIntChannelId: 678 NetEngServicesProdChannelId: 679 - Net5Preview8ChannelId: 1155 - Net5RC1ChannelId: 1157 - Net5RC2ChannelId: 1329 NetCoreSDK313xxChannelId: 759 NetCoreSDK313xxInternalChannelId: 760 NetCoreSDK314xxChannelId: 921 @@ -94,7 +90,7 @@ stages: inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 arguments: -PromoteToChannels "$(TargetChannels)" - -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview8ChannelId}},${{parameters.Net5RC1ChannelId}},${{parameters.Net5RC2ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}} + -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}} - job: displayName: NuGet Validation @@ -142,16 +138,6 @@ stages: pool: vmImage: 'windows-2019' steps: - - ${{ if eq(parameters.useBuildManifest, true) }}: - - task: DownloadBuildArtifacts@0 - displayName: Download build manifest - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: BuildManifests - task: DownloadBuildArtifacts@0 displayName: Download Package Artifacts inputs: @@ -253,6 +239,7 @@ stages: - job: displayName: Publish Using Darc dependsOn: setupMaestroVars + timeoutInMinutes: 120 variables: - name: BARBuildId value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] @@ -269,6 +256,8 @@ stages: -MaestroToken '$(MaestroApiAccessToken)' -WaitPublishingFinish ${{ parameters.waitPublishingFinish }} -PublishInstallersAndChecksums ${{ parameters.publishInstallersAndChecksums }} + -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' + -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' - ${{ if and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')) }}: - template: \eng\common\templates\post-build\channels\generic-public-channel.yml @@ -303,54 +292,6 @@ stages: shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6-symbols/nuget/v3/index.json' - - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net5_Preview8_Publish' - channelName: '.NET 5 Preview 8' - akaMSChannelName: 'net5/preview8' - channelId: ${{ parameters.Net5Preview8ChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-symbols/nuget/v3/index.json' - - - template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net5_RC1_Publish' - channelName: '.NET 5 RC 1' - akaMSChannelName: 'net5/rc1' - channelId: ${{ parameters.Net5RC1ChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' - - - template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net5_RC2_Publish' - channelName: '.NET 5 RC 2' - akaMSChannelName: 'net5/rc2' - channelId: ${{ parameters.Net5RC2ChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' - - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: BARBuildId: ${{ parameters.BARBuildId }} diff --git a/global.json b/global.json index cf1538a0..ed756d2b 100644 --- a/global.json +++ b/global.json @@ -1,8 +1,8 @@ { "tools": { - "dotnet": "5.0.100-rc.2.20479.15" + "dotnet": "5.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.20552.5" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.20561.5" } } From 1cd2ee4fdeb2905e57a4afc3b80ef3f6a229ce4a Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 23 Nov 2020 13:57:38 +0000 Subject: [PATCH 111/161] Update dependencies from https://github.com/dotnet/arcade build 20201117.7 (#676) [master] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 +-- eng/common/SetupNugetSources.ps1 | 5 +-- eng/common/SetupNugetSources.sh | 4 +-- eng/common/cross/armel/armel.jessie.patch | 43 +++++++++++++++++++++++ eng/common/cross/build-rootfs.sh | 8 ++++- global.json | 2 +- 6 files changed, 58 insertions(+), 8 deletions(-) create mode 100644 eng/common/cross/armel/armel.jessie.patch diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index d8328ad0..a1c3eb29 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - e65d307045c183af6e48b087571f6b7a05e39f01 + a9a80fb35d2e7da21509441d665a40022ce8f1b4 https://github.com/dotnet/source-build-reference-packages diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index bb361713..a0b5fc37 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -99,8 +99,9 @@ function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $Passw function EnablePrivatePackageSources($DisabledPackageSources) { $maestroPrivateSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]") ForEach ($DisabledPackageSource in $maestroPrivateSources) { - Write-Host "`tEnsuring private source '$($DisabledPackageSource.key)' is enabled" - $DisabledPackageSource.SetAttribute("value", "false") + Write-Host "`tEnsuring private source '$($DisabledPackageSource.key)' is enabled by deleting it from disabledPackageSource" + # Due to https://github.com/NuGet/Home/issues/10291, we must actually remove the disabled entries + $DisabledPackageSources.RemoveChild($DisabledPackageSource) } } diff --git a/eng/common/SetupNugetSources.sh b/eng/common/SetupNugetSources.sh index ef333829..2734601c 100644 --- a/eng/common/SetupNugetSources.sh +++ b/eng/common/SetupNugetSources.sh @@ -158,8 +158,8 @@ if [ "$?" == "0" ]; then for DisabledSourceName in ${DisabledDarcIntSources[@]} ; do if [[ $DisabledSourceName == darc-int* ]] then - OldDisableValue="add key=\"$DisabledSourceName\" value=\"true\"" - NewDisableValue="add key=\"$DisabledSourceName\" value=\"false\"" + OldDisableValue="" + NewDisableValue="" sed -i.bak "s|$OldDisableValue|$NewDisableValue|" $ConfigFile echo "Neutralized disablePackageSources entry for '$DisabledSourceName'" fi diff --git a/eng/common/cross/armel/armel.jessie.patch b/eng/common/cross/armel/armel.jessie.patch new file mode 100644 index 00000000..2d261561 --- /dev/null +++ b/eng/common/cross/armel/armel.jessie.patch @@ -0,0 +1,43 @@ +diff -u -r a/usr/include/urcu/uatomic/generic.h b/usr/include/urcu/uatomic/generic.h +--- a/usr/include/urcu/uatomic/generic.h 2014-10-22 15:00:58.000000000 -0700 ++++ b/usr/include/urcu/uatomic/generic.h 2020-10-30 21:38:28.550000000 -0700 +@@ -69,10 +69,10 @@ + #endif + #ifdef UATOMIC_HAS_ATOMIC_SHORT + case 2: +- return __sync_val_compare_and_swap_2(addr, old, _new); ++ return __sync_val_compare_and_swap_2((uint16_t*) addr, old, _new); + #endif + case 4: +- return __sync_val_compare_and_swap_4(addr, old, _new); ++ return __sync_val_compare_and_swap_4((uint32_t*) addr, old, _new); + #if (CAA_BITS_PER_LONG == 64) + case 8: + return __sync_val_compare_and_swap_8(addr, old, _new); +@@ -109,7 +109,7 @@ + return; + #endif + case 4: +- __sync_and_and_fetch_4(addr, val); ++ __sync_and_and_fetch_4((uint32_t*) addr, val); + return; + #if (CAA_BITS_PER_LONG == 64) + case 8: +@@ -148,7 +148,7 @@ + return; + #endif + case 4: +- __sync_or_and_fetch_4(addr, val); ++ __sync_or_and_fetch_4((uint32_t*) addr, val); + return; + #if (CAA_BITS_PER_LONG == 64) + case 8: +@@ -187,7 +187,7 @@ + return __sync_add_and_fetch_2(addr, val); + #endif + case 4: +- return __sync_add_and_fetch_4(addr, val); ++ return __sync_add_and_fetch_4((uint32_t*) addr, val); + #if (CAA_BITS_PER_LONG == 64) + case 8: + return __sync_add_and_fetch_8(addr, val); diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 575eee9f..6d59e181 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -336,7 +336,7 @@ elif [[ -n $__CodeName ]]; then chroot $__RootfsDir apt-get -f -y install chroot $__RootfsDir apt-get -y install $__UbuntuPackages chroot $__RootfsDir symlinks -cr /usr - chroot $__RootfsDir apt clean + chroot $__RootfsDir apt-get clean if [ $__SkipUnmount == 0 ]; then umount $__RootfsDir/* || true @@ -348,6 +348,12 @@ elif [[ -n $__CodeName ]]; then patch -p1 < $__CrossDir/$__BuildArch/trusty-lttng-2.4.patch popd fi + + if [[ "$__BuildArch" == "armel" && "$__CodeName" == "jessie" ]]; then + pushd $__RootfsDir + patch -p1 < $__CrossDir/$__BuildArch/armel.jessie.patch + popd + fi elif [[ "$__Tizen" == "tizen" ]]; then ROOTFS_DIR=$__RootfsDir $__CrossDir/$__BuildArch/tizen-build-rootfs.sh else diff --git a/global.json b/global.json index ed756d2b..fe5499c1 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "5.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.20561.5" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.20567.7" } } From 8c0a6418cd8f302d6cf29137e81b024524052f3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Luthi?= Date: Wed, 25 Nov 2020 04:22:51 +0100 Subject: [PATCH 112/161] Fix broken link in README (#668) The link was pointing to the master branch so the line numbers have changed on master since then. The new link points to a tag to ensure that line numbers won't change in the future. --- docs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/README.md b/docs/README.md index 88299796..084786f9 100644 --- a/docs/README.md +++ b/docs/README.md @@ -75,7 +75,7 @@ By setting `DeterministicSourcePaths` to true the project opts into mapping all Only set `DeterministicSourcePaths` to true on a build/CI server, never for local builds. In order for the debugger to find source files when debugging a locally built binary, the PDB must contain original, unmapped local paths. -Starting with .NET Core SDK 2.1.300, a fully deterministic build is [turned on](https://github.com/dotnet/roslyn/blob/master/src/Compilers/Core/MSBuildTask/Microsoft.Managed.Core.targets#L45-L55) when both `Deterministic` and `ContinuousIntegrationBuild` properties are set to `true`. +Starting with .NET Core SDK 2.1.300, a fully deterministic build is [turned on](https://github.com/dotnet/roslyn/blob/Visual-Studio-2019-Version-16.7.3/src/Compilers/Core/MSBuildTask/Microsoft.Managed.Core.targets#L131-L141) when both `Deterministic` and `ContinuousIntegrationBuild` properties are set to `true`. ## Example From 2062f0c2bae2674b1b8b4416b0e13a8675560f3b Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 30 Nov 2020 13:32:33 +0000 Subject: [PATCH 113/161] Update dependencies from https://github.com/dotnet/arcade build 20201123.2 (#677) [master] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- eng/common/performance/crossgen_perf.proj | 6 +++--- eng/common/performance/performance-setup.sh | 6 ++++++ eng/common/post-build/sourcelink-validation.ps1 | 2 +- global.json | 2 +- 5 files changed, 13 insertions(+), 7 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index a1c3eb29..3ed7f284 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - a9a80fb35d2e7da21509441d665a40022ce8f1b4 + 35bddd4fbfab8da3518fb920250d7c9e0c3138ff https://github.com/dotnet/source-build-reference-packages diff --git a/eng/common/performance/crossgen_perf.proj b/eng/common/performance/crossgen_perf.proj index cf09e405..eb8bdd9c 100644 --- a/eng/common/performance/crossgen_perf.proj +++ b/eng/common/performance/crossgen_perf.proj @@ -19,7 +19,7 @@ python3 - $(HelixPreCommands);chmod +x $HELIX_WORKITEM_PAYLOAD/startup/Startup;chmod +x $HELIX_WORKITEM_PAYLOAD/startup/perfcollect;sudo apt update + $(HelixPreCommands);chmod +x $HELIX_WORKITEM_PAYLOAD/startup/Startup;chmod +x $HELIX_WORKITEM_PAYLOAD/startup/perfcollect;sudo apt update;chmod +x $HELIX_WORKITEM_PAYLOAD/SOD/SizeOnDisk $HELIX_CORRELATION_PAYLOAD/Core_Root $HELIX_CORRELATION_PAYLOAD/performance/src/scenarios/ $(ScenarioDirectory)crossgen/ @@ -69,7 +69,7 @@ $(WorkItemDirectory) $(Python) $(CrossgenDirectory)pre.py crossgen --core-root $(CoreRoot) --single %(Identity) - $(Python) $(CrossgenDirectory)test.py sod --scenario-name "Crossgen %(Identity) Size" --dirs ./crossgen/ + $(Python) $(CrossgenDirectory)test.py sod --scenario-name "Crossgen %(Identity) Size" --dirs ./crossgen.out/ $(Python) $(CrossgenDirectory)post.py @@ -78,7 +78,7 @@ $(WorkItemDirectory) $(Python) $(Crossgen2Directory)pre.py crossgen2 --core-root $(CoreRoot) --single %(Identity) - $(Python) $(Crossgen2Directory)test.py sod --scenario-name "Crossgen2 %(Identity) Size" --dirs ./crossgen/ + $(Python) $(Crossgen2Directory)test.py sod --scenario-name "Crossgen2 %(Identity) Size" --dirs ./crossgen.out/ $(Python) $(Crossgen2Directory)post.py diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index 315815a9..c8e211bc 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -198,6 +198,12 @@ if [[ "$internal" == true ]]; then else queue=Ubuntu.1804.Amd64.Tiger.Perf fi +else + if [[ "$architecture" = "arm64" ]]; then + queue=ubuntu.1804.armarch.open + else + queue=Ubuntu.1804.Amd64.Open + fi fi if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "false" ]]; then diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1 index 1728b742..1c46f7b6 100644 --- a/eng/common/post-build/sourcelink-validation.ps1 +++ b/eng/common/post-build/sourcelink-validation.ps1 @@ -164,7 +164,7 @@ function CheckJobResult( [ref]$ValidationFailures, [switch]$logErrors) { if ($result -ne '0') { - if ($logError) { + if ($logErrors) { Write-PipelineTelemetryError -Category 'SourceLink' -Message "$packagePath has broken SourceLink links." } $ValidationFailures.Value++ diff --git a/global.json b/global.json index fe5499c1..47f0257b 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "5.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.20567.7" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.20573.2" } } From f3182c5db7acd80fab1e8ce3332b327886692951 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 7 Dec 2020 13:27:36 +0000 Subject: [PATCH 114/161] Update dependencies from https://github.com/dotnet/arcade build 20201201.2 (#679) [master] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- eng/common/tools.sh | 14 +++++++++----- global.json | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 3ed7f284..3a110ecd 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 35bddd4fbfab8da3518fb920250d7c9e0c3138ff + 6d8efa00a7dd2d15e07df673a83cecb8a0a3031f https://github.com/dotnet/source-build-reference-packages diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 98186e78..87152724 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -249,7 +249,7 @@ function with_retries { return 0 fi - timeout=$((2**$retries-1)) + timeout=$((3**$retries-1)) echo "Failed to execute '$@'. Waiting $timeout seconds before next attempt ($retries out of $maxRetries)." 1>&2 sleep $timeout done @@ -271,10 +271,14 @@ function GetDotNetInstallScript { # Use curl if available, otherwise use wget if command -v curl > /dev/null; then - with_retries curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || { - local exit_code=$? - Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." - ExitWithExitCode $exit_code + # first, try directly, if this fails we will retry with verbose logging + curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || { + echo "curl failed; will now retry with verbose logging." + with_retries curl "$install_script_url" -sSL --verbose --retry 10 --create-dirs -o "$install_script" || { + local exit_code=$? + Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." + ExitWithExitCode $exit_code + } } else with_retries wget -v -O "$install_script" "$install_script_url" || { diff --git a/global.json b/global.json index 47f0257b..e88afa82 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "5.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.20573.2" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.20601.2" } } From 0811fcd5406f22b086eb64b554514231a17766fd Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 14 Dec 2020 13:27:17 +0000 Subject: [PATCH 115/161] Update dependencies from https://github.com/dotnet/arcade build 20201210.4 (#680) [master] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 +- eng/common/pipeline-logging-functions.ps1 | 104 ++++++++++++--------- eng/common/pipeline-logging-functions.sh | 26 +++++- eng/common/post-build/post-build-utils.ps1 | 6 +- eng/common/tools.ps1 | 15 ++- eng/common/tools.sh | 13 ++- global.json | 2 +- 7 files changed, 115 insertions(+), 55 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 3a110ecd..f2d4206e 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 6d8efa00a7dd2d15e07df673a83cecb8a0a3031f + 3346d7b4ff047c854913fb2ded6e3aa726da6d11 https://github.com/dotnet/source-build-reference-packages diff --git a/eng/common/pipeline-logging-functions.ps1 b/eng/common/pipeline-logging-functions.ps1 index 8484451f..8e422c56 100644 --- a/eng/common/pipeline-logging-functions.ps1 +++ b/eng/common/pipeline-logging-functions.ps1 @@ -29,14 +29,14 @@ function Write-PipelineTelemetryError { [switch]$AsOutput, [switch]$Force) - $PSBoundParameters.Remove('Category') | Out-Null + $PSBoundParameters.Remove('Category') | Out-Null - if($Force -Or ((Test-Path variable:ci) -And $ci)) { - $Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message" - } - $PSBoundParameters.Remove('Message') | Out-Null - $PSBoundParameters.Add('Message', $Message) - Write-PipelineTaskError @PSBoundParameters + if ($Force -Or ((Test-Path variable:ci) -And $ci)) { + $Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message" + } + $PSBoundParameters.Remove('Message') | Out-Null + $PSBoundParameters.Add('Message', $Message) + Write-PipelineTaskError @PSBoundParameters } # Specify "-Force" to force pipeline formatted output even if "$ci" is false or not set @@ -55,8 +55,8 @@ function Write-PipelineTaskError { [switch]$Force ) - if(!$Force -And (-Not (Test-Path variable:ci) -Or !$ci)) { - if($Type -eq 'error') { + if (!$Force -And (-Not (Test-Path variable:ci) -Or !$ci)) { + if ($Type -eq 'error') { Write-Host $Message -ForegroundColor Red return } @@ -66,47 +66,61 @@ function Write-PipelineTaskError { } } - if(($Type -ne 'error') -and ($Type -ne 'warning')) { + if (($Type -ne 'error') -and ($Type -ne 'warning')) { Write-Host $Message return } $PSBoundParameters.Remove('Force') | Out-Null - if(-not $PSBoundParameters.ContainsKey('Type')) { + if (-not $PSBoundParameters.ContainsKey('Type')) { $PSBoundParameters.Add('Type', 'error') } Write-LogIssue @PSBoundParameters - } +} - function Write-PipelineSetVariable { +function Write-PipelineSetVariable { [CmdletBinding()] param( - [Parameter(Mandatory = $true)] - [string]$Name, - [string]$Value, - [switch]$Secret, - [switch]$AsOutput, - [bool]$IsMultiJobVariable=$true) - - if((Test-Path variable:ci) -And $ci) { + [Parameter(Mandatory = $true)] + [string]$Name, + [string]$Value, + [switch]$Secret, + [switch]$AsOutput, + [bool]$IsMultiJobVariable = $true) + + if ((Test-Path variable:ci) -And $ci) { Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $Value -Properties @{ - 'variable' = $Name - 'isSecret' = $Secret - 'isOutput' = $IsMultiJobVariable + 'variable' = $Name + 'isSecret' = $Secret + 'isOutput' = $IsMultiJobVariable } -AsOutput:$AsOutput - } - } + } +} - function Write-PipelinePrependPath { +function Write-PipelinePrependPath { [CmdletBinding()] param( - [Parameter(Mandatory=$true)] - [string]$Path, - [switch]$AsOutput) + [Parameter(Mandatory = $true)] + [string]$Path, + [switch]$AsOutput) - if((Test-Path variable:ci) -And $ci) { + if ((Test-Path variable:ci) -And $ci) { Write-LoggingCommand -Area 'task' -Event 'prependpath' -Data $Path -AsOutput:$AsOutput - } - } + } +} + +function Write-PipelineSetResult { + [CmdletBinding()] + param( + [ValidateSet("Succeeded", "SucceededWithIssues", "Failed", "Cancelled", "Skipped")] + [Parameter(Mandatory = $true)] + [string]$Result, + [string]$Message) + if ((Test-Path variable:ci) -And $ci) { + Write-LoggingCommand -Area 'task' -Event 'complete' -Data $Message -Properties @{ + 'result' = $Result + } + } +} <######################################## # Private functions. @@ -123,7 +137,8 @@ function Format-LoggingCommandData { foreach ($mapping in $script:loggingCommandEscapeMappings) { $Value = $Value.Replace($mapping.Token, $mapping.Replacement) } - } else { + } + else { for ($i = $script:loggingCommandEscapeMappings.Length - 1 ; $i -ge 0 ; $i--) { $mapping = $script:loggingCommandEscapeMappings[$i] $Value = $Value.Replace($mapping.Replacement, $mapping.Token) @@ -156,7 +171,8 @@ function Format-LoggingCommand { if ($first) { $null = $sb.Append(' ') $first = $false - } else { + } + else { $null = $sb.Append(';') } @@ -193,7 +209,8 @@ function Write-LoggingCommand { $command = Format-LoggingCommand -Area $Area -Event $Event -Data $Data -Properties $Properties if ($AsOutput) { $command - } else { + } + else { Write-Host $command } } @@ -212,12 +229,12 @@ function Write-LogIssue { [switch]$AsOutput) $command = Format-LoggingCommand -Area 'task' -Event 'logissue' -Data $Message -Properties @{ - 'type' = $Type - 'code' = $ErrCode - 'sourcepath' = $SourcePath - 'linenumber' = $LineNumber - 'columnnumber' = $ColumnNumber - } + 'type' = $Type + 'code' = $ErrCode + 'sourcepath' = $SourcePath + 'linenumber' = $LineNumber + 'columnnumber' = $ColumnNumber + } if ($AsOutput) { return $command } @@ -229,7 +246,8 @@ function Write-LogIssue { $foregroundColor = [System.ConsoleColor]::Red $backgroundColor = [System.ConsoleColor]::Black } - } else { + } + else { $foregroundColor = $host.PrivateData.WarningForegroundColor $backgroundColor = $host.PrivateData.WarningBackgroundColor if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) { diff --git a/eng/common/pipeline-logging-functions.sh b/eng/common/pipeline-logging-functions.sh index 6cd0a340..da5a7e61 100755 --- a/eng/common/pipeline-logging-functions.sh +++ b/eng/common/pipeline-logging-functions.sh @@ -179,4 +179,28 @@ function Write-PipelinePrependPath { if [[ "$ci" == true ]]; then echo "##vso[task.prependpath]$prepend_path" fi -} \ No newline at end of file +} + +function Write-PipelineSetResult { + local result='' + local message='' + + while [[ $# -gt 0 ]]; do + opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" + case "$opt" in + -result|-r) + result=$2 + shift + ;; + -message|-m) + message=$2 + shift + ;; + esac + shift + done + + if [[ "$ci" == true ]]; then + echo "##vso[task.complete result=$result;]$message" + fi +} diff --git a/eng/common/post-build/post-build-utils.ps1 b/eng/common/post-build/post-build-utils.ps1 index 7d497447..534f6988 100644 --- a/eng/common/post-build/post-build-utils.ps1 +++ b/eng/common/post-build/post-build-utils.ps1 @@ -69,9 +69,9 @@ function Trigger-Subscription([string]$SubscriptionId) { function Validate-MaestroVars { try { - Get-Variable MaestroApiEndPoint -Scope Global | Out-Null - Get-Variable MaestroApiVersion -Scope Global | Out-Null - Get-Variable MaestroApiAccessToken -Scope Global | Out-Null + Get-Variable MaestroApiEndPoint | Out-Null + Get-Variable MaestroApiVersion | Out-Null + Get-Variable MaestroApiAccessToken | Out-Null if (!($MaestroApiEndPoint -Match '^http[s]?://maestro-(int|prod).westus2.cloudapp.azure.com$')) { Write-PipelineTelemetryError -Category 'MaestroVars' -Message "MaestroApiEndPoint is not a valid Maestro URL. '$MaestroApiEndPoint'" diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index fc09059f..bc8b66e2 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -677,14 +677,23 @@ function MSBuild-Core() { $exitCode = Exec-Process $buildTool.Path $cmdArgs if ($exitCode -ne 0) { - Write-PipelineTelemetryError -Category 'Build' -Message 'Build failed.' + # We should not Write-PipelineTaskError here because that message shows up in the build summary + # The build already logged an error, that's the reason it failed. Producing an error here only adds noise. + Write-Host "Build failed with exit code $exitCode. Check errors above." -ForegroundColor Red $buildLog = GetMSBuildBinaryLogCommandLineArgument $args - if ($buildLog -ne $null) { + if ($null -ne $buildLog) { Write-Host "See log: $buildLog" -ForegroundColor DarkGray } - ExitWithExitCode $exitCode + if ($ci) { + Write-PipelineSetResult -Result "Failed" -Message "msbuild execution failed." + # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error + # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error + ExitWithExitCode 0 + } else { + ExitWithExitCode $exitCode + } } } diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 87152724..0295dd9f 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -441,8 +441,17 @@ function MSBuild-Core { "$_InitializeBuildTool" "$@" || { local exit_code=$? - Write-PipelineTaskError "Build failed (exit code '$exit_code')." - ExitWithExitCode $exit_code + # We should not Write-PipelineTaskError here because that message shows up in the build summary + # The build already logged an error, that's the reason it failed. Producing an error here only adds noise. + echo "Build failed with exit code $exit_code. Check errors above." + if [[ "$ci" == "true" ]]; then + Write-PipelineSetResult -result "Failed" -message "msbuild execution failed." + # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error + # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error + ExitWithExitCode 0 + else + ExitWithExitCode $exit_code + fi } } diff --git a/global.json b/global.json index e88afa82..2c1d5b22 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "5.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.20601.2" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.20610.4" } } From 9cd2fb642acb474fec620a94f3fff9fb8c8c0306 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 21 Dec 2020 13:22:38 +0000 Subject: [PATCH 116/161] Update dependencies from https://github.com/dotnet/arcade build 20201216.18 (#682) [master] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- eng/common/native/CommonLibrary.psm1 | 2 +- eng/common/performance/microbenchmarks.proj | 2 +- global.json | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index f2d4206e..82060350 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 3346d7b4ff047c854913fb2ded6e3aa726da6d11 + 26b005488dd7ddf6356873cb01a7b763a82a9622 https://github.com/dotnet/source-build-reference-packages diff --git a/eng/common/native/CommonLibrary.psm1 b/eng/common/native/CommonLibrary.psm1 index d7d1a651..adf707c8 100644 --- a/eng/common/native/CommonLibrary.psm1 +++ b/eng/common/native/CommonLibrary.psm1 @@ -48,7 +48,7 @@ function DownloadAndExtract { -Verbose:$Verbose if ($DownloadStatus -Eq $False) { - Write-Error "Download failed" + Write-Error "Download failed from $Uri" return $False } diff --git a/eng/common/performance/microbenchmarks.proj b/eng/common/performance/microbenchmarks.proj index 94b6efbc..318ca5f1 100644 --- a/eng/common/performance/microbenchmarks.proj +++ b/eng/common/performance/microbenchmarks.proj @@ -141,4 +141,4 @@ 4:00 - \ No newline at end of file + diff --git a/global.json b/global.json index 2c1d5b22..c08261ce 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "5.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.20610.4" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.20616.18" } } From a681f62457a6b30eacc4b48a62267a8481fd4d58 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 28 Dec 2020 13:22:35 +0000 Subject: [PATCH 117/161] Update dependencies from https://github.com/dotnet/arcade build 20201221.12 (#684) [master] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- eng/common/post-build/publish-using-darc.ps1 | 4 ++-- eng/common/tools.ps1 | 25 ++++++++++++++++---- global.json | 2 +- 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 82060350..90211fbd 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 26b005488dd7ddf6356873cb01a7b763a82a9622 + 3ba79fbd73d6765b67d0f75ac9dac148d6bea346 https://github.com/dotnet/source-build-reference-packages diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1 index 31cf2767..6b68ee84 100644 --- a/eng/common/post-build/publish-using-darc.ps1 +++ b/eng/common/post-build/publish-using-darc.ps1 @@ -22,12 +22,12 @@ try { $optionalParams = [System.Collections.ArrayList]::new() if ("" -ne $ArtifactsPublishingAdditionalParameters) { - $optionalParams.Add("artifact-publishing-parameters") | Out-Null + $optionalParams.Add("--artifact-publishing-parameters") | Out-Null $optionalParams.Add($ArtifactsPublishingAdditionalParameters) | Out-Null } if ("" -ne $SymbolPublishingAdditionalParameters) { - $optionalParams.Add("symbol-publishing-parameters") | Out-Null + $optionalParams.Add("--symbol-publishing-parameters") | Out-Null $optionalParams.Add($SymbolPublishingAdditionalParameters) | Out-Null } diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index bc8b66e2..ce280eb9 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -439,11 +439,26 @@ function LocateVisualStudio([object]$vsRequirements = $null){ if (!(Test-Path $vsWhereExe)) { Create-Directory $vsWhereDir Write-Host 'Downloading vswhere' - try { - Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe - } - catch { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_ + $maxRetries = 5 + $retries = 1 + + while($true) { + try { + Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe + break + } + catch{ + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_ + } + + if (++$retries -le $maxRetries) { + $delayInSeconds = [math]::Pow(2, $retries) - 1 # Exponential backoff + Write-Host "Retrying. Waiting for $delayInSeconds seconds before next attempt ($retries of $maxRetries)." + Start-Sleep -Seconds $delayInSeconds + } + else { + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unable to download file in $maxRetries attempts." + } } } diff --git a/global.json b/global.json index c08261ce..20e92295 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "5.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.20616.18" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.20621.12" } } From edbee72a41b59fc57ea6e89142653c817f75ec66 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 4 Jan 2021 13:37:39 +0000 Subject: [PATCH 118/161] Update dependencies from https://github.com/dotnet/arcade build 20201230.2 (#685) [master] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- eng/common/tools.sh | 4 +++- global.json | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 90211fbd..ef3868fc 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 3ba79fbd73d6765b67d0f75ac9dac148d6bea346 + 4c8515c18ebe0071c32fed467ee3890fbd488898 https://github.com/dotnet/source-build-reference-packages diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 0295dd9f..b160c370 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -273,7 +273,9 @@ function GetDotNetInstallScript { if command -v curl > /dev/null; then # first, try directly, if this fails we will retry with verbose logging curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || { - echo "curl failed; will now retry with verbose logging." + echo "Curl failed; dumping some information about dotnet.microsoft.com for later investigation" + echo | openssl s_client -showcerts -servername dotnet.microsoft.com -connect dotnet.microsoft.com:443 + echo "Will now retry the same URL with verbose logging." with_retries curl "$install_script_url" -sSL --verbose --retry 10 --create-dirs -o "$install_script" || { local exit_code=$? Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." diff --git a/global.json b/global.json index 20e92295..b13e4fb6 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "5.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.20621.12" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.20630.2" } } From 64beae3ef9db88c1fc505a81b3cfb227592da785 Mon Sep 17 00:00:00 2001 From: Kai <8343141+Mik4sa@users.noreply.github.com> Date: Tue, 5 Jan 2021 19:11:29 +0100 Subject: [PATCH 119/161] Added support for gitea (#674) --- SourceLink.sln | 12 ++ .../GiteaTests.cs | 168 ++++++++++++++++++ .../GetSourceLinkUrlTests.cs | 55 ++++++ ...icrosoft.SourceLink.Gitea.UnitTests.csproj | 9 + .../TranslateRepositoryUrlsTests.cs | 50 ++++++ src/SourceLink.Gitea/GetSourceLinkUrl.cs | 22 +++ .../Microsoft.SourceLink.Gitea.csproj | 33 ++++ .../Microsoft.SourceLink.Gitea.nuspec | 17 ++ .../TranslateRepositoryUrls.cs | 10 ++ .../build/Microsoft.SourceLink.Gitea.targets | 66 +++++++ .../Microsoft.SourceLink.Gitea.targets | 4 + src/SourceLink.Gitea/xlf/Resources.cs.xlf | 32 ++++ src/SourceLink.Gitea/xlf/Resources.de.xlf | 32 ++++ src/SourceLink.Gitea/xlf/Resources.es.xlf | 32 ++++ src/SourceLink.Gitea/xlf/Resources.fr.xlf | 32 ++++ src/SourceLink.Gitea/xlf/Resources.it.xlf | 32 ++++ src/SourceLink.Gitea/xlf/Resources.ja.xlf | 32 ++++ src/SourceLink.Gitea/xlf/Resources.ko.xlf | 32 ++++ src/SourceLink.Gitea/xlf/Resources.pl.xlf | 32 ++++ src/SourceLink.Gitea/xlf/Resources.pt-BR.xlf | 32 ++++ src/SourceLink.Gitea/xlf/Resources.ru.xlf | 32 ++++ src/SourceLink.Gitea/xlf/Resources.tr.xlf | 32 ++++ .../xlf/Resources.zh-Hans.xlf | 32 ++++ .../xlf/Resources.zh-Hant.xlf | 32 ++++ 24 files changed, 862 insertions(+) create mode 100644 src/SourceLink.Git.IntegrationTests/GiteaTests.cs create mode 100644 src/SourceLink.Gitea.UnitTests/GetSourceLinkUrlTests.cs create mode 100644 src/SourceLink.Gitea.UnitTests/Microsoft.SourceLink.Gitea.UnitTests.csproj create mode 100644 src/SourceLink.Gitea.UnitTests/TranslateRepositoryUrlsTests.cs create mode 100644 src/SourceLink.Gitea/GetSourceLinkUrl.cs create mode 100644 src/SourceLink.Gitea/Microsoft.SourceLink.Gitea.csproj create mode 100644 src/SourceLink.Gitea/Microsoft.SourceLink.Gitea.nuspec create mode 100644 src/SourceLink.Gitea/TranslateRepositoryUrls.cs create mode 100644 src/SourceLink.Gitea/build/Microsoft.SourceLink.Gitea.targets create mode 100644 src/SourceLink.Gitea/buildMultiTargeting/Microsoft.SourceLink.Gitea.targets create mode 100644 src/SourceLink.Gitea/xlf/Resources.cs.xlf create mode 100644 src/SourceLink.Gitea/xlf/Resources.de.xlf create mode 100644 src/SourceLink.Gitea/xlf/Resources.es.xlf create mode 100644 src/SourceLink.Gitea/xlf/Resources.fr.xlf create mode 100644 src/SourceLink.Gitea/xlf/Resources.it.xlf create mode 100644 src/SourceLink.Gitea/xlf/Resources.ja.xlf create mode 100644 src/SourceLink.Gitea/xlf/Resources.ko.xlf create mode 100644 src/SourceLink.Gitea/xlf/Resources.pl.xlf create mode 100644 src/SourceLink.Gitea/xlf/Resources.pt-BR.xlf create mode 100644 src/SourceLink.Gitea/xlf/Resources.ru.xlf create mode 100644 src/SourceLink.Gitea/xlf/Resources.tr.xlf create mode 100644 src/SourceLink.Gitea/xlf/Resources.zh-Hans.xlf create mode 100644 src/SourceLink.Gitea/xlf/Resources.zh-Hant.xlf diff --git a/SourceLink.sln b/SourceLink.sln index 39b64cbd..49104dbd 100644 --- a/SourceLink.sln +++ b/SourceLink.sln @@ -64,6 +64,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.SourceLink.Tools. EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.SourceLink.Tools.UnitTests", "src\SourceLink.Tools.UnitTests\Microsoft.SourceLink.Tools.UnitTests.csproj", "{99D113A9-24EC-471D-9F74-D2AC2F16220B}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.SourceLink.Gitea", "src\SourceLink.Gitea\Microsoft.SourceLink.Gitea.csproj", "{C16867BE-053E-4AD3-9867-3352658FC56C}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.SourceLink.Gitea.UnitTests", "src\SourceLink.Gitea.UnitTests\Microsoft.SourceLink.Gitea.UnitTests.csproj", "{04C95AC8-E3A4-4A2B-94E6-4C62E910FD8A}" +EndProject Global GlobalSection(SharedMSBuildProjectFiles) = preSolution src\SourceLink.Tools\Microsoft.SourceLink.Tools.projitems*{5df76cc2-5f0e-45a6-ad56-6bbbccbc1a78}*SharedItemsImports = 13 @@ -166,6 +170,14 @@ Global {99D113A9-24EC-471D-9F74-D2AC2F16220B}.Debug|Any CPU.Build.0 = Debug|Any CPU {99D113A9-24EC-471D-9F74-D2AC2F16220B}.Release|Any CPU.ActiveCfg = Release|Any CPU {99D113A9-24EC-471D-9F74-D2AC2F16220B}.Release|Any CPU.Build.0 = Release|Any CPU + {C16867BE-053E-4AD3-9867-3352658FC56C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C16867BE-053E-4AD3-9867-3352658FC56C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C16867BE-053E-4AD3-9867-3352658FC56C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C16867BE-053E-4AD3-9867-3352658FC56C}.Release|Any CPU.Build.0 = Release|Any CPU + {04C95AC8-E3A4-4A2B-94E6-4C62E910FD8A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {04C95AC8-E3A4-4A2B-94E6-4C62E910FD8A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {04C95AC8-E3A4-4A2B-94E6-4C62E910FD8A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {04C95AC8-E3A4-4A2B-94E6-4C62E910FD8A}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/SourceLink.Git.IntegrationTests/GiteaTests.cs b/src/SourceLink.Git.IntegrationTests/GiteaTests.cs new file mode 100644 index 00000000..d0c06e9c --- /dev/null +++ b/src/SourceLink.Git.IntegrationTests/GiteaTests.cs @@ -0,0 +1,168 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.IO; +using TestUtilities; + +namespace Microsoft.SourceLink.IntegrationTests +{ + public class GiteaTests : DotNetSdkTestBase + { + public GiteaTests() + : base("Microsoft.SourceLink.Gitea") + { + } + + [ConditionalFact(typeof(DotNetSdkAvailable))] + public void FullValidation_Https() + { + // Test non-ascii characters and escapes in the URL. + // Escaped URI reserved characters should remain escaped, non-reserved characters unescaped in the results. + var repoUrl = "https://噸.com/test-org/test-%72epo\u1234%24%2572%2F"; + var repoName = "test-repo\u1234%24%2572%2F"; + + var repo = GitUtilities.CreateGitRepositoryWithSingleCommit(ProjectDir.Path, new[] { ProjectFileName }, repoUrl); + var commitSha = repo.Head.Tip.Sha; + + VerifyValues( + customProps: @" + + true + + + + +", + customTargets: "", + targets: new[] + { + "Build", "Pack" + }, + expressions: new[] + { + "@(SourceRoot)", + "@(SourceRoot->'%(SourceLinkUrl)')", + "$(SourceLink)", + "$(PrivateRepositoryUrl)", + "$(RepositoryUrl)" + }, + expectedResults: new[] + { + NuGetPackageFolders, + ProjectSourceRoot, + $"https://噸.com/test-org/{repoName}/raw/commit/{commitSha}/*", + s_relativeSourceLinkJsonPath, + $"https://噸.com/test-org/{repoName}", + $"https://噸.com/test-org/{repoName}" + }); + + AssertEx.AreEqual( + $@"{{""documents"":{{""{ProjectSourceRoot.Replace(@"\", @"\\")}*"":""https://噸.com/test-org/{repoName}/raw/commit/{commitSha}/*""}}}}", + File.ReadAllText(Path.Combine(ProjectDir.Path, s_relativeSourceLinkJsonPath))); + + TestUtilities.ValidateAssemblyInformationalVersion( + Path.Combine(ProjectDir.Path, s_relativeOutputFilePath), + "1.0.0+" + commitSha); + + TestUtilities.ValidateNuSpecRepository( + Path.Combine(ProjectDir.Path, s_relativePackagePath), + type: "git", + commit: commitSha, + url: $"https://噸.com/test-org/{repoName}"); + } + + [ConditionalFact(typeof(DotNetSdkAvailable))] + public void FullValidation_Ssh() + { + // Test non-ascii characters and escapes in the URL. + // Escaped URI reserved characters should remain escaped, non-reserved characters unescaped in the results. + var repoUrl = "test-user@噸.com:test-org/test-%72epo\u1234%24%2572%2F"; + var repoName = "test-repo\u1234%24%2572%2F"; + + var repo = GitUtilities.CreateGitRepositoryWithSingleCommit(ProjectDir.Path, new[] { ProjectFileName }, repoUrl); + var commitSha = repo.Head.Tip.Sha; + + VerifyValues( + customProps: @" + + true + + + + +", + customTargets: "", + targets: new[] + { + "Build", "Pack" + }, + expressions: new[] + { + "@(SourceRoot)", + "@(SourceRoot->'%(SourceLinkUrl)')", + "$(SourceLink)", + "$(PrivateRepositoryUrl)", + "$(RepositoryUrl)" + }, + expectedResults: new[] + { + NuGetPackageFolders, + ProjectSourceRoot, + $"https://噸.com/test-org/{repoName}/raw/commit/{commitSha}/*", + s_relativeSourceLinkJsonPath, + $"https://噸.com/test-org/{repoName}", + $"https://噸.com/test-org/{repoName}" + }); + + AssertEx.AreEqual( + $@"{{""documents"":{{""{ProjectSourceRoot.Replace(@"\", @"\\")}*"":""https://噸.com/test-org/{repoName}/raw/commit/{commitSha}/*""}}}}", + File.ReadAllText(Path.Combine(ProjectDir.Path, s_relativeSourceLinkJsonPath))); + + TestUtilities.ValidateAssemblyInformationalVersion( + Path.Combine(ProjectDir.Path, s_relativeOutputFilePath), + "1.0.0+" + commitSha); + + TestUtilities.ValidateNuSpecRepository( + Path.Combine(ProjectDir.Path, s_relativePackagePath), + type: "git", + commit: commitSha, + url: $"https://噸.com/test-org/{repoName}"); + } + + + [ConditionalFact(typeof(DotNetSdkAvailable))] + public void ImplicitHost() + { + // Test non-ascii characters and escapes in the URL. + // Escaped URI reserved characters should remain escaped, non-reserved characters unescaped in the results. + var repoUrl = "http://噸.com/test-org/test-%72epo\u1234%24%2572%2F"; + var repoName = "test-repo\u1234%24%2572%2F"; + + var repo = GitUtilities.CreateGitRepositoryWithSingleCommit(ProjectDir.Path, new[] { ProjectFileName }, repoUrl); + var commitSha = repo.Head.Tip.Sha; + + VerifyValues( + customProps: @" + + true + +", + customTargets: "", + targets: new[] + { + "Build", "Pack" + }, + expressions: new[] + { + "@(SourceRoot->'%(SourceLinkUrl)')", + "$(PrivateRepositoryUrl)", + "$(RepositoryUrl)" + }, + expectedResults: new[] + { + $"http://噸.com/test-org/{repoName}/raw/commit/{commitSha}/*", + $"http://噸.com/test-org/{repoName}", + $"http://噸.com/test-org/{repoName}" + }); + } + } +} diff --git a/src/SourceLink.Gitea.UnitTests/GetSourceLinkUrlTests.cs b/src/SourceLink.Gitea.UnitTests/GetSourceLinkUrlTests.cs new file mode 100644 index 00000000..06e75407 --- /dev/null +++ b/src/SourceLink.Gitea.UnitTests/GetSourceLinkUrlTests.cs @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using Microsoft.Build.Tasks.SourceControl; +using TestUtilities; +using Xunit; +using static TestUtilities.KeyValuePairUtils; + +namespace Microsoft.SourceLink.Gitea.UnitTests +{ + public class GetSourceLinkUrlTests + { + [Fact] + public void EmptyHosts() + { + var engine = new MockEngine(); + + var task = new GetSourceLinkUrl() + { + BuildEngine = engine, + SourceRoot = new MockItem("x", KVP("RepositoryUrl", "http://abc.com"), KVP("SourceControl", "git")), + }; + + bool result = task.Execute(); + + AssertEx.AssertEqualToleratingWhitespaceDifferences( + "ERROR : " + string.Format(CommonResources.AtLeastOneRepositoryHostIsRequired, "SourceLinkGiteaHost", "Gitea"), engine.Log); + + Assert.False(result); + } + + [Theory] + [InlineData("", "")] + [InlineData("", "/")] + [InlineData("/", "")] + [InlineData("/", "/")] + public void BuildSourceLinkUrl(string s1, string s2) + { + var engine = new MockEngine(); + + var task = new GetSourceLinkUrl() + { + BuildEngine = engine, + SourceRoot = new MockItem("/src/", KVP("RepositoryUrl", "http://subdomain.mygitea.com:100/a/b" + s1), KVP("SourceControl", "git"), KVP("RevisionId", "0123456789abcdefABCDEF000000000000000000")), + Hosts = new[] + { + new MockItem("mygitea.com", KVP("ContentUrl", "https://domain.com/x/y" + s2)), + } + }; + + bool result = task.Execute(); + AssertEx.AssertEqualToleratingWhitespaceDifferences("", engine.Log); + AssertEx.AreEqual("https://domain.com/x/y/a/b/raw/commit/0123456789abcdefABCDEF000000000000000000/*", task.SourceLinkUrl); + Assert.True(result); + } + } +} diff --git a/src/SourceLink.Gitea.UnitTests/Microsoft.SourceLink.Gitea.UnitTests.csproj b/src/SourceLink.Gitea.UnitTests/Microsoft.SourceLink.Gitea.UnitTests.csproj new file mode 100644 index 00000000..12bdc2d6 --- /dev/null +++ b/src/SourceLink.Gitea.UnitTests/Microsoft.SourceLink.Gitea.UnitTests.csproj @@ -0,0 +1,9 @@ + + + net472;net5.0 + + + + + + diff --git a/src/SourceLink.Gitea.UnitTests/TranslateRepositoryUrlsTests.cs b/src/SourceLink.Gitea.UnitTests/TranslateRepositoryUrlsTests.cs new file mode 100644 index 00000000..3949583d --- /dev/null +++ b/src/SourceLink.Gitea.UnitTests/TranslateRepositoryUrlsTests.cs @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System.Linq; +using TestUtilities; +using Xunit; +using static TestUtilities.KeyValuePairUtils; + +namespace Microsoft.SourceLink.Gitea.UnitTests +{ + public class TranslateRepositoryUrlsTests + { + [Fact] + public void Translate() + { + var engine = new MockEngine(); + + var task = new TranslateRepositoryUrls() + { + BuildEngine = engine, + RepositoryUrl = "ssh://gitea.com/a/b", + IsSingleProvider = true, + SourceRoots = new[] + { + new MockItem("/1/", KVP("SourceControl", "git"), KVP("ScmRepositoryUrl", "ssh://gitea.com:22/a/b")), + new MockItem("/2/", KVP("SourceControl", "tfvc"), KVP("ScmRepositoryUrl", "ssh://gitea1.com/a/b")), + new MockItem("/2/", KVP("SourceControl", "git"), KVP("ScmRepositoryUrl", "ssh://gitea1.com/a/b")), + new MockItem("/2/", KVP("SourceControl", "tfvc"), KVP("ScmRepositoryUrl", "ssh://gitea2.com/a/b")), + }, + Hosts = new[] + { + new MockItem("gitea1.com") + } + }; + + bool result = task.Execute(); + AssertEx.AssertEqualToleratingWhitespaceDifferences("", engine.Log); + + AssertEx.AreEqual("https://gitea.com/a/b", task.TranslatedRepositoryUrl); + + AssertEx.Equal(new[] + { + "https://gitea.com/a/b", + "ssh://gitea1.com/a/b", + "https://gitea1.com/a/b", + "ssh://gitea2.com/a/b" + }, task.TranslatedSourceRoots?.Select(r => r.GetMetadata("ScmRepositoryUrl"))); + + Assert.True(result); + } + } +} diff --git a/src/SourceLink.Gitea/GetSourceLinkUrl.cs b/src/SourceLink.Gitea/GetSourceLinkUrl.cs new file mode 100644 index 00000000..5d5526e7 --- /dev/null +++ b/src/SourceLink.Gitea/GetSourceLinkUrl.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Microsoft.Build.Framework; +using Microsoft.Build.Tasks.SourceControl; + +namespace Microsoft.SourceLink.Gitea +{ + /// + /// The task calculates SourceLink URL for a given SourceRoot. + /// If the SourceRoot is associated with a git repository with a recognized domain the + /// output property is set to the content URL corresponding to the domain, otherwise it is set to string "N/A". + /// + public sealed class GetSourceLinkUrl : GetSourceLinkUrlGitTask + { + protected override string HostsItemGroupName => "SourceLinkGiteaHost"; + protected override string ProviderDisplayName => "Gitea"; + + protected override string? BuildSourceLinkUrl(Uri contentUri, Uri gitUri, string relativeUrl, string revisionId, ITaskItem? hostItem) + => UriUtilities.Combine(UriUtilities.Combine(contentUri.ToString(), relativeUrl), "raw/commit/" + revisionId + "/*"); + } +} diff --git a/src/SourceLink.Gitea/Microsoft.SourceLink.Gitea.csproj b/src/SourceLink.Gitea/Microsoft.SourceLink.Gitea.csproj new file mode 100644 index 00000000..b5bbc93d --- /dev/null +++ b/src/SourceLink.Gitea/Microsoft.SourceLink.Gitea.csproj @@ -0,0 +1,33 @@ + + + net472;net5.0 + true + + + true + $(MSBuildProjectName).nuspec + $(OutputPath) + + Generates source link for Gitea repositories. + MSBuild Tasks Gitea source link + true + + + + + + + + + Microsoft.Build.Tasks.SourceControl + true + + + + + + + + + + diff --git a/src/SourceLink.Gitea/Microsoft.SourceLink.Gitea.nuspec b/src/SourceLink.Gitea/Microsoft.SourceLink.Gitea.nuspec new file mode 100644 index 00000000..a4c7ff74 --- /dev/null +++ b/src/SourceLink.Gitea/Microsoft.SourceLink.Gitea.nuspec @@ -0,0 +1,17 @@ + + + + $CommonMetadataElements$ + + + + + + + $CommonFileElements$ + + + + + + \ No newline at end of file diff --git a/src/SourceLink.Gitea/TranslateRepositoryUrls.cs b/src/SourceLink.Gitea/TranslateRepositoryUrls.cs new file mode 100644 index 00000000..1bc949a7 --- /dev/null +++ b/src/SourceLink.Gitea/TranslateRepositoryUrls.cs @@ -0,0 +1,10 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.Build.Tasks.SourceControl; + +namespace Microsoft.SourceLink.Gitea +{ + public sealed class TranslateRepositoryUrls : TranslateRepositoryUrlsGitTask + { + } +} diff --git a/src/SourceLink.Gitea/build/Microsoft.SourceLink.Gitea.targets b/src/SourceLink.Gitea/build/Microsoft.SourceLink.Gitea.targets new file mode 100644 index 00000000..5b6765e5 --- /dev/null +++ b/src/SourceLink.Gitea/build/Microsoft.SourceLink.Gitea.targets @@ -0,0 +1,66 @@ + + + + <_SourceLinkGiteaAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net472\Microsoft.SourceLink.Gitea.dll + <_SourceLinkGiteaAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\net5.0\Microsoft.SourceLink.Gitea.dll + + + + + + + $(SourceLinkUrlInitializerTargets);_InitializeGiteaSourceLinkUrl + $(SourceControlManagerUrlTranslationTargets);TranslateGiteaUrlsInSourceControlInformation + + + + + + + + + + + + + + + + + + + <_TranslatedSourceRoot Remove="@(_TranslatedSourceRoot)"/> + + + + + + + + + + + + + + diff --git a/src/SourceLink.Gitea/buildMultiTargeting/Microsoft.SourceLink.Gitea.targets b/src/SourceLink.Gitea/buildMultiTargeting/Microsoft.SourceLink.Gitea.targets new file mode 100644 index 00000000..84afbc37 --- /dev/null +++ b/src/SourceLink.Gitea/buildMultiTargeting/Microsoft.SourceLink.Gitea.targets @@ -0,0 +1,4 @@ + + + + diff --git a/src/SourceLink.Gitea/xlf/Resources.cs.xlf b/src/SourceLink.Gitea/xlf/Resources.cs.xlf new file mode 100644 index 00000000..0e6540e3 --- /dev/null +++ b/src/SourceLink.Gitea/xlf/Resources.cs.xlf @@ -0,0 +1,32 @@ + + + + + + {0} item group is empty. At least one {1} repository host is required in order to generate SourceLink. + Skupina položek {0} je prázdná. Kvůli generování zdrojového odkazu se vyžaduje alespoň jeden hostitel úložiště {1}. + + + + {0} specifies an invalid URL: '{1}' + {0} určuje neplatnou adresu URL: {1}. + + + + The value of {0} with identity '{1}' is invalid: '{2}'" + Hodnota {0} s identitou {1} je neplatná: {2}" + + + + The value of {0} with identity '{1}' is not a valid commit hash: '{2}'" + Hodnota {0} s identitou {1} není platná hodnota hash potvrzení: {2}" + + + + The value passed to task parameter {0} is not a valid host name: '{1}' + Hodnota předaná parametru úlohy {0} není platný název hostitele: {1}. + + + + + \ No newline at end of file diff --git a/src/SourceLink.Gitea/xlf/Resources.de.xlf b/src/SourceLink.Gitea/xlf/Resources.de.xlf new file mode 100644 index 00000000..dcc9a38f --- /dev/null +++ b/src/SourceLink.Gitea/xlf/Resources.de.xlf @@ -0,0 +1,32 @@ + + + + + + {0} item group is empty. At least one {1} repository host is required in order to generate SourceLink. + Die Elementgruppe "{0}" ist leer. Mindestens ein {1}-Repositoryhost wird benötigt, um SourceLink zu generieren. + + + + {0} specifies an invalid URL: '{1}' + "{0}" gibt eine ungültige URL an: {1} + + + + The value of {0} with identity '{1}' is invalid: '{2}'" + Der Wert von "{0}" mit der Identität "{1}" ist ungültig: "{2}" + + + + The value of {0} with identity '{1}' is not a valid commit hash: '{2}'" + Der Wert von "{0}" mit der Identität "{1}" ist kein gültiger Commithash: "{2}" + + + + The value passed to task parameter {0} is not a valid host name: '{1}' + Der an den Taskparameter "{0}" übergebene Wert ist kein gültiger Hostname: "{1}" + + + + + \ No newline at end of file diff --git a/src/SourceLink.Gitea/xlf/Resources.es.xlf b/src/SourceLink.Gitea/xlf/Resources.es.xlf new file mode 100644 index 00000000..17b78f7c --- /dev/null +++ b/src/SourceLink.Gitea/xlf/Resources.es.xlf @@ -0,0 +1,32 @@ + + + + + + {0} item group is empty. At least one {1} repository host is required in order to generate SourceLink. + El grupo de elementos {0} está vacío. Se requiere al menos un repositorio {1} a fin de generar SourceLink. + + + + {0} specifies an invalid URL: '{1}' + {0} especifica una dirección URL no válida: "{1}" + + + + The value of {0} with identity '{1}' is invalid: '{2}'" + El valor de {0} con identidad '{1}' no es válido '{2}'" + + + + The value of {0} with identity '{1}' is not a valid commit hash: '{2}'" + El valor de {0} con identidad '{1}' no es un hash de confirmación válido '{2}'" + + + + The value passed to task parameter {0} is not a valid host name: '{1}' + El valor pasado al parámetro de tarea {0} no es un nombre de host válido: "{1}" + + + + + \ No newline at end of file diff --git a/src/SourceLink.Gitea/xlf/Resources.fr.xlf b/src/SourceLink.Gitea/xlf/Resources.fr.xlf new file mode 100644 index 00000000..21751341 --- /dev/null +++ b/src/SourceLink.Gitea/xlf/Resources.fr.xlf @@ -0,0 +1,32 @@ + + + + + + {0} item group is empty. At least one {1} repository host is required in order to generate SourceLink. + Le groupe d'éléments {0} est vide. Au moins un hôte de dépôt {1} est nécessaire pour générer SourceLink. + + + + {0} specifies an invalid URL: '{1}' + {0} spécifie une URL non valide : '{1}' + + + + The value of {0} with identity '{1}' is invalid: '{2}'" + La valeur de {0} avec l'identité '{1}' n'est pas valide : '{2}'" + + + + The value of {0} with identity '{1}' is not a valid commit hash: '{2}'" + La valeur de {0} avec l'identité '{1}' n'est pas un hachage de validation valide : '{2}'" + + + + The value passed to task parameter {0} is not a valid host name: '{1}' + La valeur passée au paramètre de tâche {0} n'est pas un nom d'hôte valide : '{1}' + + + + + \ No newline at end of file diff --git a/src/SourceLink.Gitea/xlf/Resources.it.xlf b/src/SourceLink.Gitea/xlf/Resources.it.xlf new file mode 100644 index 00000000..f6f3cb05 --- /dev/null +++ b/src/SourceLink.Gitea/xlf/Resources.it.xlf @@ -0,0 +1,32 @@ + + + + + + {0} item group is empty. At least one {1} repository host is required in order to generate SourceLink. + Il gruppo di elementi {0} è vuoto. Per generare SourceLink, è necessario almeno un host di repository di {1}. + + + + {0} specifies an invalid URL: '{1}' + {0} specifica un URL non valido: '{1}' + + + + The value of {0} with identity '{1}' is invalid: '{2}'" + Il valore di {0} con identità '{1}' non è valido: '{2}'" + + + + The value of {0} with identity '{1}' is not a valid commit hash: '{2}'" + Il valore di {0} con identità '{1}' non è un hash commit valido: '{2}'" + + + + The value passed to task parameter {0} is not a valid host name: '{1}' + Il valore passato al parametro {0} dell'attività non è un nome host valido: '{1}' + + + + + \ No newline at end of file diff --git a/src/SourceLink.Gitea/xlf/Resources.ja.xlf b/src/SourceLink.Gitea/xlf/Resources.ja.xlf new file mode 100644 index 00000000..98145efd --- /dev/null +++ b/src/SourceLink.Gitea/xlf/Resources.ja.xlf @@ -0,0 +1,32 @@ + + + + + + {0} item group is empty. At least one {1} repository host is required in order to generate SourceLink. + {0} 項目グループが空です。SourceLink を生成するには、少なくとも 1 つの {1} リポジトリ ホストが必要です。 + + + + {0} specifies an invalid URL: '{1}' + {0} は無効な URL を指定しています: '{1}' + + + + The value of {0} with identity '{1}' is invalid: '{2}'" + ID '{1}' の {0} の値は無効です: '{2}'" + + + + The value of {0} with identity '{1}' is not a valid commit hash: '{2}'" + ID '{1}' の {0} の値は有効なコミット ハッシュではありません: '{2}'" + + + + The value passed to task parameter {0} is not a valid host name: '{1}' + タスク パラメーター {0} に渡された値は有効なホスト名ではありません: '{1}' + + + + + \ No newline at end of file diff --git a/src/SourceLink.Gitea/xlf/Resources.ko.xlf b/src/SourceLink.Gitea/xlf/Resources.ko.xlf new file mode 100644 index 00000000..ce5571ba --- /dev/null +++ b/src/SourceLink.Gitea/xlf/Resources.ko.xlf @@ -0,0 +1,32 @@ + + + + + + {0} item group is empty. At least one {1} repository host is required in order to generate SourceLink. + {0} 항목 그룹이 비어 있습니다. SourceLink를 생성하려면 {1} 리포지토리 호스트가 하나 이상 필요합니다. + + + + {0} specifies an invalid URL: '{1}' + {0}이(가) 잘못된 URL을 지정합니다. '{1}' + + + + The value of {0} with identity '{1}' is invalid: '{2}'" + ID가 '{1}'인 {0}의 값이 잘못되었습니다. '{2}'" + + + + The value of {0} with identity '{1}' is not a valid commit hash: '{2}'" + ID가 '{1}'인 {0}의 값이 유효한 커밋 해시가 아닙니다. '{2}'" + + + + The value passed to task parameter {0} is not a valid host name: '{1}' + 작업 매개 변수 {0}에 전달되는 값이 유효한 호스트 이름이 아닙니다. '{1}' + + + + + \ No newline at end of file diff --git a/src/SourceLink.Gitea/xlf/Resources.pl.xlf b/src/SourceLink.Gitea/xlf/Resources.pl.xlf new file mode 100644 index 00000000..5f2e9268 --- /dev/null +++ b/src/SourceLink.Gitea/xlf/Resources.pl.xlf @@ -0,0 +1,32 @@ + + + + + + {0} item group is empty. At least one {1} repository host is required in order to generate SourceLink. + Grupa elementów {0} jest pusta. Wymagany jest co najmniej jeden host repozytorium {1} w celu wygenerowania elementu SourceLink. + + + + {0} specifies an invalid URL: '{1}' + Element {0} określa nieprawidłowy adres URL: „{1}” + + + + The value of {0} with identity '{1}' is invalid: '{2}'" + Wartość elementu {0} z tożsamością „{1}” jest nieprawidłowa: „{2}”" + + + + The value of {0} with identity '{1}' is not a valid commit hash: '{2}'" + Wartość elementu {0} z tożsamością „{1}” nie jest prawidłowym skrótem zatwierdzenia: „{2}”" + + + + The value passed to task parameter {0} is not a valid host name: '{1}' + Wartość przekazana do parametru zadania {0} nie jest prawidłową nazwą hosta: „{1}” + + + + + \ No newline at end of file diff --git a/src/SourceLink.Gitea/xlf/Resources.pt-BR.xlf b/src/SourceLink.Gitea/xlf/Resources.pt-BR.xlf new file mode 100644 index 00000000..4a461578 --- /dev/null +++ b/src/SourceLink.Gitea/xlf/Resources.pt-BR.xlf @@ -0,0 +1,32 @@ + + + + + + {0} item group is empty. At least one {1} repository host is required in order to generate SourceLink. + O grupo de itens {0} está vazio. Pelo menos um host de repositório {1} é necessário para gerar o SourceLink. + + + + {0} specifies an invalid URL: '{1}' + {0} especifica uma URL inválida: '{1}' + + + + The value of {0} with identity '{1}' is invalid: '{2}'" + O valor de {0} com a identidade '{1}' é inválido: '{2}'" + + + + The value of {0} with identity '{1}' is not a valid commit hash: '{2}'" + O valor de {0} com a identidade '{1}' não é um hash de confirmação válido: '{2}'" + + + + The value passed to task parameter {0} is not a valid host name: '{1}' + O valor passado para o parâmetro de tarefa {0} não é um nome de host válido: '{1}' + + + + + \ No newline at end of file diff --git a/src/SourceLink.Gitea/xlf/Resources.ru.xlf b/src/SourceLink.Gitea/xlf/Resources.ru.xlf new file mode 100644 index 00000000..8683b5a7 --- /dev/null +++ b/src/SourceLink.Gitea/xlf/Resources.ru.xlf @@ -0,0 +1,32 @@ + + + + + + {0} item group is empty. At least one {1} repository host is required in order to generate SourceLink. + Группа элементов {0} пуста. Для создания SourceLink нужно указать хотя бы один хост репозитория {1}. + + + + {0} specifies an invalid URL: '{1}' + В {0} указан недопустимый URL-адрес: "{1}" + + + + The value of {0} with identity '{1}' is invalid: '{2}'" + Значение {0} с идентификатором "{1}" недопустимо: "{2}" + + + + The value of {0} with identity '{1}' is not a valid commit hash: '{2}'" + Значение {0} с идентификатором "{1}" не является допустимым хэш-кодом фиксации: "{2}" + + + + The value passed to task parameter {0} is not a valid host name: '{1}' + Значение, передаваемое параметру задачи {0}, не является допустимым именем узла: "{1}" + + + + + \ No newline at end of file diff --git a/src/SourceLink.Gitea/xlf/Resources.tr.xlf b/src/SourceLink.Gitea/xlf/Resources.tr.xlf new file mode 100644 index 00000000..cb9305b8 --- /dev/null +++ b/src/SourceLink.Gitea/xlf/Resources.tr.xlf @@ -0,0 +1,32 @@ + + + + + + {0} item group is empty. At least one {1} repository host is required in order to generate SourceLink. + {0} öğe grubu boş. SourceLink oluşturmak için en az bir {1} depo konağı gereklidir. + + + + {0} specifies an invalid URL: '{1}' + {0} geçersiz bir URL belirtiyor: '{1}' + + + + The value of {0} with identity '{1}' is invalid: '{2}'" + '{1}' kimliğine sahip {0} değeri geçersiz: '{2}'" + + + + The value of {0} with identity '{1}' is not a valid commit hash: '{2}'" + '{1}' kimliğine sahip {0} değeri geçerli bir işleme karması değil: '{2}'" + + + + The value passed to task parameter {0} is not a valid host name: '{1}' + {0} görev parametresine geçirilen değer geçerli bir konak adı değil: '{1}' + + + + + \ No newline at end of file diff --git a/src/SourceLink.Gitea/xlf/Resources.zh-Hans.xlf b/src/SourceLink.Gitea/xlf/Resources.zh-Hans.xlf new file mode 100644 index 00000000..e4a4010d --- /dev/null +++ b/src/SourceLink.Gitea/xlf/Resources.zh-Hans.xlf @@ -0,0 +1,32 @@ + + + + + + {0} item group is empty. At least one {1} repository host is required in order to generate SourceLink. + {0} 项目组为空。至少需要一个 {1} 存储库主机才能生成 SourceLink。 + + + + {0} specifies an invalid URL: '{1}' + {0} 指定无效的 URL:“{1}” + + + + The value of {0} with identity '{1}' is invalid: '{2}'" + 带有 '{1}' 标识的 {0} 的值无效: '{2}'" + + + + The value of {0} with identity '{1}' is not a valid commit hash: '{2}'" + 带有 '{1}' 标识的 {0} 的值不是有效的提交哈希: '{2}'" + + + + The value passed to task parameter {0} is not a valid host name: '{1}' + 传递给任务参数 {0} 的值不是有效的主机名:“{1}” + + + + + \ No newline at end of file diff --git a/src/SourceLink.Gitea/xlf/Resources.zh-Hant.xlf b/src/SourceLink.Gitea/xlf/Resources.zh-Hant.xlf new file mode 100644 index 00000000..ab6c29b8 --- /dev/null +++ b/src/SourceLink.Gitea/xlf/Resources.zh-Hant.xlf @@ -0,0 +1,32 @@ + + + + + + {0} item group is empty. At least one {1} repository host is required in order to generate SourceLink. + {0} 項目群組是空的。至少需要一個 {1} 存放庫主機,才可產生 SourceLink。 + + + + {0} specifies an invalid URL: '{1}' + {0} 指定的 URL 無效: '{1}' + + + + The value of {0} with identity '{1}' is invalid: '{2}'" + 識別為 {1} 之 {0} 的值無效: '{2}'" + + + + The value of {0} with identity '{1}' is not a valid commit hash: '{2}'" + 識別為 {1} 之 {0} 的值不是有效的認可雜湊: '{2}'" + + + + The value passed to task parameter {0} is not a valid host name: '{1}' + 傳遞給工作參數 {0} 的值並非有效的主機名稱: '{1}' + + + + + \ No newline at end of file From 5935ccd670197b059b98d299e5537ad1f10adecb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Matou=C5=A1ek?= Date: Tue, 5 Jan 2021 11:51:59 -0800 Subject: [PATCH 120/161] Add gitea to README (#686) --- README.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4b235a94..f4c6c566 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,17 @@ For projects hosted on-prem via [gitweb](https://git-scm.com/docs/gitweb) refere ```xml - + + +``` + +### gitea (pre-release) + +For projects hosted on-prem via [gitea](https://gitea.io) reference [Microsoft.SourceLink.Gitea](https://www.nuget.org/packages/Microsoft.SourceLink.Gitea) package: + +```xml + + ``` From ccdb7ce5e41f26ad4d09c2813fee2eeee0a60480 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 11 Jan 2021 13:27:29 +0000 Subject: [PATCH 121/161] Update dependencies from https://github.com/dotnet/arcade build 20210108.3 (#688) [master] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 +- eng/common/build.sh | 2 +- eng/common/cross/build-android-rootfs.sh | 2 +- eng/common/cross/build-rootfs.sh | 2 +- eng/common/darc-init.sh | 2 +- eng/common/dotnet-install.sh | 2 +- eng/common/init-tools-native.sh | 113 ++++++++++++++---- eng/common/internal-feed-operations.sh | 2 +- eng/common/msbuild.sh | 2 +- eng/common/native/install-cmake-test.sh | 6 +- eng/common/native/install-cmake.sh | 6 +- eng/common/performance/performance-setup.sh | 2 +- eng/common/pipeline-logging-functions.sh | 10 +- .../templates/job/publish-build-assets.yml | 7 ++ .../templates/job/source-index-stage1.yml | 46 +++++++ eng/common/templates/jobs/jobs.yml | 11 ++ eng/common/tools.ps1 | 6 +- eng/common/tools.sh | 31 +++-- global.json | 2 +- 19 files changed, 200 insertions(+), 58 deletions(-) create mode 100644 eng/common/templates/job/source-index-stage1.yml diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index ef3868fc..e109a776 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 4c8515c18ebe0071c32fed467ee3890fbd488898 + 1571d6b095014ad63fdb48b10f5dea912f96872e https://github.com/dotnet/source-build-reference-packages diff --git a/eng/common/build.sh b/eng/common/build.sh index 252b6360..55b298f1 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -81,7 +81,7 @@ runtime_source_feed_key='' properties='' while [[ $# > 0 ]]; do - opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" + opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" case "$opt" in -help|-h) usage diff --git a/eng/common/cross/build-android-rootfs.sh b/eng/common/cross/build-android-rootfs.sh index e7f12edb..42516bbe 100755 --- a/eng/common/cross/build-android-rootfs.sh +++ b/eng/common/cross/build-android-rootfs.sh @@ -27,7 +27,7 @@ __AndroidToolchain=aarch64-linux-android for i in "$@" do - lowerI="$(echo $i | awk '{print tolower($0)}')" + lowerI="$(echo $i | tr "[:upper:]" "[:lower:]")" case $lowerI in -?|-h|--help) usage diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 6d59e181..ae0cd1aa 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -82,7 +82,7 @@ while :; do break fi - lowerI="$(echo $1 | awk '{print tolower($0)}')" + lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" case $lowerI in -?|-h|--help) usage diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh index d981d7bb..39abdbec 100755 --- a/eng/common/darc-init.sh +++ b/eng/common/darc-init.sh @@ -6,7 +6,7 @@ versionEndpoint='https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc verbosity='minimal' while [[ $# > 0 ]]; do - opt="$(echo "$1" | awk '{print tolower($0)}')" + opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" case "$opt" in --darcversion) darcVersion=$2 diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh index ead6a1d9..25b69f87 100755 --- a/eng/common/dotnet-install.sh +++ b/eng/common/dotnet-install.sh @@ -19,7 +19,7 @@ runtime='dotnet' runtimeSourceFeed='' runtimeSourceFeedKey='' while [[ $# > 0 ]]; do - opt="$(echo "$1" | awk '{print tolower($0)}')" + opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" case "$opt" in -version|-v) shift diff --git a/eng/common/init-tools-native.sh b/eng/common/init-tools-native.sh index 29fc5db8..5bd205b5 100755 --- a/eng/common/init-tools-native.sh +++ b/eng/common/init-tools-native.sh @@ -16,7 +16,7 @@ declare -A native_assets . $scriptroot/native/common-library.sh while (($# > 0)); do - lowerI="$(echo $1 | awk '{print tolower($0)}')" + lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" case $lowerI in --baseuri) base_uri=$2 @@ -76,24 +76,89 @@ while (($# > 0)); do done function ReadGlobalJsonNativeTools { - # Get the native-tools section from the global.json. - local native_tools_section=$(cat $global_json_file | awk '/"native-tools"/,/}/') - # Only extract the contents of the object. - local native_tools_list=$(echo $native_tools_section | awk -F"[{}]" '{print $2}') - native_tools_list=${native_tools_list//[\" ]/} - native_tools_list=$( echo "$native_tools_list" | sed 's/\s//g' | sed 's/,/\n/g' ) - - local old_IFS=$IFS - while read -r line; do - # Lines are of the form: 'tool:version' - IFS=: - while read -r key value; do - native_assets[$key]=$value - done <<< "$line" - done <<< "$native_tools_list" - IFS=$old_IFS - - return 0; + # happy path: we have a proper JSON parsing tool `jq(1)` in PATH! + if command -v jq &> /dev/null; then + + # jq: read each key/value pair under "native-tools" entry and emit: + # KEY="" VALUE="" + # followed by a null byte. + # + # bash: read line with null byte delimeter and push to array (for later `eval`uation). + + while IFS= read -rd '' line; do + native_assets+=("$line") + done < <(jq -r '. | + select(has("native-tools")) | + ."native-tools" | + keys[] as $k | + @sh "KEY=\($k) VALUE=\(.[$k])\u0000"' "$global_json_file") + + return + fi + + # Warning: falling back to manually parsing JSON, which is not recommended. + + # Following routine matches the output and escaping logic of jq(1)'s @sh formatter used above. + # It has been tested with several weird strings with escaped characters in entries (key and value) + # and results were compared with the output of jq(1) in binary representation using xxd(1); + # just before the assignment to 'native_assets' array (above and below). + + # try to capture the section under "native-tools". + if [[ ! "$(cat "$global_json_file")" =~ \"native-tools\"[[:space:]\:\{]*([^\}]+) ]]; then + return + fi + + section="${BASH_REMATCH[1]}" + + parseStarted=0 + possibleEnd=0 + escaping=0 + escaped=0 + isKey=1 + + for (( i=0; i<${#section}; i++ )); do + char="${section:$i:1}" + if ! ((parseStarted)) && [[ "$char" =~ [[:space:],:] ]]; then continue; fi + + if ! ((escaping)) && [[ "$char" == "\\" ]]; then + escaping=1 + elif ((escaping)) && ! ((escaped)); then + escaped=1 + fi + + if ! ((parseStarted)) && [[ "$char" == "\"" ]]; then + parseStarted=1 + possibleEnd=0 + elif [[ "$char" == "'" ]]; then + token="$token'\\\''" + possibleEnd=0 + elif ((escaping)) || [[ "$char" != "\"" ]]; then + token="$token$char" + possibleEnd=1 + fi + + if ((possibleEnd)) && ! ((escaping)) && [[ "$char" == "\"" ]]; then + # Use printf to unescape token to match jq(1)'s @sh formatting rules. + # do not use 'token="$(printf "$token")"' syntax, as $() eats the trailing linefeed. + printf -v token "'$token'" + + if ((isKey)); then + KEY="$token" + isKey=0 + else + line="KEY=$KEY VALUE=$token" + native_assets+=("$line") + isKey=1 + fi + + # reset for next token + parseStarted=0 + token= + elif ((escaping)) && ((escaped)); then + escaping=0 + escaped=0 + fi + done } native_base_dir=$install_directory @@ -111,14 +176,14 @@ if [[ ${#native_assets[@]} -eq 0 ]]; then exit 0; else native_installer_dir="$scriptroot/native" - for tool in "${!native_assets[@]}" - do - tool_version=${native_assets[$tool]} - installer_path="$native_installer_dir/install-$tool.sh" + for index in "${!native_assets[@]}"; do + eval "${native_assets["$index"]}" + + installer_path="$native_installer_dir/install-$KEY.sh" installer_command="$installer_path" installer_command+=" --baseuri $base_uri" installer_command+=" --installpath $install_bin" - installer_command+=" --version $tool_version" + installer_command+=" --version $VALUE" echo $installer_command if [[ $force = true ]]; then diff --git a/eng/common/internal-feed-operations.sh b/eng/common/internal-feed-operations.sh index 9ed225e7..a27410f6 100755 --- a/eng/common/internal-feed-operations.sh +++ b/eng/common/internal-feed-operations.sh @@ -103,7 +103,7 @@ authToken='' repoName='' while [[ $# > 0 ]]; do - opt="$(echo "$1" | awk '{print tolower($0)}')" + opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" case "$opt" in --operation) operation=$2 diff --git a/eng/common/msbuild.sh b/eng/common/msbuild.sh index 8160cd5a..20d3dad5 100755 --- a/eng/common/msbuild.sh +++ b/eng/common/msbuild.sh @@ -19,7 +19,7 @@ prepare_machine=false extra_args='' while (($# > 0)); do - lowerI="$(echo $1 | awk '{print tolower($0)}')" + lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" case $lowerI in --verbosity) verbosity=$2 diff --git a/eng/common/native/install-cmake-test.sh b/eng/common/native/install-cmake-test.sh index 12339a40..8a5e7cf0 100755 --- a/eng/common/native/install-cmake-test.sh +++ b/eng/common/native/install-cmake-test.sh @@ -14,7 +14,7 @@ download_retries=5 retry_wait_time_seconds=30 while (($# > 0)); do - lowerI="$(echo $1 | awk '{print tolower($0)}')" + lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" case $lowerI in --baseuri) base_uri=$2 @@ -63,7 +63,7 @@ done tool_name="cmake-test" tool_os=$(GetCurrentOS) -tool_folder=$(echo $tool_os | awk '{print tolower($0)}') +tool_folder="$(echo $tool_os | tr "[:upper:]" "[:lower:]")" tool_arch="x86_64" tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch" tool_install_directory="$install_path/$tool_name/$version" @@ -114,4 +114,4 @@ if [[ $? != 0 ]]; then exit 1 fi -exit 0 \ No newline at end of file +exit 0 diff --git a/eng/common/native/install-cmake.sh b/eng/common/native/install-cmake.sh index 18041be8..de496bee 100755 --- a/eng/common/native/install-cmake.sh +++ b/eng/common/native/install-cmake.sh @@ -14,7 +14,7 @@ download_retries=5 retry_wait_time_seconds=30 while (($# > 0)); do - lowerI="$(echo $1 | awk '{print tolower($0)}')" + lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" case $lowerI in --baseuri) base_uri=$2 @@ -63,7 +63,7 @@ done tool_name="cmake" tool_os=$(GetCurrentOS) -tool_folder=$(echo $tool_os | awk '{print tolower($0)}') +tool_folder="$(echo $tool_os | tr "[:upper:]" "[:lower:]")" tool_arch="x86_64" tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch" tool_install_directory="$install_path/$tool_name/$version" @@ -114,4 +114,4 @@ if [[ $? != 0 ]]; then exit 1 fi -exit 0 \ No newline at end of file +exit 0 diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index c8e211bc..7321feb5 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -29,7 +29,7 @@ using_wasm=false use_latest_dotnet=false while (($# > 0)); do - lowerI="$(echo $1 | awk '{print tolower($0)}')" + lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" case $lowerI in --sourcedirectory) source_directory=$2 diff --git a/eng/common/pipeline-logging-functions.sh b/eng/common/pipeline-logging-functions.sh index da5a7e61..6a0b2255 100755 --- a/eng/common/pipeline-logging-functions.sh +++ b/eng/common/pipeline-logging-functions.sh @@ -6,7 +6,7 @@ function Write-PipelineTelemetryError { local function_args=() local message='' while [[ $# -gt 0 ]]; do - opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" + opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" case "$opt" in -category|-c) telemetry_category=$2 @@ -48,7 +48,7 @@ function Write-PipelineTaskError { local force=false while [[ $# -gt 0 ]]; do - opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" + opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" case "$opt" in -type|-t) message_type=$2 @@ -122,7 +122,7 @@ function Write-PipelineSetVariable { local is_multi_job_variable=true while [[ $# -gt 0 ]]; do - opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" + opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" case "$opt" in -name|-n) name=$2 @@ -164,7 +164,7 @@ function Write-PipelinePrependPath { local prepend_path='' while [[ $# -gt 0 ]]; do - opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" + opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" case "$opt" in -path|-p) prepend_path=$2 @@ -186,7 +186,7 @@ function Write-PipelineSetResult { local message='' while [[ $# -gt 0 ]]; do - opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" + opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" case "$opt" in -result|-r) result=$2 diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index d0c3cc2b..66ef7364 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -37,6 +37,7 @@ jobs: - name: _BuildConfig value: ${{ parameters.configuration }} - group: Publish-Build-Assets + - group: AzureDevOps-Artifact-Feeds-Pats # Skip component governance and codesign validation for SDL. These jobs # create no content. - name: skipComponentGovernanceDetection @@ -57,6 +58,12 @@ jobs: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - task: NuGetAuthenticate@0 + - task: PowerShell@2 + displayName: Enable cross-org NuGet feed authentication + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/enable-cross-org-publishing.ps1 + arguments: -token $(dn-bot-all-orgs-artifact-feeds-rw) + - task: PowerShell@2 displayName: Publish Build Assets inputs: diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml new file mode 100644 index 00000000..b33f06b9 --- /dev/null +++ b/eng/common/templates/job/source-index-stage1.yml @@ -0,0 +1,46 @@ +parameters: + runAsPublic: false + sourceIndexPackageVersion: 1.0.0-beta5 + sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json + sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" + binlogPath: artifacts/log/Debug/Build.binlog + pool: + vmImage: vs2017-win2016 + +jobs: +- job: SourceIndexStage1 + variables: + - name: SourceIndexPackageVersion + value: ${{ parameters.sourceIndexPackageVersion }} + - name: SourceIndexPackageSource + value: ${{ parameters.sourceIndexPackageSource }} + - name: BinlogPath + value: ${{ parameters.binlogPath }} + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: source-dot-net stage1 variables + + pool: ${{ parameters.pool }} + steps: + - task: UseDotNet@2 + displayName: Use .NET Core sdk 3.1 + inputs: + packageType: sdk + version: 3.1.x + + - script: | + dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path .source-index/tools + dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path .source-index/tools + echo ##vso[task.prependpath]$(Build.SourcesDirectory)/.source-index/tools + displayName: Download Tools + + - script: ${{ parameters.sourceIndexBuildCommand }} + displayName: Build Repository + + - script: BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output + displayName: Process Binlog into indexable sln + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - script: UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) + displayName: Upload stage1 artifacts to source index + env: + BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml index 08845950..79ffd6e6 100644 --- a/eng/common/templates/jobs/jobs.yml +++ b/eng/common/templates/jobs/jobs.yml @@ -31,6 +31,9 @@ parameters: # See /eng/common/templates/jobs/source-build.yml for options sourceBuildParameters: [] + enableSourceIndex: false + sourceIndexParams: {} + # Internal resources (telemetry, microbuild) can only be accessed from non-public projects, # and some (Microbuild) should only be applied to non-PR cases for internal builds. @@ -57,7 +60,15 @@ jobs: ${{ each parameter in parameters.sourceBuildParameters }}: ${{ parameter.key }}: ${{ parameter.value }} +- ${{ if eq(parameters.enableSourceIndex, 'true') }}: + - template: ../job/source-index-stage1.yml + parameters: + runAsPublic: ${{ parameters.runAsPublic }} + ${{ each parameter in parameters.sourceIndexParams }}: + ${{ parameter.key }}: ${{ parameter.value }} + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: - template: ../job/publish-build-assets.yml parameters: diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index ce280eb9..10e98593 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -645,7 +645,11 @@ function MSBuild() { $toolsetBuildProject = InitializeToolset $path = Split-Path -parent $toolsetBuildProject - $path = Join-Path $path (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll') + $path = Join-Path $path (Join-Path $buildTool.Framework 'Microsoft.DotNet.ArcadeLogging.dll') + if (-not (Test-Path $path)) { + $path = Split-Path -parent $toolsetBuildProject + $path = Join-Path $path (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll') + } $args += "/logger:$path" } diff --git a/eng/common/tools.sh b/eng/common/tools.sh index b160c370..0920f596 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -89,16 +89,16 @@ function ResolvePath { function ReadGlobalVersion { local key=$1 - local line=$(awk "/$key/ {print; exit}" "$global_json_file") - local pattern="\"$key\" *: *\"(.*)\"" + if command -v jq &> /dev/null; then + _ReadGlobalVersion="$(jq -r ".[] | select(has(\"$key\")) | .\"$key\"" "$global_json_file")" + elif [[ "$(cat "$global_json_file")" =~ \"$key\"[[:space:]\:]*\"([^\"]+) ]]; then + _ReadGlobalVersion=${BASH_REMATCH[1]} + fi - if [[ ! $line =~ $pattern ]]; then + if [[ -z "$_ReadGlobalVersion" ]]; then Write-PipelineTelemetryError -category 'Build' "Error: Cannot find \"$key\" in $global_json_file" ExitWithExitCode 1 fi - - # return value - _ReadGlobalVersion=${BASH_REMATCH[1]} } function InitializeDotNetCli { @@ -273,8 +273,11 @@ function GetDotNetInstallScript { if command -v curl > /dev/null; then # first, try directly, if this fails we will retry with verbose logging curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || { - echo "Curl failed; dumping some information about dotnet.microsoft.com for later investigation" - echo | openssl s_client -showcerts -servername dotnet.microsoft.com -connect dotnet.microsoft.com:443 + if command -v openssl &> /dev/null + then + echo "Curl failed; dumping some information about dotnet.microsoft.com for later investigation" + echo | openssl s_client -showcerts -servername dotnet.microsoft.com -connect dotnet.microsoft.com:443 + fi echo "Will now retry the same URL with verbose logging." with_retries curl "$install_script_url" -sSL --verbose --retry 10 --create-dirs -o "$install_script" || { local exit_code=$? @@ -411,7 +414,10 @@ function MSBuild { fi local toolset_dir="${_InitializeToolset%/*}" - local logger_path="$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" + local logger_path="$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.ArcadeLogging.dll" + if [[ ! -f $logger_path ]]; then + logger_path="$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" + fi args=( "${args[@]}" "-logger:$logger_path" ) fi @@ -476,8 +482,11 @@ temp_dir="$artifacts_dir/tmp/$configuration" global_json_file="$repo_root/global.json" # determine if global.json contains a "runtimes" entry global_json_has_runtimes=false -dotnetlocal_key=$(awk "/runtimes/ {print; exit}" "$global_json_file") || true -if [[ -n "$dotnetlocal_key" ]]; then +if command -v jq &> /dev/null; then + if jq -er '. | select(has("runtimes"))' "$global_json_file" &> /dev/null; then + global_json_has_runtimes=true + fi +elif [[ "$(cat "$global_json_file")" =~ \"runtimes\"[[:space:]\:]*\{ ]]; then global_json_has_runtimes=true fi diff --git a/global.json b/global.json index b13e4fb6..feb516af 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "5.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.20630.2" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21058.3" } } From 650f95502cca1a6831146e52fef43b004ff24ddb Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 18 Jan 2021 13:42:42 +0000 Subject: [PATCH 122/161] Update dependencies from https://github.com/dotnet/arcade build 20210112.10 (#693) [master] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- eng/common/templates/job/source-index-stage1.yml | 12 ++++++++++++ eng/common/templates/post-build/post-build.yml | 2 +- global.json | 2 +- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index e109a776..e85b9424 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 1571d6b095014ad63fdb48b10f5dea912f96872e + ff8fec9066eca51a26abf723dd7d92e20926a90e https://github.com/dotnet/source-build-reference-packages diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml index b33f06b9..61263307 100644 --- a/eng/common/templates/job/source-index-stage1.yml +++ b/eng/common/templates/job/source-index-stage1.yml @@ -3,6 +3,7 @@ parameters: sourceIndexPackageVersion: 1.0.0-beta5 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" + preSteps: [] binlogPath: artifacts/log/Debug/Build.binlog pool: vmImage: vs2017-win2016 @@ -21,12 +22,20 @@ jobs: pool: ${{ parameters.pool }} steps: + - ${{ each preStep in parameters.preSteps }}: + - ${{ preStep }} + - task: UseDotNet@2 displayName: Use .NET Core sdk 3.1 inputs: packageType: sdk version: 3.1.x + - task: UseDotNet@2 + displayName: Use .NET Core sdk + inputs: + useGlobalJson: true + - script: | dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path .source-index/tools dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path .source-index/tools @@ -38,9 +47,12 @@ jobs: - script: BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output displayName: Process Binlog into indexable sln + env: + DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX: 2 - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - script: UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) displayName: Upload stage1 artifacts to source index env: BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) + DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX: 2 diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 41f2d96a..bd776c76 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -224,7 +224,7 @@ stages: - ${{ if or(ge(parameters.publishingInfraVersion, 3), eq(parameters.inline, 'false')) }}: - stage: publish_using_darc ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: - dependsOn: Validate + dependsOn: ${{ parameters.publishDependsOn }} ${{ if and(ne(parameters.enableNugetValidation, 'true'), ne(parameters.enableSigningValidation, 'true'), ne(parameters.enableSourceLinkValidation, 'true'), ne(parameters.SDLValidationParameters.enable, 'true')) }}: dependsOn: ${{ parameters.validateDependsOn }} displayName: Publish using Darc diff --git a/global.json b/global.json index feb516af..9eb4c0bf 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "5.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21058.3" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21062.10" } } From 0f3cb283e81fbdc930f0c7f7e52e5bc396d06a86 Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Tue, 19 Jan 2021 15:56:43 -0600 Subject: [PATCH 123/161] Enable source-build with Arcade SDK fixes (#692) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Enable source build * Update dependencies from https://github.com/dotnet/arcade build 20210119.2 Co-authored-by: Tomáš Matoušek --- azure-pipelines.yml | 1 + eng/Version.Details.xml | 4 ++-- eng/common/post-build/publish-using-darc.ps1 | 4 ++-- eng/common/templates/job/source-build.yml | 8 ++++++++ eng/common/templates/jobs/jobs.yml | 20 +++++++++---------- eng/common/templates/jobs/source-build.yml | 12 +++++------ .../templates/post-build/post-build.yml | 3 +++ global.json | 2 +- 8 files changed, 32 insertions(+), 22 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7bd4424f..b67505a9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -37,6 +37,7 @@ stages: enablePublishBuildAssets: true enablePublishUsingPipelines: $(_PublishUsingPipelines) enableTelemetry: true + enableSourceBuild: true helixRepo: dotnet/symreader jobs: - job: Windows diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index e85b9424..d513c886 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - ff8fec9066eca51a26abf723dd7d92e20926a90e + ee0a6a7d9acc6f5886ae214b935f3e3d989235e4 https://github.com/dotnet/source-build-reference-packages diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1 index 6b68ee84..599a1172 100644 --- a/eng/common/post-build/publish-using-darc.ps1 +++ b/eng/common/post-build/publish-using-darc.ps1 @@ -16,8 +16,8 @@ param( try { . $PSScriptRoot\post-build-utils.ps1 - # Hard coding darc version till the next arcade-services roll out, cos this version has required API changes for darc add-build-to-channel - $darc = Get-Darc "1.1.0-beta.20418.1" + + $darc = Get-Darc $optionalParams = [System.Collections.ArrayList]::new() diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml index 9332f5ec..aad41464 100644 --- a/eng/common/templates/job/source-build.yml +++ b/eng/common/templates/job/source-build.yml @@ -28,6 +28,11 @@ parameters: # container and pool. platform: {} + # The default VM host AzDO pool. This should be capable of running Docker containers: almost all + # source-build builds run in Docker, including the default managed platform. + defaultContainerHostPool: + vmImage: ubuntu-20.04 + jobs: - job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} displayName: Source-Build (${{ parameters.platform.name }}) @@ -37,6 +42,9 @@ jobs: ${{ if ne(parameters.platform.container, '') }}: container: ${{ parameters.platform.container }} + + ${{ if eq(parameters.platform.pool, '') }}: + pool: ${{ parameters.defaultContainerHostPool }} ${{ if ne(parameters.platform.pool, '') }}: pool: ${{ parameters.platform.pool }} diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml index 79ffd6e6..a1f8fce9 100644 --- a/eng/common/templates/jobs/jobs.yml +++ b/eng/common/templates/jobs/jobs.yml @@ -7,7 +7,14 @@ parameters: # Optional: Enable publishing using release pipelines enablePublishUsingPipelines: false - + + # Optional: Enable running the source-build jobs to build repo from source + enableSourceBuild: false + + # Optional: Parameters for source-build template. + # See /eng/common/templates/jobs/source-build.yml for options + sourceBuildParameters: [] + graphFileGeneration: # Optional: Enable generating the graph files at the end of the build enabled: false @@ -24,13 +31,6 @@ parameters: # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. runAsPublic: false - # Optional: Enable running the source-build jobs to build repo from source - runSourceBuild: false - - # Optional: Parameters for source-build template. - # See /eng/common/templates/jobs/source-build.yml for options - sourceBuildParameters: [] - enableSourceIndex: false sourceIndexParams: {} @@ -53,7 +53,7 @@ jobs: name: ${{ job.job }} -- ${{ if eq(parameters.runSourceBuild, true) }}: +- ${{ if eq(parameters.enableSourceBuild, true) }}: - template: /eng/common/templates/jobs/source-build.yml parameters: allCompletedJobId: Source_Build_Complete @@ -80,7 +80,7 @@ jobs: - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: - ${{ each job in parameters.jobs }}: - ${{ job.job }} - - ${{ if eq(parameters.runSourceBuild, true) }}: + - ${{ if eq(parameters.enableSourceBuild, true) }}: - Source_Build_Complete pool: vmImage: vs2017-win2016 diff --git a/eng/common/templates/jobs/source-build.yml b/eng/common/templates/jobs/source-build.yml index f463011e..00aa98eb 100644 --- a/eng/common/templates/jobs/source-build.yml +++ b/eng/common/templates/jobs/source-build.yml @@ -11,16 +11,14 @@ parameters: # See /eng/common/templates/job/source-build.yml jobNamePrefix: 'Source_Build' - # If changed to true, causes this template to include the default platform for a managed-only - # repo. The exact Docker image used for this build will be provided by Arcade. This has some risk, - # but since the repo is supposed to be managed-only, the risk should be very low. - includeDefaultManagedPlatform: false + # This is the default platform provided by Arcade, intended for use by a managed-only repo. defaultManagedPlatform: name: 'Managed' container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-3e800f1-20190501005343' # Defines the platforms on which to run build jobs. One job is created for each platform, and the - # object in this array is sent to the job template as 'platform'. + # object in this array is sent to the job template as 'platform'. If no platforms are specified, + # one job runs on 'defaultManagedPlatform'. platforms: [] jobs: @@ -32,7 +30,7 @@ jobs: dependsOn: - ${{ each platform in parameters.platforms }}: - ${{ parameters.jobNamePrefix }}_${{ platform.name }} - - ${{ if eq(parameters.includeDefaultManagedPlatform, true) }}: + - ${{ if eq(length(parameters.platforms), 0) }}: - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }} - ${{ each platform in parameters.platforms }}: @@ -41,7 +39,7 @@ jobs: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ platform }} -- ${{ if eq(parameters.includeDefaultManagedPlatform, true) }}: +- ${{ if eq(length(parameters.platforms), 0) }}: - template: /eng/common/templates/job/source-build.yml parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index bd776c76..375e91ac 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -147,6 +147,9 @@ stages: pipeline: $(AzDOPipelineId) buildId: $(AzDOBuildId) artifactName: PackageArtifacts + itemPattern: | + ** + !**/Microsoft.SourceBuild.Intermediate.*.nupkg # This is necessary whenever we want to publish/restore to an AzDO private feed # Since sdk-task.ps1 tries to restore packages we need to do this authentication here diff --git a/global.json b/global.json index 9eb4c0bf..4a467f57 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "5.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21062.10" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21069.2" } } From dfe619dc722be42d475595c755c958afe6177554 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 25 Jan 2021 15:53:57 +0000 Subject: [PATCH 124/161] Update dependencies from https://github.com/dotnet/arcade build 20210121.2 (#697) [master] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index d513c886..46841213 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - ee0a6a7d9acc6f5886ae214b935f3e3d989235e4 + 9a437ab126433012dd2e479c5edfb552b83e872b https://github.com/dotnet/source-build-reference-packages diff --git a/global.json b/global.json index 4a467f57..d479fe48 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "5.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21069.2" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21071.2" } } From ca11cbd5e7fac44ae5a7228f5cd89d9df99dd90e Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 1 Feb 2021 13:27:02 +0000 Subject: [PATCH 125/161] Update dependencies from https://github.com/dotnet/arcade build 20210128.12 (#698) [master] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- eng/common/sdl/init-sdl.ps1 | 13 +------------ eng/common/sdl/packages.config | 2 +- eng/common/templates/job/execute-sdl.yml | 2 +- eng/common/templates/post-build/post-build.yml | 18 +++++++++++++++++- global.json | 2 +- 6 files changed, 23 insertions(+), 18 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 46841213..7f15558b 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 9a437ab126433012dd2e479c5edfb552b83e872b + 3233c41c837f72827efd8f827b538e047334847d https://github.com/dotnet/source-build-reference-packages diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1 index a68bf0b8..bb6a4297 100644 --- a/eng/common/sdl/init-sdl.ps1 +++ b/eng/common/sdl/init-sdl.ps1 @@ -29,18 +29,7 @@ $zipFile = "$WorkingDirectory/gdn.zip" Add-Type -AssemblyName System.IO.Compression.FileSystem $gdnFolder = (Join-Path $WorkingDirectory '.gdn') -try { - # We try to download the zip; if the request fails (e.g. the file doesn't exist), we catch it and init guardian instead - Write-Host 'Downloading gdn folder from internal config repostiory...' - Invoke-WebRequest -Headers @{ "Accept"="application/zip"; "Authorization"="Basic $encodedPat" } -Uri $uri -OutFile $zipFile - if (Test-Path $gdnFolder) { - # Remove the gdn folder if it exists (it shouldn't unless there's too much caching; this is just in case) - Remove-Item -Force -Recurse $gdnFolder - } - [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFile, $WorkingDirectory) - Write-Host $gdnFolder - ExitWithExitCode 0 -} catch [System.Net.WebException] { } # Catch and ignore webexception + try { # if the folder does not exist, we'll do a guardian init and push it to the remote repository Write-Host 'Initializing Guardian...' diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config index 968b39be..3bd8b29e 100644 --- a/eng/common/sdl/packages.config +++ b/eng/common/sdl/packages.config @@ -1,4 +1,4 @@ - + diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index c64c4f56..53c10022 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -83,7 +83,7 @@ jobs: continueOnError: ${{ parameters.sdlContinueOnError }} - ${{ if eq(parameters.overrideParameters, '') }}: - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 - -GuardianPackageName Microsoft.Guardian.Cli.win10-x64.0.20.1 + -GuardianPackageName Microsoft.Guardian.Cli.0.53.3 -NugetPackageDirectory $(Build.SourcesDirectory)\.packages -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) ${{ parameters.additionalParameters }} diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 375e91ac..b9aa5b60 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -61,6 +61,7 @@ parameters: VS167ChannelId: 1011 VS168ChannelId: 1154 VSMasterChannelId: 1012 + VS169ChannelId: 1473 stages: - ${{ if or(and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')), eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: @@ -90,7 +91,7 @@ stages: inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 arguments: -PromoteToChannels "$(TargetChannels)" - -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}} + -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}},${{parameters.VS169ChannelId}} - job: displayName: NuGet Validation @@ -552,3 +553,18 @@ stages: transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'VS_16_9_Publishing' + channelName: 'VS 16.9' + channelId: ${{ parameters.VS169ChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' diff --git a/global.json b/global.json index d479fe48..17fcd3a6 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "5.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21071.2" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21078.12" } } From 1145841dc46cd37dfb8f4438a577e94f0fb8ecd0 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 8 Feb 2021 13:37:24 +0000 Subject: [PATCH 126/161] Update dependencies from https://github.com/dotnet/arcade build 20210205.5 (#700) [master] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- eng/common/dotnet-install.sh | 9 ++------- eng/common/internal-feed-operations.ps1 | 2 -- eng/common/internal-feed-operations.sh | 2 -- .../templates/post-build/post-build.yml | 20 +++++++++++++++++-- global.json | 2 +- 6 files changed, 23 insertions(+), 16 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 7f15558b..f8899e9d 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 3233c41c837f72827efd8f827b538e047334847d + fc83e59329203724d4a63c4f6c843be62983a35e https://github.com/dotnet/source-build-reference-packages diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh index 25b69f87..d6efeb44 100755 --- a/eng/common/dotnet-install.sh +++ b/eng/common/dotnet-install.sh @@ -49,13 +49,8 @@ while [[ $# > 0 ]]; do shift done -# Use uname to determine what the CPU is. -cpuname=$(uname -p) -# Some Linux platforms report unknown for platform, but the arch for machine. -if [[ "$cpuname" == "unknown" ]]; then - cpuname=$(uname -m) -fi - +# Use uname to determine what the CPU is, see https://en.wikipedia.org/wiki/Uname#Examples +cpuname=$(uname -m) case $cpuname in aarch64) buildarch=arm64 diff --git a/eng/common/internal-feed-operations.ps1 b/eng/common/internal-feed-operations.ps1 index b8f6529f..418c0993 100644 --- a/eng/common/internal-feed-operations.ps1 +++ b/eng/common/internal-feed-operations.ps1 @@ -63,8 +63,6 @@ function SetupCredProvider { } if (($endpoints | Measure-Object).Count -gt 0) { - # [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Endpoint code example with no real credentials.")] - # Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}' $endpointCredentials = @{endpointCredentials=$endpoints} | ConvertTo-Json -Compress # Create the environment variables the AzDo way diff --git a/eng/common/internal-feed-operations.sh b/eng/common/internal-feed-operations.sh index a27410f6..e2233e78 100755 --- a/eng/common/internal-feed-operations.sh +++ b/eng/common/internal-feed-operations.sh @@ -62,8 +62,6 @@ function SetupCredProvider { endpoints+=']' if [ ${#endpoints} -gt 2 ]; then - # [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Endpoint code example with no real credentials.")] - # Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}' local endpointCredentials="{\"endpointCredentials\": "$endpoints"}" echo "##vso[task.setvariable variable=VSS_NUGET_EXTERNAL_FEED_ENDPOINTS]$endpointCredentials" diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index b9aa5b60..822ff597 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -62,7 +62,8 @@ parameters: VS168ChannelId: 1154 VSMasterChannelId: 1012 VS169ChannelId: 1473 - + VS1610ChannelId: 1692 + stages: - ${{ if or(and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')), eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: - stage: Validate @@ -91,7 +92,7 @@ stages: inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 arguments: -PromoteToChannels "$(TargetChannels)" - -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}},${{parameters.VS169ChannelId}} + -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}},${{parameters.VS169ChannelId}},${{parameters.VS1610ChannelId}} - job: displayName: NuGet Validation @@ -568,3 +569,18 @@ stages: transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'VS_16_10_Publishing' + channelName: 'VS 16.10' + channelId: ${{ parameters.VS1610ChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' diff --git a/global.json b/global.json index 17fcd3a6..e202b867 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "5.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21078.12" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21105.5" } } From 92f1252a59eb67391c1883a5c39920a385157f1a Mon Sep 17 00:00:00 2001 From: dotnet bot Date: Tue, 9 Feb 2021 15:00:01 -0800 Subject: [PATCH 127/161] Fixup feeds (#701) Co-authored-by: Matt Mitchell --- NuGet.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NuGet.config b/NuGet.config index 502f9feb..90ff4da1 100644 --- a/NuGet.config +++ b/NuGet.config @@ -2,7 +2,7 @@ - + From 6cef6be0fcc782dbd4f869ce0b730e87409a7175 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 15 Feb 2021 13:22:09 +0000 Subject: [PATCH 128/161] Update dependencies from https://github.com/dotnet/arcade build 20210205.12 (#703) [master] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- eng/common/tools.ps1 | 27 ++++++++++++++++++++------- eng/common/tools.sh | 23 ++++++++++++++++++----- global.json | 2 +- 4 files changed, 41 insertions(+), 15 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index f8899e9d..d3dcc6dc 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - fc83e59329203724d4a63c4f6c843be62983a35e + 938b3e8b4edcd96ca0f0cbbae63c87b3f51f7afe https://github.com/dotnet/source-build-reference-packages diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 10e98593..572da3b9 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -508,7 +508,7 @@ function InitializeBuildTool() { ExitWithExitCode 1 } $dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet') - $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'netcoreapp2.1' } + $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'netcoreapp3.1' } } elseif ($msbuildEngine -eq "vs") { try { $msbuildPath = InitializeVisualStudioMSBuild -install:$restore @@ -644,13 +644,26 @@ function MSBuild() { } $toolsetBuildProject = InitializeToolset - $path = Split-Path -parent $toolsetBuildProject - $path = Join-Path $path (Join-Path $buildTool.Framework 'Microsoft.DotNet.ArcadeLogging.dll') - if (-not (Test-Path $path)) { - $path = Split-Path -parent $toolsetBuildProject - $path = Join-Path $path (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll') + $basePath = Split-Path -parent $toolsetBuildProject + $possiblePaths = @( + # new scripts need to work with old packages, so we need to look for the old names/versions + (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.ArcadeLogging.dll')), + (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll')), + (Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.ArcadeLogging.dll')), + (Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.Arcade.Sdk.dll')) + ) + $selectedPath = $null + foreach ($path in $possiblePaths) { + if (Test-Path $path -PathType Leaf) { + $selectedPath = $path + break + } + } + if (-not $selectedPath) { + Write-PipelineTelemetryError -Category 'Build' -Message 'Unable to find arcade sdk logger assembly.' + ExitWithExitCode 1 } - $args += "/logger:$path" + $args += "/logger:$selectedPath" } MSBuild-Core @args diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 0920f596..9019b7f5 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -307,7 +307,7 @@ function InitializeBuildTool { # return values _InitializeBuildTool="$_InitializeDotNetCli/dotnet" _InitializeBuildToolCommand="msbuild" - _InitializeBuildToolFramework="netcoreapp2.1" + _InitializeBuildToolFramework="netcoreapp3.1" } # Set RestoreNoCache as a workaround for https://github.com/NuGet/Home/issues/3116 @@ -414,11 +414,24 @@ function MSBuild { fi local toolset_dir="${_InitializeToolset%/*}" - local logger_path="$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.ArcadeLogging.dll" - if [[ ! -f $logger_path ]]; then - logger_path="$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" + # new scripts need to work with old packages, so we need to look for the old names/versions + local selectedPath= + local possiblePaths=() + possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.ArcadeLogging.dll" ) + possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" ) + possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.ArcadeLogging.dll" ) + possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.Arcade.Sdk.dll" ) + for path in "${possiblePaths[@]}"; do + if [[ -f $path ]]; then + selectedPath=$path + break + fi + done + if [[ -z "$selectedPath" ]]; then + Write-PipelineTelemetryError -category 'Build' "Unable to find arcade sdk logger assembly." + ExitWithExitCode 1 fi - args=( "${args[@]}" "-logger:$logger_path" ) + args+=( "-logger:$selectedPath" ) fi MSBuild-Core ${args[@]} diff --git a/global.json b/global.json index e202b867..a4eb4120 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "5.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21105.5" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21105.12" } } From 99defc1944a90ef96ec966cb378780caf18c3922 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 1 Mar 2021 13:27:10 +0000 Subject: [PATCH 129/161] Update dependencies from https://github.com/dotnet/arcade build 20210225.5 (#704) [master] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 +- eng/common/cross/build-rootfs.sh | 36 +++++++--- eng/common/performance/performance-setup.ps1 | 8 --- eng/common/performance/performance-setup.sh | 23 ++++--- eng/common/sdl/execute-all-sdl-tools.ps1 | 4 -- eng/common/sdl/init-sdl.ps1 | 1 - eng/common/sdl/push-gdn.ps1 | 69 ------------------- .../templates/job/source-index-stage1.yml | 2 +- .../templates/post-build/post-build.yml | 2 +- eng/common/tools.ps1 | 2 +- eng/common/tools.sh | 3 +- global.json | 2 +- 12 files changed, 45 insertions(+), 111 deletions(-) delete mode 100644 eng/common/sdl/push-gdn.ps1 diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index d3dcc6dc..3dad9464 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 938b3e8b4edcd96ca0f0cbbae63c87b3f51f7afe + 15246f4af00a1cb2e580783d32ec2937b1878a64 https://github.com/dotnet/source-build-reference-packages diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index ae0cd1aa..b2662244 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -6,7 +6,7 @@ usage() { echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [--skipunmount] --rootfsdir ]" echo "BuildArch can be: arm(default), armel, arm64, x86" - echo "CodeName - optional, Code name for Linux, can be: trusty, xenial(default), zesty, bionic, alpine. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." + echo "CodeName - optional, Code name for Linux, can be: trusty, xenial(default), zesty, bionic, alpine, alpine3.9 or alpine3.13. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." echo " for FreeBSD can be: freebsd11 or freebsd12." echo " for illumos can be: illumos." echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FReeBSD" @@ -183,9 +183,20 @@ while :; do __UbuntuRepo= __Tizen=tizen ;; - alpine) + alpine|alpine3.9) __CodeName=alpine __UbuntuRepo= + __AlpineVersion=3.9 + ;; + alpine3.13) + __CodeName=alpine + __UbuntuRepo= + __AlpineVersion=3.13 + # Alpine 3.13 has all the packages we need in the 3.13 repository + __AlpinePackages+=$__AlpinePackagesEdgeCommunity + __AlpinePackagesEdgeCommunity= + __AlpinePackages+=$__AlpinePackagesEdgeMain + __AlpinePackagesEdgeMain= ;; freebsd11) __FreeBSDBase="11.3-RELEASE" @@ -243,7 +254,6 @@ __RootfsDir="$( cd "$__RootfsDir" && pwd )" if [[ "$__CodeName" == "alpine" ]]; then __ApkToolsVersion=2.9.1 - __AlpineVersion=3.9 __ApkToolsDir=$(mktemp -d) wget https://github.com/alpinelinux/apk-tools/releases/download/v$__ApkToolsVersion/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -P $__ApkToolsDir tar -xf $__ApkToolsDir/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -C $__ApkToolsDir @@ -256,15 +266,19 @@ if [[ "$__CodeName" == "alpine" ]]; then -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ add $__AlpinePackages - $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \ - -X http://dl-cdn.alpinelinux.org/alpine/edge/main \ - -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ - add $__AlpinePackagesEdgeMain + if [[ -n "$__AlpinePackagesEdgeMain" ]]; then + $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \ + -X http://dl-cdn.alpinelinux.org/alpine/edge/main \ + -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ + add $__AlpinePackagesEdgeMain + fi - $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \ - -X http://dl-cdn.alpinelinux.org/alpine/edge/community \ - -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ - add $__AlpinePackagesEdgeCommunity + if [[ -n "$__AlpinePackagesEdgeCommunity" ]]; then + $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \ + -X http://dl-cdn.alpinelinux.org/alpine/edge/community \ + -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ + add $__AlpinePackagesEdgeCommunity + fi rm -r $__ApkToolsDir elif [[ "$__CodeName" == "freebsd" ]]; then diff --git a/eng/common/performance/performance-setup.ps1 b/eng/common/performance/performance-setup.ps1 index 0edb2ae2..c6f1045e 100644 --- a/eng/common/performance/performance-setup.ps1 +++ b/eng/common/performance/performance-setup.ps1 @@ -79,14 +79,6 @@ $CommonSetupArguments="--channel master --queue $Queue --build-number $BuildNumb $SetupArguments = "--repository https://github.com/$Repository --branch $Branch --get-perf-hash --commit-sha $CommitSha $CommonSetupArguments" -#This grabs the LKG version number of dotnet and passes it to our scripts -$VersionJSON = Get-Content global.json | ConvertFrom-Json -$DotNetVersion = $VersionJSON.tools.dotnet -# TODO: Change this back to parsing when we have a good story for dealing with TFM changes or when the LKG in runtime gets updated to include net6.0 -# $SetupArguments = "--dotnet-versions $DotNetVersion $SetupArguments" -$SetupArguments = "--dotnet-versions 6.0.100-alpha.1.20553.6 $SetupArguments" - - if ($RunFromPerformanceRepo) { $SetupArguments = "--perf-hash $CommitSha $CommonSetupArguments" diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index 7321feb5..cd4f233d 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -88,6 +88,10 @@ while (($# > 0)); do internal=true shift 1 ;; + --alpine) + alpine=true + shift 1 + ;; --llvm) llvm=true shift 1 @@ -143,6 +147,7 @@ while (($# > 0)); do echo " --monodotnet Pass the path to the mono dotnet for mono performance testing." echo " --wasm Path to the unpacked wasm runtime pack." echo " --latestdotnet --dotnet-versions will not be specified. --dotnet-versions defaults to LKG version in global.json " + echo " --alpine Set for runs on Alpine" echo "" exit 0 ;; @@ -198,12 +203,20 @@ if [[ "$internal" == true ]]; then else queue=Ubuntu.1804.Amd64.Tiger.Perf fi + + if [[ "$alpine" = "true" ]]; then + queue=alpine.amd64.tiger.perf + fi else if [[ "$architecture" = "arm64" ]]; then queue=ubuntu.1804.armarch.open else queue=Ubuntu.1804.Amd64.Open fi + + if [[ "$alpine" = "true" ]]; then + queue=alpine.amd64.tiger.perf + fi fi if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "false" ]]; then @@ -224,16 +237,6 @@ fi common_setup_arguments="--channel master --queue $queue --build-number $build_number --build-configs $configurations --architecture $architecture" setup_arguments="--repository https://github.com/$repository --branch $branch --get-perf-hash --commit-sha $commit_sha $common_setup_arguments" - -if [[ "$use_latest_dotnet" = false ]]; then - # Get the tools section from the global.json. - # This grabs the LKG version number of dotnet and passes it to our scripts - dotnet_version=`cat global.json | python3 -c 'import json,sys;obj=json.load(sys.stdin);print(obj["tools"]["dotnet"])'` - # TODO: Change this back to parsing when we have a good story for dealing with TFM changes or when the LKG in runtime gets updated to include net6.0 - # setup_arguments="--dotnet-versions $dotnet_version $setup_arguments" - setup_arguments="--dotnet-versions 6.0.100-alpha.1.20553.6 $setup_arguments" -fi - if [[ "$run_from_perf_repo" = true ]]; then payload_directory= workitem_directory=$source_directory diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1 index b681d797..81b729f7 100644 --- a/eng/common/sdl/execute-all-sdl-tools.ps1 +++ b/eng/common/sdl/execute-all-sdl-tools.ps1 @@ -87,10 +87,6 @@ try { & $(Join-Path $PSScriptRoot 'run-sdl.ps1') -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams } - if ($UpdateBaseline) { - & (Join-Path $PSScriptRoot 'push-gdn.ps1') -Repository $RepoName -BranchName $BranchName -GdnFolder $GdnFolder -AzureDevOpsAccessToken $AzureDevOpsAccessToken -PushReason 'Update baseline' - } - if ($TsaPublish) { if ($TsaBranchName -and $BuildNumber) { if (-not $TsaRepositoryName) { diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1 index bb6a4297..1fe92711 100644 --- a/eng/common/sdl/init-sdl.ps1 +++ b/eng/common/sdl/init-sdl.ps1 @@ -46,7 +46,6 @@ try { Write-PipelineTelemetryError -Force -Category 'Build' -Message "Guardian baseline failed with exit code $LASTEXITCODE." ExitWithExitCode $LASTEXITCODE } - & $(Join-Path $PSScriptRoot 'push-gdn.ps1') -Repository $Repository -BranchName $BranchName -GdnFolder $gdnFolder -AzureDevOpsAccessToken $AzureDevOpsAccessToken -PushReason 'Initialize gdn folder' ExitWithExitCode 0 } catch { diff --git a/eng/common/sdl/push-gdn.ps1 b/eng/common/sdl/push-gdn.ps1 deleted file mode 100644 index d8fd2d82..00000000 --- a/eng/common/sdl/push-gdn.ps1 +++ /dev/null @@ -1,69 +0,0 @@ -Param( - [string] $Repository, - [string] $BranchName='master', - [string] $GdnFolder, - [string] $AzureDevOpsAccessToken, - [string] $PushReason -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 -$disableConfigureToolsetImport = $true -$LASTEXITCODE = 0 - -try { - # `tools.ps1` checks $ci to perform some actions. Since the SDL - # scripts don't necessarily execute in the same agent that run the - # build.ps1/sh script this variable isn't automatically set. - $ci = $true - . $PSScriptRoot\..\tools.ps1 - - # We create the temp directory where we'll store the sdl-config repository - $sdlDir = Join-Path $env:TEMP 'sdl' - if (Test-Path $sdlDir) { - Remove-Item -Force -Recurse $sdlDir - } - - Write-Host "git clone https://dnceng:`$AzureDevOpsAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir" - git clone https://dnceng:$AzureDevOpsAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir - if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Git clone failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE - } - # We copy the .gdn folder from our local run into the git repository so it can be committed - $sdlRepositoryFolder = Join-Path (Join-Path (Join-Path $sdlDir $Repository) $BranchName) '.gdn' - if (Get-Command Robocopy) { - Robocopy /S $GdnFolder $sdlRepositoryFolder - } else { - rsync -r $GdnFolder $sdlRepositoryFolder - } - # cd to the sdl-config directory so we can run git there - Push-Location $sdlDir - # git add . --> git commit --> git push - Write-Host 'git add .' - git add . - if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Git add failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE - } - Write-Host "git -c user.email=`"dn-bot@microsoft.com`" -c user.name=`"Dotnet Bot`" commit -m `"$PushReason for $Repository/$BranchName`"" - git -c user.email="dn-bot@microsoft.com" -c user.name="Dotnet Bot" commit -m "$PushReason for $Repository/$BranchName" - if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Git commit failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE - } - Write-Host 'git push' - git push - if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Git push failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE - } - - # Return to the original directory - Pop-Location -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'Sdl' -Message $_ - ExitWithExitCode 1 -} diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml index 61263307..c002a2b1 100644 --- a/eng/common/templates/job/source-index-stage1.yml +++ b/eng/common/templates/job/source-index-stage1.yml @@ -1,6 +1,6 @@ parameters: runAsPublic: false - sourceIndexPackageVersion: 1.0.0-beta5 + sourceIndexPackageVersion: 1.0.1-20210225.1 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" preSteps: [] diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 822ff597..c84ac55e 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -128,7 +128,7 @@ stages: - job: displayName: Signing Validation dependsOn: setupMaestroVars - condition: eq( ${{ parameters.enableSigningValidation }}, 'true') + condition: and( eq( ${{ parameters.enableSigningValidation }}, 'true'), ne( variables['PostBuildSign'], 'true')) variables: - template: common-variables.yml - name: AzDOProjectName diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 572da3b9..be5ce4a2 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -169,7 +169,7 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { Set-Content -Path $sdkCacheFileTemp -Value $dotnetRoot try { - Rename-Item -Force -Path $sdkCacheFileTemp 'sdk.txt' + Move-Item -Force $sdkCacheFileTemp (Join-Path $ToolsetDir 'sdk.txt') } catch { # Somebody beat us Remove-Item -Path $sdkCacheFileTemp diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 9019b7f5..5fad1846 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -273,8 +273,7 @@ function GetDotNetInstallScript { if command -v curl > /dev/null; then # first, try directly, if this fails we will retry with verbose logging curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || { - if command -v openssl &> /dev/null - then + if command -v openssl &> /dev/null; then echo "Curl failed; dumping some information about dotnet.microsoft.com for later investigation" echo | openssl s_client -showcerts -servername dotnet.microsoft.com -connect dotnet.microsoft.com:443 fi diff --git a/global.json b/global.json index a4eb4120..86995a1f 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "5.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21105.12" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21125.5" } } From d76759199c95377d6f5734ae5aaaf984bf2c70fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Matou=C5=A1ek?= Date: Wed, 17 Mar 2021 14:42:51 -0700 Subject: [PATCH 130/161] Update azure-pipelines.yml --- azure-pipelines.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b67505a9..3297dab9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -3,6 +3,7 @@ pr: branches: include: - master + - main - release/* paths: exclude: @@ -15,7 +16,8 @@ schedules: displayName: Daily Official Build branches: include: - - master + - master + - main variables: - name: _TeamName From 36ddc5a01fa4b3fa507e3cc4f93ec5c021d7be80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Matou=C5=A1ek?= Date: Wed, 17 Mar 2021 15:33:48 -0700 Subject: [PATCH 131/161] Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f4c6c566..13d0163e 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Source Link is a language- and source-control agnostic system for providing first-class source debugging experiences for binaries. The goal of the project is to enable anyone building [NuGet libraries to provide source debugging](https://github.com/dotnet/designs/blob/main/accepted/2020/diagnostics/debugging-with-symbols-and-sources.md) for their users with almost no effort. Microsoft libraries, such as .NET Core and Roslyn have enabled Source Link. Source Link is supported by Microsoft. -Source Link is a [set of packages](https://www.nuget.org/packages?q=Microsoft.SourceLink) and a [specification](https://github.com/dotnet/designs/blob/master/accepted/2020/diagnostics/source-link.md#source-link-file-specification) for describing source control metadata that can be embedded in symbols, binaries and packages. +Source Link is a [set of packages](https://www.nuget.org/packages?q=Microsoft.SourceLink) and a [specification](https://github.com/dotnet/designs/blob/main/accepted/2020/diagnostics/source-link.md#source-link-file-specification) for describing source control metadata that can be embedded in symbols, binaries and packages. Visual Studio 15.3+ supports reading Source Link information from symbols while debugging. It downloads and displays the appropriate commit-specific source for users, such as from [raw.githubusercontent](https://raw.githubusercontent.com/dotnet/roslyn/681cbc414542ffb9fb13ded613d26a88ea73a44b/src/VisualStudio/Core/Def/Implementation/ProjectSystem/AbstractProject.cs), enabling breakpoints and all other sources debugging experience on arbitrary NuGet dependencies. Visual Studio 15.7+ supports downloading source files from private GitHub and Azure DevOps (former VSTS) repositories that require authentication. @@ -136,7 +136,7 @@ For projects hosted on-prem via [gitea](https://gitea.io) reference [Microsoft.S ### Multiple providers, repositories with submodules -If your repository contains submodules hosted by other git providers reference packages of all these providers. For example, projects in a repository hosted by Azure Repos that links a GitHub repository via a submodule should reference both [Microsoft.SourceLink.AzureRepos.Git](https://www.nuget.org/packages/Microsoft.SourceLink.AzureRepos.Git) and [Microsoft.SourceLink.GitHub](https://www.nuget.org/packages/Microsoft.SourceLink.GitHub) packages. [Additional configuration](https://github.com/dotnet/sourcelink/blob/master/docs/README.md#configuring-projects-with-multiple-sourcelink-providers) might be needed if multiple Source Link packages are used in the project. +If your repository contains submodules hosted by other git providers reference packages of all these providers. For example, projects in a repository hosted by Azure Repos that links a GitHub repository via a submodule should reference both [Microsoft.SourceLink.AzureRepos.Git](https://www.nuget.org/packages/Microsoft.SourceLink.AzureRepos.Git) and [Microsoft.SourceLink.GitHub](https://www.nuget.org/packages/Microsoft.SourceLink.GitHub) packages. [Additional configuration](https://github.com/dotnet/sourcelink/blob/main/docs/README.md#configuring-projects-with-multiple-sourcelink-providers) might be needed if multiple Source Link packages are used in the project. ## Using Source Link in C++ projects @@ -204,7 +204,7 @@ For managed projects, consider switching to Portable PDBs by setting `DebugType` Pre-release builds are available from Azure DevOps public feed: `https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json` ([browse](https://dev.azure.com/dnceng/public/_packaging?_a=feed&feed=dotnet-tools)). -[![Build Status](https://dnceng.visualstudio.com/public/_apis/build/status/SourceLink%20PR?branchName=master)](https://dnceng.visualstudio.com/public/_build/latest?definitionId=297?branchName=master) +[![Build Status](https://dnceng.visualstudio.com/public/_apis/build/status/SourceLink%20PR?branchName=main)](https://dnceng.visualstudio.com/public/_build/latest?definitionId=297?branchName=main) ## Experience in Visual Studio From 9c91c34cc760927755cb2c2d0cca356ceed3ae9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Matou=C5=A1ek?= Date: Wed, 17 Mar 2021 15:45:15 -0700 Subject: [PATCH 132/161] Update azure-pipelines.yml --- azure-pipelines.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3297dab9..f3533039 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -2,7 +2,6 @@ pr: branches: include: - - master - main - release/* paths: @@ -16,7 +15,6 @@ schedules: displayName: Daily Official Build branches: include: - - master - main variables: From 4b584dbc392bb1aad49c2eb1ab84d8b489b6dccc Mon Sep 17 00:00:00 2001 From: Jon Fortescue Date: Wed, 31 Mar 2021 11:39:44 -0700 Subject: [PATCH 133/161] Migrate to OneLocBuild (#711) * Add OneLocBuild task * Add sourcelink package * Update eng/common files --- azure-pipelines.yml | 6 ++ eng/common/generate-locproject.ps1 | 101 +++++++++++++++++++++++ eng/common/templates/job/onelocbuild.yml | 78 +++++++++++++++++ 3 files changed, 185 insertions(+) create mode 100644 eng/common/generate-locproject.ps1 create mode 100644 eng/common/templates/job/onelocbuild.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f3533039..8eea90b0 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -29,6 +29,12 @@ stages: - stage: build displayName: Build jobs: + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - template: /eng/common/templates/job/onelocbuild.yml + parameters: + CreatePr: false + LclSource: lclFilesfromPackage + LclPackageId: 'LCL-JUNO-PROD-SOURCELINK' - template: /eng/common/templates/jobs/jobs.yml parameters: enableMicrobuild: true diff --git a/eng/common/generate-locproject.ps1 b/eng/common/generate-locproject.ps1 new file mode 100644 index 00000000..7225ddc6 --- /dev/null +++ b/eng/common/generate-locproject.ps1 @@ -0,0 +1,101 @@ +Param( + [Parameter(Mandatory=$true)][string] $SourcesDirectory, # Directory where source files live; if using a Localize directory it should live in here + [string] $LanguageSet = 'VS_Main_Languages', # Language set to be used in the LocProject.json + [switch] $UseCheckedInLocProjectJson, # When set, generates a LocProject.json and compares it to one that already exists in the repo; otherwise just generates one + [switch] $CreateNeutralXlfs # Creates neutral xlf files. Only set to false when running locally +) + +# Generates LocProject.json files for the OneLocBuild task. OneLocBuildTask is described here: +# https://ceapex.visualstudio.com/CEINTL/_wiki/wikis/CEINTL.wiki/107/Localization-with-OneLocBuild-Task + +Set-StrictMode -Version 2.0 +$ErrorActionPreference = "Stop" +. $PSScriptRoot\tools.ps1 + +Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1') + +$exclusionsFilePath = "$SourcesDirectory\Localize\LocExclusions.json" +$exclusions = @{ Exclusions = @() } +if (Test-Path -Path $exclusionsFilePath) +{ + $exclusions = Get-Content "$exclusionsFilePath" | ConvertFrom-Json +} + +Push-Location "$SourcesDirectory" # push location for Resolve-Path -Relative to work + +# Template files +$jsonFiles = @() +$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\en\..+\.json" } # .NET templating pattern +$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern + +$xlfFiles = @() + +$allXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.xlf" +$langXlfFiles = @() +if ($allXlfFiles) { + $null = $allXlfFiles[0].FullName -Match "\.([\w-]+)\.xlf" # matches '[langcode].xlf' + $firstLangCode = $Matches.1 + $langXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.$firstLangCode.xlf" +} +$langXlfFiles | ForEach-Object { + $null = $_.Name -Match "(.+)\.[\w-]+\.xlf" # matches '[filename].[langcode].xlf + + $destinationFile = "$($_.Directory.FullName)\$($Matches.1).xlf" + $xlfFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru +} + +$locFiles = $jsonFiles + $xlfFiles + +$locJson = @{ + Projects = @( + @{ + LanguageSet = $LanguageSet + LocItems = @( + $locFiles | ForEach-Object { + $outputPath = "$(($_.DirectoryName | Resolve-Path -Relative) + "\")" + $continue = $true + foreach ($exclusion in $exclusions.Exclusions) { + if ($outputPath.Contains($exclusion)) + { + $continue = $false + } + } + $sourceFile = ($_.FullName | Resolve-Path -Relative) + if (!$CreateNeutralXlfs -and $_.Extension -eq '.xlf') { + Remove-Item -Path $sourceFile + } + if ($continue) + { + return @{ + SourceFile = $sourceFile + CopyOption = "LangIDOnName" + OutputPath = $outputPath + } + } + } + ) + } + ) +} + +$json = ConvertTo-Json $locJson -Depth 5 +Write-Host "LocProject.json generated:`n`n$json`n`n" +Pop-Location + +if (!$UseCheckedInLocProjectJson) { + New-Item "$SourcesDirectory\Localize\LocProject.json" -Force # Need this to make sure the Localize directory is created + Set-Content "$SourcesDirectory\Localize\LocProject.json" $json +} +else { + New-Item "$SourcesDirectory\Localize\LocProject-generated.json" -Force # Need this to make sure the Localize directory is created + Set-Content "$SourcesDirectory\Localize\LocProject-generated.json" $json + + if ((Get-FileHash "$SourcesDirectory\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\Localize\LocProject.json").Hash) { + Write-PipelineTelemetryError -Category "OneLocBuild" -Message "Existing LocProject.json differs from generated LocProject.json. Download LocProject-generated.json and compare them." + + exit 1 + } + else { + Write-Host "Generated LocProject.json and current LocProject.json are identical." + } +} \ No newline at end of file diff --git a/eng/common/templates/job/onelocbuild.yml b/eng/common/templates/job/onelocbuild.yml new file mode 100644 index 00000000..928a70cd --- /dev/null +++ b/eng/common/templates/job/onelocbuild.yml @@ -0,0 +1,78 @@ +parameters: + # Optional: dependencies of the job + dependsOn: '' + + # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool + pool: + vmImage: vs2017-win2016 + + CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex + GithubPat: $(BotAccount-dotnet-bot-repo-PAT) + + SourcesDirectory: $(Build.SourcesDirectory) + CreatePr: true + UseCheckedInLocProjectJson: false + LanguageSet: VS_Main_Languages + LclSource: lclFilesInRepo + LclPackageId: '' + RepoType: gitHub + +jobs: +- job: OneLocBuild + + dependsOn: ${{ parameters.dependsOn }} + + displayName: OneLocBuild + + pool: ${{ parameters.pool }} + + variables: + - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat + - name: _GenerateLocProjectArguments + value: -SourcesDirectory ${{ parameters.SourcesDirectory }} + -LanguageSet "${{ parameters.LanguageSet }}" + -CreateNeutralXlfs + - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}: + - name: _GenerateLocProjectArguments + value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson + + + steps: + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1 + arguments: $(_GenerateLocProjectArguments) + displayName: Generate LocProject.json + + - task: OneLocBuild@2 + displayName: OneLocBuild + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + inputs: + locProj: Localize/LocProject.json + outDir: $(Build.ArtifactStagingDirectory) + lclSource: ${{ parameters.LclSource }} + lclPackageId: ${{ parameters.LclPackageId }} + isCreatePrSelected: ${{ parameters.CreatePr }} + packageSourceAuth: patAuth + patVariable: ${{ parameters.CeapexPat }} + ${{ if eq(parameters.RepoType, 'gitHub') }}: + repoType: ${{ parameters.RepoType }} + gitHubPatVariable: "${{ parameters.GithubPat }}" + condition: always() + + - task: PublishBuildArtifacts@1 + displayName: Publish Localization Files + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/loc' + PublishLocation: Container + ArtifactName: Loc + condition: always() + + - task: PublishBuildArtifacts@1 + displayName: Publish LocProject.json + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/Localize/' + PublishLocation: Container + ArtifactName: Loc + condition: always() \ No newline at end of file From 4604c2e536f4762ebbb5b8651926eaa7b76723c4 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Tue, 27 Jul 2021 17:59:19 +0000 Subject: [PATCH 134/161] [main] Update dependencies from dotnet/arcade (#710) [main] Update dependencies from dotnet/arcade - Target tests to net6.0 - Target net6.0 --- eng/Version.Details.xml | 4 +- eng/common/SetupNugetSources.ps1 | 6 + eng/common/SetupNugetSources.sh | 24 ++ eng/common/build.ps1 | 2 + eng/common/cross/arm64/tizen-fetch.sh | 2 +- eng/common/cross/armel/tizen-fetch.sh | 2 +- eng/common/cross/build-rootfs.sh | 10 +- eng/common/dotnet-install.sh | 2 +- eng/common/generate-locproject.ps1 | 42 ++- eng/common/internal-feed-operations.ps1 | 6 +- eng/common/internal-feed-operations.sh | 2 +- eng/common/msbuild.ps1 | 1 + eng/common/native/install-tool.ps1 | 2 +- eng/common/performance/blazor_perf.proj | 30 -- eng/common/performance/crossgen_perf.proj | 110 ------- eng/common/performance/microbenchmarks.proj | 144 --------- eng/common/performance/performance-setup.ps1 | 141 -------- eng/common/performance/performance-setup.sh | 301 ------------------ eng/common/post-build/publish-using-darc.ps1 | 2 +- .../post-build/sourcelink-validation.ps1 | 80 +++-- eng/common/post-build/symbols-validation.ps1 | 124 +++++--- eng/common/sdk-task.ps1 | 8 +- eng/common/sdl/configure-sdl-tool.ps1 | 109 +++++++ eng/common/sdl/execute-all-sdl-tools.ps1 | 73 ++++- eng/common/sdl/extract-artifact-archives.ps1 | 63 ++++ eng/common/sdl/init-sdl.ps1 | 2 +- eng/common/sdl/run-sdl.ps1 | 52 +-- eng/common/templates/job/execute-sdl.yml | 110 ++++++- eng/common/templates/job/onelocbuild.yml | 25 +- eng/common/templates/job/performance.yml | 95 ------ .../templates/job/publish-build-assets.yml | 1 + eng/common/templates/job/source-build.yml | 3 + .../templates/job/source-index-stage1.yml | 6 +- .../templates/phases/publish-build-assets.yml | 1 + .../channels/generic-internal-channel.yml | 8 + .../channels/generic-public-channel.yml | 8 + .../templates/post-build/post-build.yml | 3 + .../post-build/setup-maestro-vars.yml | 1 + .../templates/steps/perf-send-to-helix.yml | 50 --- eng/common/templates/steps/send-to-helix.yml | 4 +- eng/common/templates/steps/source-build.yml | 13 +- eng/common/tools.ps1 | 184 +++++++---- eng/common/tools.sh | 13 +- global.json | 4 +- .../Microsoft.Build.StandardCI.csproj | 2 +- ...Microsoft.Build.Tasks.Git.UnitTests.csproj | 2 +- .../Microsoft.Build.Tasks.Git.csproj | 2 +- .../build/Microsoft.Build.Tasks.Git.props | 2 +- ...ink.AzureDevOpsServer.Git.UnitTests.csproj | 2 +- ...ft.SourceLink.AzureDevOpsServer.Git.csproj | 2 +- ...t.SourceLink.AzureDevOpsServer.Git.targets | 2 +- ...SourceLink.AzureRepos.Git.UnitTests.csproj | 2 +- ...Microsoft.SourceLink.AzureRepos.Git.csproj | 2 +- ...icrosoft.SourceLink.AzureRepos.Git.targets | 2 +- ...crosoft.SourceLink.AzureRepos.Tfvc.targets | 2 +- ....SourceLink.Bitbucket.Git.UnitTests.csproj | 2 +- .../Microsoft.SourceLink.Bitbucket.Git.csproj | 2 +- ...Microsoft.SourceLink.Bitbucket.Git.targets | 2 +- ...crosoft.SourceLink.Common.UnitTests.csproj | 2 +- .../Microsoft.SourceLink.Common.csproj | 2 +- .../build/Microsoft.SourceLink.Common.props | 2 +- ...oft.SourceLink.Git.IntegrationTests.csproj | 2 +- ...crosoft.SourceLink.GitHub.UnitTests.csproj | 2 +- .../Microsoft.SourceLink.GitHub.csproj | 2 +- .../build/Microsoft.SourceLink.GitHub.targets | 2 +- ...crosoft.SourceLink.GitLab.UnitTests.csproj | 2 +- .../Microsoft.SourceLink.GitLab.csproj | 2 +- .../build/Microsoft.SourceLink.GitLab.targets | 2 +- ...crosoft.SourceLink.GitWeb.UnitTests.csproj | 2 +- .../Microsoft.SourceLink.GitWeb.csproj | 2 +- .../build/Microsoft.SourceLink.GitWeb.targets | 2 +- ...icrosoft.SourceLink.Gitea.UnitTests.csproj | 2 +- .../Microsoft.SourceLink.Gitea.csproj | 2 +- .../build/Microsoft.SourceLink.Gitea.targets | 2 +- ...icrosoft.SourceLink.Tools.UnitTests.csproj | 2 +- .../Microsoft.SourceLink.Tools.Package.csproj | 2 +- src/TestUtilities/TestUtilities.csproj | 2 +- 77 files changed, 791 insertions(+), 1148 deletions(-) delete mode 100644 eng/common/performance/blazor_perf.proj delete mode 100644 eng/common/performance/crossgen_perf.proj delete mode 100644 eng/common/performance/microbenchmarks.proj delete mode 100644 eng/common/performance/performance-setup.ps1 delete mode 100755 eng/common/performance/performance-setup.sh create mode 100644 eng/common/sdl/configure-sdl-tool.ps1 create mode 100644 eng/common/sdl/extract-artifact-archives.ps1 delete mode 100644 eng/common/templates/job/performance.yml delete mode 100644 eng/common/templates/steps/perf-send-to-helix.yml diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 3dad9464..4f6463c0 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 15246f4af00a1cb2e580783d32ec2937b1878a64 + c6a28c81f96d196338b3ea520bc1e6dc7c440ee2 https://github.com/dotnet/source-build-reference-packages diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index a0b5fc37..18823840 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -158,4 +158,10 @@ if ($dotnet5Source -ne $null) { AddPackageSource -Sources $sources -SourceName "dotnet5-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password } +$dotnet6Source = $sources.SelectSingleNode("add[@key='dotnet6']") +if ($dotnet6Source -ne $null) { + AddPackageSource -Sources $sources -SourceName "dotnet6-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet6-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password +} + $doc.Save($filename) diff --git a/eng/common/SetupNugetSources.sh b/eng/common/SetupNugetSources.sh index 2734601c..ad3fb74f 100644 --- a/eng/common/SetupNugetSources.sh +++ b/eng/common/SetupNugetSources.sh @@ -129,6 +129,30 @@ if [ "$?" == "0" ]; then PackageSources+=('dotnet5-internal-transport') fi +# Ensure dotnet6-internal and dotnet6-internal-transport are in the packageSources if the public dotnet6 feeds are present +grep -i "" + + sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile + fi + PackageSources+=('dotnet6-internal') + + grep -i "" $ConfigFile + if [ "$?" != "0" ]; then + echo "Adding dotnet6-internal-transport to the packageSources." + PackageSourcesNodeFooter="" + PackageSourceTemplate="${TB}" + + sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile + fi + PackageSources+=('dotnet6-internal-transport') +fi + # I want things split line by line PrevIFS=$IFS IFS=$'\n' diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 94a91c08..8943da24 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -25,6 +25,7 @@ Param( [switch] $prepareMachine, [string] $runtimeSourceFeed = '', [string] $runtimeSourceFeedKey = '', + [switch] $excludePrereleaseVS, [switch] $help, [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties ) @@ -65,6 +66,7 @@ function Print-Usage() { Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build" Write-Host " -warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." + Write-Host " -excludePrereleaseVS Set to exclude build engines in prerelease versions of Visual Studio" Write-Host "" Write-Host "Command line arguments not listed above are passed thru to msbuild." diff --git a/eng/common/cross/arm64/tizen-fetch.sh b/eng/common/cross/arm64/tizen-fetch.sh index a48a6f51..16d1301f 100644 --- a/eng/common/cross/arm64/tizen-fetch.sh +++ b/eng/common/cross/arm64/tizen-fetch.sh @@ -157,7 +157,7 @@ fetch_tizen_pkgs() Inform "Initialize arm base" fetch_tizen_pkgs_init standard base Inform "fetch common packages" -fetch_tizen_pkgs aarch64 gcc glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel +fetch_tizen_pkgs aarch64 gcc glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils Inform "fetch coreclr packages" fetch_tizen_pkgs aarch64 lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu Inform "fetch corefx packages" diff --git a/eng/common/cross/armel/tizen-fetch.sh b/eng/common/cross/armel/tizen-fetch.sh index 2776cbba..64f0187e 100755 --- a/eng/common/cross/armel/tizen-fetch.sh +++ b/eng/common/cross/armel/tizen-fetch.sh @@ -157,7 +157,7 @@ fetch_tizen_pkgs() Inform "Initialize arm base" fetch_tizen_pkgs_init standard base Inform "fetch common packages" -fetch_tizen_pkgs armv7l gcc glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel +fetch_tizen_pkgs armv7l gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils Inform "fetch coreclr packages" fetch_tizen_pkgs armv7l lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu Inform "fetch corefx packages" diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index b2662244..735a4c82 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -33,7 +33,6 @@ __AlpinePackages="alpine-base" __AlpinePackages+=" build-base" __AlpinePackages+=" linux-headers" __AlpinePackagesEdgeCommunity=" lldb-dev" -__AlpinePackagesEdgeMain=" llvm10-libs" __AlpinePackagesEdgeMain+=" python3" __AlpinePackagesEdgeMain+=" libedit" @@ -74,6 +73,10 @@ __IllumosPackages+=" mit-krb5-1.16.2nb4" __IllumosPackages+=" openssl-1.1.1e" __IllumosPackages+=" zlib-1.2.11" +# ML.NET dependencies +__UbuntuPackages+=" libomp5" +__UbuntuPackages+=" libomp-dev" + __UseMirror=0 __UnprocessedBuildArgs= @@ -111,6 +114,8 @@ while :; do __UbuntuArch=s390x __UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/" __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libunwind8-dev//') + __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp-dev//') + __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp5//') unset __LLDB_Package ;; x86) @@ -187,6 +192,8 @@ while :; do __CodeName=alpine __UbuntuRepo= __AlpineVersion=3.9 + __AlpinePackagesEdgeMain+=" llvm11-libs" + __AlpinePackagesEdgeMain+=" clang-libs" ;; alpine3.13) __CodeName=alpine @@ -197,6 +204,7 @@ while :; do __AlpinePackagesEdgeCommunity= __AlpinePackages+=$__AlpinePackagesEdgeMain __AlpinePackagesEdgeMain= + __AlpinePackages+=" llvm10-libs" ;; freebsd11) __FreeBSDBase="11.3-RELEASE" diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh index d6efeb44..fdfeea66 100755 --- a/eng/common/dotnet-install.sh +++ b/eng/common/dotnet-install.sh @@ -70,7 +70,7 @@ case $cpuname in ;; esac -dotnetRoot="$repo_root/.dotnet" +dotnetRoot="${repo_root}.dotnet" if [[ $architecture != "" ]] && [[ $architecture != $buildarch ]]; then dotnetRoot="$dotnetRoot/$architecture" fi diff --git a/eng/common/generate-locproject.ps1 b/eng/common/generate-locproject.ps1 index 7225ddc6..25e97ac0 100644 --- a/eng/common/generate-locproject.ps1 +++ b/eng/common/generate-locproject.ps1 @@ -14,7 +14,7 @@ $ErrorActionPreference = "Stop" Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1') -$exclusionsFilePath = "$SourcesDirectory\Localize\LocExclusions.json" +$exclusionsFilePath = "$SourcesDirectory\eng\Localize\LocExclusions.json" $exclusions = @{ Exclusions = @() } if (Test-Path -Path $exclusionsFilePath) { @@ -25,8 +25,15 @@ Push-Location "$SourcesDirectory" # push location for Resolve-Path -Relative to # Template files $jsonFiles = @() -$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\en\..+\.json" } # .NET templating pattern -$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern +$jsonTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\.+\.en\.json" } # .NET templating pattern +$jsonTemplateFiles | ForEach-Object { + $null = $_.Name -Match "(.+)\.[\w-]+\.json" # matches '[filename].[langcode].json + + $destinationFile = "$($_.Directory.FullName)\$($Matches.1).json" + $jsonFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru +} + +$jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern $xlfFiles = @() @@ -44,7 +51,7 @@ $langXlfFiles | ForEach-Object { $xlfFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru } -$locFiles = $jsonFiles + $xlfFiles +$locFiles = $jsonFiles + $jsonWinformsTemplateFiles + $xlfFiles $locJson = @{ Projects = @( @@ -66,10 +73,19 @@ $locJson = @{ } if ($continue) { - return @{ - SourceFile = $sourceFile - CopyOption = "LangIDOnName" - OutputPath = $outputPath + if ($_.Directory.Name -eq 'en' -and $_.Extension -eq '.json') { + return @{ + SourceFile = $sourceFile + CopyOption = "LangIDOnPath" + OutputPath = "$($_.Directory.Parent.FullName | Resolve-Path -Relative)\" + } + } + else { + return @{ + SourceFile = $sourceFile + CopyOption = "LangIDOnName" + OutputPath = $outputPath + } } } } @@ -83,14 +99,14 @@ Write-Host "LocProject.json generated:`n`n$json`n`n" Pop-Location if (!$UseCheckedInLocProjectJson) { - New-Item "$SourcesDirectory\Localize\LocProject.json" -Force # Need this to make sure the Localize directory is created - Set-Content "$SourcesDirectory\Localize\LocProject.json" $json + New-Item "$SourcesDirectory\eng\Localize\LocProject.json" -Force # Need this to make sure the Localize directory is created + Set-Content "$SourcesDirectory\eng\Localize\LocProject.json" $json } else { - New-Item "$SourcesDirectory\Localize\LocProject-generated.json" -Force # Need this to make sure the Localize directory is created - Set-Content "$SourcesDirectory\Localize\LocProject-generated.json" $json + New-Item "$SourcesDirectory\eng\Localize\LocProject-generated.json" -Force # Need this to make sure the Localize directory is created + Set-Content "$SourcesDirectory\eng\Localize\LocProject-generated.json" $json - if ((Get-FileHash "$SourcesDirectory\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\Localize\LocProject.json").Hash) { + if ((Get-FileHash "$SourcesDirectory\eng\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\eng\Localize\LocProject.json").Hash) { Write-PipelineTelemetryError -Category "OneLocBuild" -Message "Existing LocProject.json differs from generated LocProject.json. Download LocProject-generated.json and compare them." exit 1 diff --git a/eng/common/internal-feed-operations.ps1 b/eng/common/internal-feed-operations.ps1 index 418c0993..92b77347 100644 --- a/eng/common/internal-feed-operations.ps1 +++ b/eng/common/internal-feed-operations.ps1 @@ -45,11 +45,11 @@ function SetupCredProvider { # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable # feeds successfully - $nugetConfigPath = "$RepoRoot\NuGet.config" + $nugetConfigPath = Join-Path $RepoRoot "NuGet.config" if (-Not (Test-Path -Path $nugetConfigPath)) { Write-PipelineTelemetryError -Category 'Build' -Message 'NuGet.config file not found in repo root!' - ExitWithExitCode 1 + ExitWithExitCode 1 } $endpoints = New-Object System.Collections.ArrayList @@ -85,7 +85,7 @@ function SetupCredProvider { #Workaround for https://github.com/microsoft/msbuild/issues/4430 function InstallDotNetSdkAndRestoreArcade { - $dotnetTempDir = "$RepoRoot\dotnet" + $dotnetTempDir = Join-Path $RepoRoot "dotnet" $dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*) $dotnet = "$dotnetTempDir\dotnet.exe" $restoreProjPath = "$PSScriptRoot\restore.proj" diff --git a/eng/common/internal-feed-operations.sh b/eng/common/internal-feed-operations.sh index e2233e78..9378223b 100755 --- a/eng/common/internal-feed-operations.sh +++ b/eng/common/internal-feed-operations.sh @@ -39,7 +39,7 @@ function SetupCredProvider { # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable # feeds successfully - local nugetConfigPath="$repo_root/NuGet.config" + local nugetConfigPath="{$repo_root}NuGet.config" if [ ! "$nugetConfigPath" ]; then Write-PipelineTelemetryError -category 'Build' "NuGet.config file not found in repo's root!" diff --git a/eng/common/msbuild.ps1 b/eng/common/msbuild.ps1 index c6401230..eea19cd8 100644 --- a/eng/common/msbuild.ps1 +++ b/eng/common/msbuild.ps1 @@ -5,6 +5,7 @@ Param( [bool] $nodeReuse = $true, [switch] $ci, [switch] $prepareMachine, + [switch] $excludePrereleaseVS, [Parameter(ValueFromRemainingArguments=$true)][String[]]$extraArgs ) diff --git a/eng/common/native/install-tool.ps1 b/eng/common/native/install-tool.ps1 index f397e1c7..78f2d84a 100644 --- a/eng/common/native/install-tool.ps1 +++ b/eng/common/native/install-tool.ps1 @@ -105,7 +105,7 @@ try { Write-Error "There are multiple copies of $ToolName in $($ToolInstallDirectory): `n$(@($ToolFilePath | out-string))" exit 1 } elseif (@($ToolFilePath).Length -Lt 1) { - Write-Host "$ToolName was not found in $ToolFilePath." + Write-Host "$ToolName was not found in $ToolInstallDirectory." exit 1 } diff --git a/eng/common/performance/blazor_perf.proj b/eng/common/performance/blazor_perf.proj deleted file mode 100644 index 3b25359c..00000000 --- a/eng/common/performance/blazor_perf.proj +++ /dev/null @@ -1,30 +0,0 @@ - - - python3 - $(HelixPreCommands);chmod +x $HELIX_WORKITEM_PAYLOAD/SOD/SizeOnDisk - - - - - %(Identity) - - - - - %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\ - $(ScenarioDirectory)blazor\ - - - $HELIX_CORRELATION_PAYLOAD/performance/src/scenarios/ - $(ScenarioDirectory)blazor/ - - - - - $(WorkItemDirectory) - cd $(BlazorDirectory);$(Python) pre.py publish --msbuild %27/p:_TrimmerDumpDependencies=true%27 --msbuild-static AdditionalMonoLinkerOptions=%27"%24(AdditionalMonoLinkerOptions) --dump-dependencies"%27 --binlog %27./traces/blazor_publish.binlog%27 - $(Python) test.py sod --scenario-name "%(Identity)" - $(Python) post.py - - - \ No newline at end of file diff --git a/eng/common/performance/crossgen_perf.proj b/eng/common/performance/crossgen_perf.proj deleted file mode 100644 index eb8bdd9c..00000000 --- a/eng/common/performance/crossgen_perf.proj +++ /dev/null @@ -1,110 +0,0 @@ - - - - - %(Identity) - - - - - - py -3 - $(HelixPreCommands) - %HELIX_CORRELATION_PAYLOAD%\Core_Root - %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\ - $(ScenarioDirectory)crossgen\ - $(ScenarioDirectory)crossgen2\ - - - python3 - $(HelixPreCommands);chmod +x $HELIX_WORKITEM_PAYLOAD/startup/Startup;chmod +x $HELIX_WORKITEM_PAYLOAD/startup/perfcollect;sudo apt update;chmod +x $HELIX_WORKITEM_PAYLOAD/SOD/SizeOnDisk - $HELIX_CORRELATION_PAYLOAD/Core_Root - $HELIX_CORRELATION_PAYLOAD/performance/src/scenarios/ - $(ScenarioDirectory)crossgen/ - $(ScenarioDirectory)crossgen2/ - - - - - - - - - - - - - - - - - - - - - - - $(WorkItemDirectory) - $(Python) $(CrossgenDirectory)test.py crossgen --core-root $(CoreRoot) --test-name %(Identity) - - - - - - $(WorkItemDirectory) - $(Python) $(Crossgen2Directory)test.py crossgen2 --core-root $(CoreRoot) --single %(Identity) - - - - - - $(WorkItemDirectory) - $(Python) $(Crossgen2Directory)test.py crossgen2 --core-root $(CoreRoot) --single %(Identity) --singlethreaded True - - - - - - $(WorkItemDirectory) - $(Python) $(CrossgenDirectory)pre.py crossgen --core-root $(CoreRoot) --single %(Identity) - $(Python) $(CrossgenDirectory)test.py sod --scenario-name "Crossgen %(Identity) Size" --dirs ./crossgen.out/ - $(Python) $(CrossgenDirectory)post.py - - - - - - $(WorkItemDirectory) - $(Python) $(Crossgen2Directory)pre.py crossgen2 --core-root $(CoreRoot) --single %(Identity) - $(Python) $(Crossgen2Directory)test.py sod --scenario-name "Crossgen2 %(Identity) Size" --dirs ./crossgen.out/ - $(Python) $(Crossgen2Directory)post.py - - - - - - - 4:00 - - - - 4:00 - - - 4:00 - - - $(WorkItemDirectory) - $(Python) $(Crossgen2Directory)test.py crossgen2 --core-root $(CoreRoot) --composite $(Crossgen2Directory)framework-r2r.dll.rsp - 1:00 - - - 4:00 - - - 4:00 - - - \ No newline at end of file diff --git a/eng/common/performance/microbenchmarks.proj b/eng/common/performance/microbenchmarks.proj deleted file mode 100644 index 318ca5f1..00000000 --- a/eng/common/performance/microbenchmarks.proj +++ /dev/null @@ -1,144 +0,0 @@ - - - - %HELIX_CORRELATION_PAYLOAD%\performance\scripts\benchmarks_ci.py --csproj %HELIX_CORRELATION_PAYLOAD%\performance\$(TargetCsproj) - --dotnet-versions %DOTNET_VERSION% --cli-source-info args --cli-branch %PERFLAB_BRANCH% --cli-commit-sha %PERFLAB_HASH% --cli-repository https://github.com/%PERFLAB_REPO% --cli-source-timestamp %PERFLAB_BUILDTIMESTAMP% - py -3 - %HELIX_CORRELATION_PAYLOAD%\Core_Root\CoreRun.exe - %HELIX_CORRELATION_PAYLOAD%\Baseline_Core_Root\CoreRun.exe - - $(HelixPreCommands);call %HELIX_CORRELATION_PAYLOAD%\performance\tools\machine-setup.cmd;set PYTHONPATH=%HELIX_WORKITEM_PAYLOAD%\scripts%3B%HELIX_WORKITEM_PAYLOAD% - %HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts - %HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts_Baseline - %HELIX_CORRELATION_PAYLOAD%\performance\src\tools\ResultsComparer\ResultsComparer.csproj - %HELIX_CORRELATION_PAYLOAD%\performance\tools\dotnet\$(Architecture)\dotnet.exe - %25%25 - %HELIX_WORKITEM_ROOT%\testResults.xml - - - - $HELIX_CORRELATION_PAYLOAD - $(BaseDirectory)/performance - - - - $HELIX_WORKITEM_PAYLOAD - $(BaseDirectory) - - - - $(PerformanceDirectory)/scripts/benchmarks_ci.py --csproj $(PerformanceDirectory)/$(TargetCsproj) - --dotnet-versions $DOTNET_VERSION --cli-source-info args --cli-branch $PERFLAB_BRANCH --cli-commit-sha $PERFLAB_HASH --cli-repository https://github.com/$PERFLAB_REPO --cli-source-timestamp $PERFLAB_BUILDTIMESTAMP - python3 - $(BaseDirectory)/Core_Root/corerun - $(BaseDirectory)/Baseline_Core_Root/corerun - $(HelixPreCommands);chmod +x $(PerformanceDirectory)/tools/machine-setup.sh;. $(PerformanceDirectory)/tools/machine-setup.sh - $(BaseDirectory)/artifacts/BenchmarkDotNet.Artifacts - $(BaseDirectory)/artifacts/BenchmarkDotNet.Artifacts_Baseline - $(PerformanceDirectory)/src/tools/ResultsComparer/ResultsComparer.csproj - $(PerformanceDirectory)/tools/dotnet/$(Architecture)/dotnet - %25 - $HELIX_WORKITEM_ROOT/testResults.xml - - - - $(CliArguments) --wasm - - - - --corerun %HELIX_CORRELATION_PAYLOAD%\dotnet-mono\shared\Microsoft.NETCore.App\6.0.0\corerun.exe - - - --corerun $(BaseDirectory)/dotnet-mono/shared/Microsoft.NETCore.App/6.0.0/corerun - - - - --corerun $(CoreRun) - - - - --corerun $(BaselineCoreRun) - - - - $(Python) $(WorkItemCommand) --incremental no --architecture $(Architecture) -f $(_Framework) $(PerfLabArguments) - - - - $(WorkItemCommand) $(CliArguments) - - - - 2:30 - 0:15 - - - - - %(Identity) - - - - - 30 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - false - - - - - - $(WorkItemDirectory) - $(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" - $(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" - $(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults);$(FinalCommand) - $(WorkItemTimeout) - - - - - - $(WorkItemDirectory) - $(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument)" - $(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument)" - $(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults) - 4:00 - - - diff --git a/eng/common/performance/performance-setup.ps1 b/eng/common/performance/performance-setup.ps1 deleted file mode 100644 index c6f1045e..00000000 --- a/eng/common/performance/performance-setup.ps1 +++ /dev/null @@ -1,141 +0,0 @@ -Param( - [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, - [string] $CoreRootDirectory, - [string] $BaselineCoreRootDirectory, - [string] $Architecture="x64", - [string] $Framework="net5.0", - [string] $CompilationMode="Tiered", - [string] $Repository=$env:BUILD_REPOSITORY_NAME, - [string] $Branch=$env:BUILD_SOURCEBRANCH, - [string] $CommitSha=$env:BUILD_SOURCEVERSION, - [string] $BuildNumber=$env:BUILD_BUILDNUMBER, - [string] $RunCategories="Libraries Runtime", - [string] $Csproj="src\benchmarks\micro\MicroBenchmarks.csproj", - [string] $Kind="micro", - [switch] $LLVM, - [switch] $MonoInterpreter, - [switch] $MonoAOT, - [switch] $Internal, - [switch] $Compare, - [string] $MonoDotnet="", - [string] $Configurations="CompilationMode=$CompilationMode RunKind=$Kind" -) - -$RunFromPerformanceRepo = ($Repository -eq "dotnet/performance") -or ($Repository -eq "dotnet-performance") -$UseCoreRun = ($CoreRootDirectory -ne [string]::Empty) -$UseBaselineCoreRun = ($BaselineCoreRootDirectory -ne [string]::Empty) - -$PayloadDirectory = (Join-Path $SourceDirectory "Payload") -$PerformanceDirectory = (Join-Path $PayloadDirectory "performance") -$WorkItemDirectory = (Join-Path $SourceDirectory "workitem") -$ExtraBenchmarkDotNetArguments = "--iterationCount 1 --warmupCount 0 --invocationCount 1 --unrollFactor 1 --strategy ColdStart --stopOnFirstError true" -$Creator = $env:BUILD_DEFINITIONNAME -$PerfLabArguments = "" -$HelixSourcePrefix = "pr" - -$Queue = "Windows.10.Amd64.ClientRS4.DevEx.15.8.Open" - -# TODO: Implement a better logic to determine if Framework is .NET Core or >= .NET 5. -if ($Framework.StartsWith("netcoreapp") -or ($Framework -eq "net5.0")) { - $Queue = "Windows.10.Amd64.ClientRS5.Open" -} - -if ($Compare) { - $Queue = "Windows.10.Amd64.19H1.Tiger.Perf.Open" - $PerfLabArguments = "" - $ExtraBenchmarkDotNetArguments = "" -} - -if ($Internal) { - $Queue = "Windows.10.Amd64.19H1.Tiger.Perf" - $PerfLabArguments = "--upload-to-perflab-container" - $ExtraBenchmarkDotNetArguments = "" - $Creator = "" - $HelixSourcePrefix = "official" -} - -if($MonoInterpreter) -{ - $ExtraBenchmarkDotNetArguments = "--category-exclusion-filter NoInterpreter" -} - -if($MonoDotnet -ne "") -{ - $Configurations += " LLVM=$LLVM MonoInterpreter=$MonoInterpreter MonoAOT=$MonoAOT" - if($ExtraBenchmarkDotNetArguments -eq "") - { - #FIX ME: We need to block these tests as they don't run on mono for now - $ExtraBenchmarkDotNetArguments = "--exclusion-filter *Perf_Image* *Perf_NamedPipeStream*" - } - else - { - #FIX ME: We need to block these tests as they don't run on mono for now - $ExtraBenchmarkDotNetArguments += " --exclusion-filter *Perf_Image* *Perf_NamedPipeStream*" - } -} - -# FIX ME: This is a workaround until we get this from the actual pipeline -$CommonSetupArguments="--channel master --queue $Queue --build-number $BuildNumber --build-configs $Configurations --architecture $Architecture" -$SetupArguments = "--repository https://github.com/$Repository --branch $Branch --get-perf-hash --commit-sha $CommitSha $CommonSetupArguments" - - -if ($RunFromPerformanceRepo) { - $SetupArguments = "--perf-hash $CommitSha $CommonSetupArguments" - - robocopy $SourceDirectory $PerformanceDirectory /E /XD $PayloadDirectory $SourceDirectory\artifacts $SourceDirectory\.git -} -else { - git clone --branch master --depth 1 --quiet https://github.com/dotnet/performance $PerformanceDirectory -} - -if($MonoDotnet -ne "") -{ - $UsingMono = "true" - $MonoDotnetPath = (Join-Path $PayloadDirectory "dotnet-mono") - Move-Item -Path $MonoDotnet -Destination $MonoDotnetPath -} - -if ($UseCoreRun) { - $NewCoreRoot = (Join-Path $PayloadDirectory "Core_Root") - Move-Item -Path $CoreRootDirectory -Destination $NewCoreRoot -} -if ($UseBaselineCoreRun) { - $NewBaselineCoreRoot = (Join-Path $PayloadDirectory "Baseline_Core_Root") - Move-Item -Path $BaselineCoreRootDirectory -Destination $NewBaselineCoreRoot -} - -$DocsDir = (Join-Path $PerformanceDirectory "docs") -robocopy $DocsDir $WorkItemDirectory - -# Set variables that we will need to have in future steps -$ci = $true - -. "$PSScriptRoot\..\pipeline-logging-functions.ps1" - -# Directories -Write-PipelineSetVariable -Name 'PayloadDirectory' -Value "$PayloadDirectory" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'PerformanceDirectory' -Value "$PerformanceDirectory" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'WorkItemDirectory' -Value "$WorkItemDirectory" -IsMultiJobVariable $false - -# Script Arguments -Write-PipelineSetVariable -Name 'Python' -Value "py -3" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'ExtraBenchmarkDotNetArguments' -Value "$ExtraBenchmarkDotNetArguments" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'SetupArguments' -Value "$SetupArguments" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'PerfLabArguments' -Value "$PerfLabArguments" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'BDNCategories' -Value "$RunCategories" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'TargetCsproj' -Value "$Csproj" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'Kind' -Value "$Kind" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'Architecture' -Value "$Architecture" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'UseCoreRun' -Value "$UseCoreRun" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'UseBaselineCoreRun' -Value "$UseBaselineCoreRun" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'RunFromPerfRepo' -Value "$RunFromPerformanceRepo" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'Compare' -Value "$Compare" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'MonoDotnet' -Value "$UsingMono" -IsMultiJobVariable $false - -# Helix Arguments -Write-PipelineSetVariable -Name 'Creator' -Value "$Creator" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'Queue' -Value "$Queue" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'HelixSourcePrefix' -Value "$HelixSourcePrefix" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name '_BuildConfig' -Value "$Architecture.$Kind.$Framework" -IsMultiJobVariable $false - -exit 0 \ No newline at end of file diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh deleted file mode 100755 index cd4f233d..00000000 --- a/eng/common/performance/performance-setup.sh +++ /dev/null @@ -1,301 +0,0 @@ -#!/usr/bin/env bash - -source_directory=$BUILD_SOURCESDIRECTORY -core_root_directory= -baseline_core_root_directory= -architecture=x64 -framework=net5.0 -compilation_mode=tiered -repository=$BUILD_REPOSITORY_NAME -branch=$BUILD_SOURCEBRANCH -commit_sha=$BUILD_SOURCEVERSION -build_number=$BUILD_BUILDNUMBER -internal=false -compare=false -mono_dotnet= -kind="micro" -llvm=false -monointerpreter=false -monoaot=false -run_categories="Libraries Runtime" -csproj="src\benchmarks\micro\MicroBenchmarks.csproj" -configurations="CompliationMode=$compilation_mode RunKind=$kind" -run_from_perf_repo=false -use_core_run=true -use_baseline_core_run=true -using_mono=false -wasm_runtime_loc= -using_wasm=false -use_latest_dotnet=false - -while (($# > 0)); do - lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" - case $lowerI in - --sourcedirectory) - source_directory=$2 - shift 2 - ;; - --corerootdirectory) - core_root_directory=$2 - shift 2 - ;; - --baselinecorerootdirectory) - baseline_core_root_directory=$2 - shift 2 - ;; - --architecture) - architecture=$2 - shift 2 - ;; - --framework) - framework=$2 - shift 2 - ;; - --compilationmode) - compilation_mode=$2 - shift 2 - ;; - --repository) - repository=$2 - shift 2 - ;; - --branch) - branch=$2 - shift 2 - ;; - --commitsha) - commit_sha=$2 - shift 2 - ;; - --buildnumber) - build_number=$2 - shift 2 - ;; - --kind) - kind=$2 - configurations="CompilationMode=$compilation_mode RunKind=$kind" - shift 2 - ;; - --runcategories) - run_categories=$2 - shift 2 - ;; - --csproj) - csproj=$2 - shift 2 - ;; - --internal) - internal=true - shift 1 - ;; - --alpine) - alpine=true - shift 1 - ;; - --llvm) - llvm=true - shift 1 - ;; - --monointerpreter) - monointerpreter=true - shift 1 - ;; - --monoaot) - monoaot=true - shift 1 - ;; - --monodotnet) - mono_dotnet=$2 - shift 2 - ;; - --wasm) - wasm_runtime_loc=$2 - shift 2 - ;; - --compare) - compare=true - shift 1 - ;; - --configurations) - configurations=$2 - shift 2 - ;; - --latestdotnet) - use_latest_dotnet=true - shift 1 - ;; - *) - echo "Common settings:" - echo " --corerootdirectory Directory where Core_Root exists, if running perf testing with --corerun" - echo " --architecture Architecture of the testing being run" - echo " --configurations List of key=value pairs that will be passed to perf testing infrastructure." - echo " ex: --configurations \"CompilationMode=Tiered OptimzationLevel=PGO\"" - echo " --help Print help and exit" - echo "" - echo "Advanced settings:" - echo " --framework The framework to run, if not running in master" - echo " --compliationmode The compilation mode if not passing --configurations" - echo " --sourcedirectory The directory of the sources. Defaults to env:BUILD_SOURCESDIRECTORY" - echo " --repository The name of the repository in the / format. Defaults to env:BUILD_REPOSITORY_NAME" - echo " --branch The name of the branch. Defaults to env:BUILD_SOURCEBRANCH" - echo " --commitsha The commit sha1 to run against. Defaults to env:BUILD_SOURCEVERSION" - echo " --buildnumber The build number currently running. Defaults to env:BUILD_BUILDNUMBER" - echo " --csproj The relative path to the benchmark csproj whose tests should be run. Defaults to src\benchmarks\micro\MicroBenchmarks.csproj" - echo " --kind Related to csproj. The kind of benchmarks that should be run. Defaults to micro" - echo " --runcategories Related to csproj. Categories of benchmarks to run. Defaults to \"coreclr corefx\"" - echo " --internal If the benchmarks are running as an official job." - echo " --monodotnet Pass the path to the mono dotnet for mono performance testing." - echo " --wasm Path to the unpacked wasm runtime pack." - echo " --latestdotnet --dotnet-versions will not be specified. --dotnet-versions defaults to LKG version in global.json " - echo " --alpine Set for runs on Alpine" - echo "" - exit 0 - ;; - esac -done - -if [ "$repository" == "dotnet/performance" ] || [ "$repository" == "dotnet-performance" ]; then - run_from_perf_repo=true -fi - -if [ -z "$configurations" ]; then - configurations="CompilationMode=$compilation_mode" -fi - -if [ -z "$core_root_directory" ]; then - use_core_run=false -fi - -if [ -z "$baseline_core_root_directory" ]; then - use_baseline_core_run=false -fi - -payload_directory=$source_directory/Payload -performance_directory=$payload_directory/performance -workitem_directory=$source_directory/workitem -extra_benchmark_dotnet_arguments="--iterationCount 1 --warmupCount 0 --invocationCount 1 --unrollFactor 1 --strategy ColdStart --stopOnFirstError true" -perflab_arguments= -queue=Ubuntu.1804.Amd64.Open -creator=$BUILD_DEFINITIONNAME -helix_source_prefix="pr" - -if [[ "$compare" == true ]]; then - extra_benchmark_dotnet_arguments= - perflab_arguments= - - # No open queues for arm64 - if [[ "$architecture" = "arm64" ]]; then - echo "Compare not available for arm64" - exit 1 - fi - - queue=Ubuntu.1804.Amd64.Tiger.Perf.Open -fi - -if [[ "$internal" == true ]]; then - perflab_arguments="--upload-to-perflab-container" - helix_source_prefix="official" - creator= - extra_benchmark_dotnet_arguments= - - if [[ "$architecture" = "arm64" ]]; then - queue=Ubuntu.1804.Arm64.Perf - else - queue=Ubuntu.1804.Amd64.Tiger.Perf - fi - - if [[ "$alpine" = "true" ]]; then - queue=alpine.amd64.tiger.perf - fi -else - if [[ "$architecture" = "arm64" ]]; then - queue=ubuntu.1804.armarch.open - else - queue=Ubuntu.1804.Amd64.Open - fi - - if [[ "$alpine" = "true" ]]; then - queue=alpine.amd64.tiger.perf - fi -fi - -if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "false" ]]; then - configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot" - extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoMono" -fi - -if [[ "$wasm_runtime_loc" != "" ]]; then - configurations="CompilationMode=wasm RunKind=$kind" - extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter NoWASM NoMono" -fi - -if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "true" ]]; then - configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot" - extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter NoMono" -fi - -common_setup_arguments="--channel master --queue $queue --build-number $build_number --build-configs $configurations --architecture $architecture" -setup_arguments="--repository https://github.com/$repository --branch $branch --get-perf-hash --commit-sha $commit_sha $common_setup_arguments" - -if [[ "$run_from_perf_repo" = true ]]; then - payload_directory= - workitem_directory=$source_directory - performance_directory=$workitem_directory - setup_arguments="--perf-hash $commit_sha $common_setup_arguments" -else - git clone --branch master --depth 1 --quiet https://github.com/dotnet/performance $performance_directory - - docs_directory=$performance_directory/docs - mv $docs_directory $workitem_directory -fi - -if [[ "$wasm_runtime_loc" != "" ]]; then - using_wasm=true - wasm_dotnet_path=$payload_directory/dotnet-wasm - mv $wasm_runtime_loc $wasm_dotnet_path - extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --wasmMainJS \$HELIX_CORRELATION_PAYLOAD/dotnet-wasm/runtime-test.js --wasmEngine /home/helixbot/.jsvu/v8 --customRuntimePack \$HELIX_CORRELATION_PAYLOAD/dotnet-wasm" -fi - -if [[ "$mono_dotnet" != "" ]]; then - using_mono=true - mono_dotnet_path=$payload_directory/dotnet-mono - mv $mono_dotnet $mono_dotnet_path -fi - -if [[ "$use_core_run" = true ]]; then - new_core_root=$payload_directory/Core_Root - mv $core_root_directory $new_core_root -fi - -if [[ "$use_baseline_core_run" = true ]]; then - new_baseline_core_root=$payload_directory/Baseline_Core_Root - mv $baseline_core_root_directory $new_baseline_core_root -fi - -ci=true - -_script_dir=$(pwd)/eng/common -. "$_script_dir/pipeline-logging-functions.sh" - -# Make sure all of our variables are available for future steps -Write-PipelineSetVariable -name "UseCoreRun" -value "$use_core_run" -is_multi_job_variable false -Write-PipelineSetVariable -name "UseBaselineCoreRun" -value "$use_baseline_core_run" -is_multi_job_variable false -Write-PipelineSetVariable -name "Architecture" -value "$architecture" -is_multi_job_variable false -Write-PipelineSetVariable -name "PayloadDirectory" -value "$payload_directory" -is_multi_job_variable false -Write-PipelineSetVariable -name "PerformanceDirectory" -value "$performance_directory" -is_multi_job_variable false -Write-PipelineSetVariable -name "WorkItemDirectory" -value "$workitem_directory" -is_multi_job_variable false -Write-PipelineSetVariable -name "Queue" -value "$queue" -is_multi_job_variable false -Write-PipelineSetVariable -name "SetupArguments" -value "$setup_arguments" -is_multi_job_variable false -Write-PipelineSetVariable -name "Python" -value "python3" -is_multi_job_variable false -Write-PipelineSetVariable -name "PerfLabArguments" -value "$perflab_arguments" -is_multi_job_variable false -Write-PipelineSetVariable -name "ExtraBenchmarkDotNetArguments" -value "$extra_benchmark_dotnet_arguments" -is_multi_job_variable false -Write-PipelineSetVariable -name "BDNCategories" -value "$run_categories" -is_multi_job_variable false -Write-PipelineSetVariable -name "TargetCsproj" -value "$csproj" -is_multi_job_variable false -Write-PipelineSetVariable -name "RunFromPerfRepo" -value "$run_from_perf_repo" -is_multi_job_variable false -Write-PipelineSetVariable -name "Creator" -value "$creator" -is_multi_job_variable false -Write-PipelineSetVariable -name "HelixSourcePrefix" -value "$helix_source_prefix" -is_multi_job_variable false -Write-PipelineSetVariable -name "Kind" -value "$kind" -is_multi_job_variable false -Write-PipelineSetVariable -name "_BuildConfig" -value "$architecture.$kind.$framework" -is_multi_job_variable false -Write-PipelineSetVariable -name "Compare" -value "$compare" -is_multi_job_variable false -Write-PipelineSetVariable -name "MonoDotnet" -value "$using_mono" -is_multi_job_variable false -Write-PipelineSetVariable -name "WasmDotnet" -value "$using_wasm" -is_multi_job_variable false diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1 index 599a1172..2427ca6b 100644 --- a/eng/common/post-build/publish-using-darc.ps1 +++ b/eng/common/post-build/publish-using-darc.ps1 @@ -60,7 +60,7 @@ try { --id $buildId ` --publishing-infra-version $PublishingInfraVersion ` --default-channels ` - --source-branch master ` + --source-branch main ` --azdev-pat $AzdoToken ` --bar-uri $MaestroApiEndPoint ` --password $MaestroToken ` diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1 index 1c46f7b6..85c89861 100644 --- a/eng/common/post-build/sourcelink-validation.ps1 +++ b/eng/common/post-build/sourcelink-validation.ps1 @@ -14,7 +14,9 @@ param( $global:RepoFiles = @{} # Maximum number of jobs to run in parallel -$MaxParallelJobs = 6 +$MaxParallelJobs = 16 + +$MaxRetries = 5 # Wait time between check for system load $SecondsBetweenLoadChecks = 10 @@ -29,7 +31,10 @@ $ValidatePackage = { # Ensure input file exist if (!(Test-Path $PackagePath)) { Write-Host "Input file does not exist: $PackagePath" - return 1 + return [pscustomobject]@{ + result = 1 + packagePath = $PackagePath + } } # Extensions for which we'll look for SourceLink information @@ -59,7 +64,10 @@ $ValidatePackage = { # We ignore resource DLLs if ($FileName.EndsWith('.resources.dll')) { - return + return [pscustomobject]@{ + result = 0 + packagePath = $PackagePath + } } [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true) @@ -91,36 +99,49 @@ $ValidatePackage = { $Status = 200 $Cache = $using:RepoFiles - if ( !($Cache.ContainsKey($FilePath)) ) { - try { - $Uri = $Link -as [System.URI] - - # Only GitHub links are valid - if ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match 'github' -or $Uri.Host -match 'githubusercontent')) { - $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode + $totalRetries = 0 + + while ($totalRetries -lt $using:MaxRetries) { + if ( !($Cache.ContainsKey($FilePath)) ) { + try { + $Uri = $Link -as [System.URI] + + # Only GitHub links are valid + if ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match 'github' -or $Uri.Host -match 'githubusercontent')) { + $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode + } + else { + # If it's not a github link, we want to break out of the loop and not retry. + $Status = 0 + $totalRetries = $using:MaxRetries + } } - else { + catch { + Write-Host $_ $Status = 0 } } - catch { - write-host $_ - $Status = 0 - } - } - if ($Status -ne 200) { - if ($NumFailedLinks -eq 0) { - if ($FailedFiles.Value -eq 0) { - Write-Host + if ($Status -ne 200) { + $totalRetries++ + + if ($totalRetries -ge $using:MaxRetries) { + if ($NumFailedLinks -eq 0) { + if ($FailedFiles.Value -eq 0) { + Write-Host + } + + Write-Host "`tFile $RealPath has broken links:" + } + + Write-Host "`t`tFailed to retrieve $Link" + + $NumFailedLinks++ } - - Write-Host "`tFile $RealPath has broken links:" } - - Write-Host "`t`tFailed to retrieve $Link" - - $NumFailedLinks++ + else { + break + } } } } @@ -136,7 +157,7 @@ $ValidatePackage = { } } catch { - + Write-Host $_ } finally { $zip.Dispose() @@ -220,6 +241,7 @@ function ValidateSourceLinkLinks { # Process each NuGet package in parallel Get-ChildItem "$InputPath\*.symbols.nupkg" | ForEach-Object { + Write-Host "Starting $($_.FullName)" Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName | Out-Null $NumJobs = @(Get-Job -State 'Running').Count @@ -267,6 +289,10 @@ function InstallSourcelinkCli { try { InstallSourcelinkCli + foreach ($Job in @(Get-Job)) { + Remove-Job -Id $Job.Id + } + ValidateSourceLinkLinks } catch { diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 index 99bf28cd..a5af041b 100644 --- a/eng/common/post-build/symbols-validation.ps1 +++ b/eng/common/post-build/symbols-validation.ps1 @@ -1,13 +1,14 @@ param( - [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored - [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation - [Parameter(Mandatory=$true)][string] $DotnetSymbolVersion, # Version of dotnet symbol to use - [Parameter(Mandatory=$false)][switch] $ContinueOnError, # If we should keep checking symbols after an error - [Parameter(Mandatory=$false)][switch] $Clean # Clean extracted symbols directory after checking symbols + [Parameter(Mandatory = $true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored + [Parameter(Mandatory = $true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation + [Parameter(Mandatory = $true)][string] $DotnetSymbolVersion, # Version of dotnet symbol to use + [Parameter(Mandatory = $false)][switch] $CheckForWindowsPdbs, # If we should check for the existence of windows pdbs in addition to portable PDBs + [Parameter(Mandatory = $false)][switch] $ContinueOnError, # If we should keep checking symbols after an error + [Parameter(Mandatory = $false)][switch] $Clean # Clean extracted symbols directory after checking symbols ) # Maximum number of jobs to run in parallel -$MaxParallelJobs = 6 +$MaxParallelJobs = 16 # Max number of retries $MaxRetry = 5 @@ -19,9 +20,15 @@ $SecondsBetweenLoadChecks = 10 Set-Variable -Name "ERROR_BADEXTRACT" -Option Constant -Value -1 Set-Variable -Name "ERROR_FILEDOESNOTEXIST" -Option Constant -Value -2 +$WindowsPdbVerificationParam = "" +if ($CheckForWindowsPdbs) { + $WindowsPdbVerificationParam = "--windows-pdbs" +} + $CountMissingSymbols = { param( - [string] $PackagePath # Path to a NuGet package + [string] $PackagePath, # Path to a NuGet package + [string] $WindowsPdbVerificationParam # If we should check for the existence of windows pdbs in addition to portable PDBs ) . $using:PSScriptRoot\..\tools.ps1 @@ -34,7 +41,7 @@ $CountMissingSymbols = { if (!(Test-Path $PackagePath)) { Write-PipelineTaskError "Input file does not exist: $PackagePath" return [pscustomobject]@{ - result = $using:ERROR_FILEDOESNOTEXIST + result = $using:ERROR_FILEDOESNOTEXIST packagePath = $PackagePath } } @@ -57,24 +64,25 @@ $CountMissingSymbols = { Write-Host "Something went wrong extracting $PackagePath" Write-Host $_ return [pscustomobject]@{ - result = $using:ERROR_BADEXTRACT + result = $using:ERROR_BADEXTRACT packagePath = $PackagePath } } Get-ChildItem -Recurse $ExtractPath | - Where-Object {$RelevantExtensions -contains $_.Extension} | - ForEach-Object { - $FileName = $_.FullName - if ($FileName -Match '\\ref\\') { - Write-Host "`t Ignoring reference assembly file " $FileName - return - } + Where-Object { $RelevantExtensions -contains $_.Extension } | + ForEach-Object { + $FileName = $_.FullName + if ($FileName -Match '\\ref\\') { + Write-Host "`t Ignoring reference assembly file " $FileName + return + } - $FirstMatchingSymbolDescriptionOrDefault = { + $FirstMatchingSymbolDescriptionOrDefault = { param( - [string] $FullPath, # Full path to the module that has to be checked - [string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols + [string] $FullPath, # Full path to the module that has to be checked + [string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols + [string] $WindowsPdbVerificationParam, # Parameter to pass to potential check for windows-pdbs. [string] $SymbolsPath ) @@ -99,15 +107,16 @@ $CountMissingSymbols = { # DWARF file for a .dylib $DylibDwarf = $SymbolPath.Replace($Extension, '.dylib.dwarf') - + $dotnetSymbolExe = "$env:USERPROFILE\.dotnet\tools" $dotnetSymbolExe = Resolve-Path "$dotnetSymbolExe\dotnet-symbol.exe" $totalRetries = 0 while ($totalRetries -lt $using:MaxRetry) { + # Save the output and get diagnostic output - $output = & $dotnetSymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath --diagnostics | Out-String + $output = & $dotnetSymbolExe --symbols --modules $WindowsPdbVerificationParam $TargetServerParam $FullPath -o $SymbolsPath --diagnostics | Out-String if (Test-Path $PdbPath) { return 'PDB' @@ -124,42 +133,50 @@ $CountMissingSymbols = { elseif (Test-Path $SymbolPath) { return 'Module' } - elseif ($output.Contains("503 Service Unavailable")) { - # If we got a 503 error, we should retry. + else + { $totalRetries++ } - else { - return $null - } } return $null } - $SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault $FileName '--microsoft-symbol-server' $SymbolsPath - $SymbolsOnSymWeb = & $FirstMatchingSymbolDescriptionOrDefault $FileName '--internal-server' $SymbolsPath - - Write-Host -NoNewLine "`t Checking file " $FileName "... " + $FileGuid = New-Guid + $ExpandedSymbolsPath = Join-Path -Path $SymbolsPath -ChildPath $FileGuid + + $SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault ` + -FullPath $FileName ` + -TargetServerParam '--microsoft-symbol-server' ` + -SymbolsPath "$ExpandedSymbolsPath-msdl" ` + -WindowsPdbVerificationParam $WindowsPdbVerificationParam + $SymbolsOnSymWeb = & $FirstMatchingSymbolDescriptionOrDefault ` + -FullPath $FileName ` + -TargetServerParam '--internal-server' ` + -SymbolsPath "$ExpandedSymbolsPath-symweb" ` + -WindowsPdbVerificationParam $WindowsPdbVerificationParam + + Write-Host -NoNewLine "`t Checking file " $FileName "... " - if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) { - Write-Host "Symbols found on MSDL ($SymbolsOnMSDL) and SymWeb ($SymbolsOnSymWeb)" + if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) { + Write-Host "Symbols found on MSDL ($SymbolsOnMSDL) and SymWeb ($SymbolsOnSymWeb)" + } + else { + $MissingSymbols++ + + if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) { + Write-Host 'No symbols found on MSDL or SymWeb!' } else { - $MissingSymbols++ - - if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) { - Write-Host 'No symbols found on MSDL or SymWeb!' + if ($SymbolsOnMSDL -eq $null) { + Write-Host 'No symbols found on MSDL!' } else { - if ($SymbolsOnMSDL -eq $null) { - Write-Host 'No symbols found on MSDL!' - } - else { - Write-Host 'No symbols found on SymWeb!' - } + Write-Host 'No symbols found on SymWeb!' } } } + } if ($using:Clean) { Remove-Item $ExtractPath -Recurse -Force @@ -168,16 +185,16 @@ $CountMissingSymbols = { Pop-Location return [pscustomobject]@{ - result = $MissingSymbols - packagePath = $PackagePath - } + result = $MissingSymbols + packagePath = $PackagePath + } } function CheckJobResult( - $result, - $packagePath, - [ref]$DupedSymbols, - [ref]$TotalFailures) { + $result, + $packagePath, + [ref]$DupedSymbols, + [ref]$TotalFailures) { if ($result -eq $ERROR_BADEXTRACT) { Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$packagePath has duplicated symbol files" $DupedSymbols.Value++ @@ -200,6 +217,7 @@ function CheckSymbolsAvailable { Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue } + $TotalPackages = 0 $TotalFailures = 0 $DupedSymbols = 0 @@ -222,7 +240,9 @@ function CheckSymbolsAvailable { return } - Start-Job -ScriptBlock $CountMissingSymbols -ArgumentList $FullName | Out-Null + $TotalPackages++ + + Start-Job -ScriptBlock $CountMissingSymbols -ArgumentList @($FullName,$WindowsPdbVerificationParam) | Out-Null $NumJobs = @(Get-Job -State 'Running').Count @@ -247,11 +267,11 @@ function CheckSymbolsAvailable { if ($TotalFailures -gt 0 -or $DupedSymbols -gt 0) { if ($TotalFailures -gt 0) { - Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Symbols missing for $TotalFailures packages" + Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Symbols missing for $TotalFailures/$TotalPackages packages" } if ($DupedSymbols -gt 0) { - Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$DupedSymbols packages had duplicated symbol files" + Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$DupedSymbols/$TotalPackages packages had duplicated symbol files and could not be extracted" } ExitWithExitCode 1 diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index f55c43c6..b1bca63a 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -34,7 +34,7 @@ function Print-Usage() { function Build([string]$target) { $logSuffix = if ($target -eq 'Execute') { '' } else { ".$target" } $log = Join-Path $LogDir "$task$logSuffix.binlog" - $outputPath = Join-Path $ToolsetDir "$task\\" + $outputPath = Join-Path $ToolsetDir "$task\" MSBuild $taskProject ` /bl:$log ` @@ -53,7 +53,7 @@ try { } if ($task -eq "") { - Write-PipelineTelemetryError -Category 'Build' -Message "Missing required parameter '-task '" -ForegroundColor Red + Write-PipelineTelemetryError -Category 'Build' -Message "Missing required parameter '-task '" Print-Usage ExitWithExitCode 1 } @@ -64,7 +64,7 @@ try { $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty } if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { - $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.8.0-preview3" -MemberType NoteProperty + $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.10.0-preview2" -MemberType NoteProperty } if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true @@ -78,7 +78,7 @@ try { $taskProject = GetSdkTaskProject $task if (!(Test-Path $taskProject)) { - Write-PipelineTelemetryError -Category 'Build' -Message "Unknown task: $task" -ForegroundColor Red + Write-PipelineTelemetryError -Category 'Build' -Message "Unknown task: $task" ExitWithExitCode 1 } diff --git a/eng/common/sdl/configure-sdl-tool.ps1 b/eng/common/sdl/configure-sdl-tool.ps1 new file mode 100644 index 00000000..4999c307 --- /dev/null +++ b/eng/common/sdl/configure-sdl-tool.ps1 @@ -0,0 +1,109 @@ +Param( + [string] $GuardianCliLocation, + [string] $WorkingDirectory, + [string] $TargetDirectory, + [string] $GdnFolder, + # The list of Guardian tools to configure. For each object in the array: + # - If the item is a [hashtable], it must contain these entries: + # - Name = The tool name as Guardian knows it. + # - Scenario = (Optional) Scenario-specific name for this configuration entry. It must be unique + # among all tool entries with the same Name. + # - Args = (Optional) Array of Guardian tool configuration args, like '@("Target > C:\temp")' + # - If the item is a [string] $v, it is treated as '@{ Name="$v" }' + [object[]] $ToolsList, + [string] $GuardianLoggerLevel='Standard', + # Optional: Additional params to add to any tool using CredScan. + [string[]] $CrScanAdditionalRunConfigParams, + # Optional: Additional params to add to any tool using PoliCheck. + [string[]] $PoliCheckAdditionalRunConfigParams +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version 2.0 +$disableConfigureToolsetImport = $true +$global:LASTEXITCODE = 0 + +try { + # `tools.ps1` checks $ci to perform some actions. Since the SDL + # scripts don't necessarily execute in the same agent that run the + # build.ps1/sh script this variable isn't automatically set. + $ci = $true + . $PSScriptRoot\..\tools.ps1 + + # Normalize tools list: all in [hashtable] form with defined values for each key. + $ToolsList = $ToolsList | + ForEach-Object { + if ($_ -is [string]) { + $_ = @{ Name = $_ } + } + + if (-not ($_['Scenario'])) { $_.Scenario = "" } + if (-not ($_['Args'])) { $_.Args = @() } + $_ + } + + Write-Host "List of tools to configure:" + $ToolsList | ForEach-Object { $_ | Out-String | Write-Host } + + # We store config files in the r directory of .gdn + $gdnConfigPath = Join-Path $GdnFolder 'r' + $ValidPath = Test-Path $GuardianCliLocation + + if ($ValidPath -eq $False) + { + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Invalid Guardian CLI Location." + ExitWithExitCode 1 + } + + foreach ($tool in $ToolsList) { + # Put together the name and scenario to make a unique key. + $toolConfigName = $tool.Name + if ($tool.Scenario) { + $toolConfigName += "_" + $tool.Scenario + } + + Write-Host "=== Configuring $toolConfigName..." + + $gdnConfigFile = Join-Path $gdnConfigPath "$toolConfigName-configure.gdnconfig" + + # For some tools, add default and automatic args. + if ($tool.Name -eq 'credscan') { + if ($targetDirectory) { + $tool.Args += "TargetDirectory < $TargetDirectory" + } + $tool.Args += "OutputType < pre" + $tool.Args += $CrScanAdditionalRunConfigParams + } elseif ($tool.Name -eq 'policheck') { + if ($targetDirectory) { + $tool.Args += "Target < $TargetDirectory" + } + $tool.Args += $PoliCheckAdditionalRunConfigParams + } + + # Create variable pointing to the args array directly so we can use splat syntax later. + $toolArgs = $tool.Args + + # Configure the tool. If args array is provided or the current tool has some default arguments + # defined, add "--args" and splat each element on the end. Arg format is "{Arg id} < {Value}", + # one per parameter. Doc page for "guardian configure": + # https://dev.azure.com/securitytools/SecurityIntegration/_wiki/wikis/Guardian/1395/configure + Exec-BlockVerbosely { + & $GuardianCliLocation configure ` + --working-directory $WorkingDirectory ` + --tool $tool.Name ` + --output-path $gdnConfigFile ` + --logger-level $GuardianLoggerLevel ` + --noninteractive ` + --force ` + $(if ($toolArgs) { "--args" }) @toolArgs + Exit-IfNZEC "Sdl" + } + + Write-Host "Created '$toolConfigName' configuration file: $gdnConfigFile" + } +} +catch { + Write-Host $_.ScriptStackTrace + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ + ExitWithExitCode 1 +} diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1 index 81b729f7..1157151f 100644 --- a/eng/common/sdl/execute-all-sdl-tools.ps1 +++ b/eng/common/sdl/execute-all-sdl-tools.ps1 @@ -7,8 +7,17 @@ Param( [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, # Required: the directory where source files are located [string] $ArtifactsDirectory = (Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY ('artifacts')), # Required: the directory where build artifacts are located [string] $AzureDevOpsAccessToken, # Required: access token for dnceng; should be provided via KeyVault - [string[]] $SourceToolsList, # Optional: list of SDL tools to run on source code - [string[]] $ArtifactToolsList, # Optional: list of SDL tools to run on built artifacts + + # Optional: list of SDL tools to run on source code. See 'configure-sdl-tool.ps1' for tools list + # format. + [object[]] $SourceToolsList, + # Optional: list of SDL tools to run on built artifacts. See 'configure-sdl-tool.ps1' for tools + # list format. + [object[]] $ArtifactToolsList, + # Optional: list of SDL tools to run without automatically specifying a target directory. See + # 'configure-sdl-tool.ps1' for tools list format. + [object[]] $CustomToolsList, + [bool] $TsaPublish=$False, # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs. [string] $TsaBranchName=$env:BUILD_SOURCEBRANCH, # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs. [string] $TsaRepositoryName=$env:BUILD_REPOSITORY_NAME, # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs. @@ -32,7 +41,7 @@ try { $ErrorActionPreference = 'Stop' Set-StrictMode -Version 2.0 $disableConfigureToolsetImport = $true - $LASTEXITCODE = 0 + $global:LASTEXITCODE = 0 # `tools.ps1` checks $ci to perform some actions. Since the SDL # scripts don't necessarily execute in the same agent that run the @@ -63,13 +72,16 @@ try { ExitWithExitCode 1 } - & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel + Exec-BlockVerbosely { + & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel + } $gdnFolder = Join-Path $workingDirectory '.gdn' if ($TsaOnboard) { if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) { - Write-Host "$guardianCliLocation tsa-onboard --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel" - & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel + Exec-BlockVerbosely { + & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel + } if ($LASTEXITCODE -ne 0) { Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-onboard failed with exit code $LASTEXITCODE." ExitWithExitCode $LASTEXITCODE @@ -80,11 +92,41 @@ try { } } - if ($ArtifactToolsList -and $ArtifactToolsList.Count -gt 0) { - & $(Join-Path $PSScriptRoot 'run-sdl.ps1') -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams + # Configure a list of tools with a default target directory. Populates the ".gdn/r" directory. + function Configure-ToolsList([object[]] $tools, [string] $targetDirectory) { + if ($tools -and $tools.Count -gt 0) { + Exec-BlockVerbosely { + & $(Join-Path $PSScriptRoot 'configure-sdl-tool.ps1') ` + -GuardianCliLocation $guardianCliLocation ` + -WorkingDirectory $workingDirectory ` + -TargetDirectory $targetDirectory ` + -GdnFolder $gdnFolder ` + -ToolsList $tools ` + -AzureDevOpsAccessToken $AzureDevOpsAccessToken ` + -GuardianLoggerLevel $GuardianLoggerLevel ` + -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams ` + -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams + if ($BreakOnFailure) { + Exit-IfNZEC "Sdl" + } + } + } } - if ($SourceToolsList -and $SourceToolsList.Count -gt 0) { - & $(Join-Path $PSScriptRoot 'run-sdl.ps1') -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams + + # Configure Artifact and Source tools with default Target directories. + Configure-ToolsList $ArtifactToolsList $ArtifactsDirectory + Configure-ToolsList $SourceToolsList $SourceDirectory + # Configure custom tools with no default Target directory. + Configure-ToolsList $CustomToolsList $null + + # At this point, all tools are configured in the ".gdn" directory. Run them all in a single call. + # (If we used "run" multiple times, each run would overwrite data from earlier runs.) + Exec-BlockVerbosely { + & $(Join-Path $PSScriptRoot 'run-sdl.ps1') ` + -GuardianCliLocation $guardianCliLocation ` + -WorkingDirectory $workingDirectory ` + -UpdateBaseline $UpdateBaseline ` + -GdnFolder $gdnFolder } if ($TsaPublish) { @@ -92,8 +134,9 @@ try { if (-not $TsaRepositoryName) { $TsaRepositoryName = "$($Repository)-$($BranchName)" } - Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel" - & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel + Exec-BlockVerbosely { + & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel + } if ($LASTEXITCODE -ne 0) { Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-publish failed with exit code $LASTEXITCODE." ExitWithExitCode $LASTEXITCODE @@ -106,7 +149,11 @@ try { if ($BreakOnFailure) { Write-Host "Failing the build in case of breaking results..." - & $guardianCliLocation break + Exec-BlockVerbosely { + & $guardianCliLocation break --working-directory $workingDirectory --logger-level $GuardianLoggerLevel + } + } else { + Write-Host "Letting the build pass even if there were breaking results..." } } catch { diff --git a/eng/common/sdl/extract-artifact-archives.ps1 b/eng/common/sdl/extract-artifact-archives.ps1 new file mode 100644 index 00000000..68da4fbf --- /dev/null +++ b/eng/common/sdl/extract-artifact-archives.ps1 @@ -0,0 +1,63 @@ +# This script looks for each archive file in a directory and extracts it into the target directory. +# For example, the file "$InputPath/bin.tar.gz" extracts to "$ExtractPath/bin.tar.gz.extracted/**". +# Uses the "tar" utility added to Windows 10 / Windows 2019 that supports tar.gz and zip. +param( + # Full path to directory where archives are stored. + [Parameter(Mandatory=$true)][string] $InputPath, + # Full path to directory to extract archives into. May be the same as $InputPath. + [Parameter(Mandatory=$true)][string] $ExtractPath +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version 2.0 + +$disableConfigureToolsetImport = $true + +try { + # `tools.ps1` checks $ci to perform some actions. Since the SDL + # scripts don't necessarily execute in the same agent that run the + # build.ps1/sh script this variable isn't automatically set. + $ci = $true + . $PSScriptRoot\..\tools.ps1 + + Measure-Command { + $jobs = @() + + # Find archive files for non-Windows and Windows builds. + $archiveFiles = @( + Get-ChildItem (Join-Path $InputPath "*.tar.gz") + Get-ChildItem (Join-Path $InputPath "*.zip") + ) + + foreach ($targzFile in $archiveFiles) { + $jobs += Start-Job -ScriptBlock { + $file = $using:targzFile + $fileName = [System.IO.Path]::GetFileName($file) + $extractDir = Join-Path $using:ExtractPath "$fileName.extracted" + + New-Item $extractDir -ItemType Directory -Force | Out-Null + + Write-Host "Extracting '$file' to '$extractDir'..." + + # Pipe errors to stdout to prevent PowerShell detecting them and quitting the job early. + # This type of quit skips the catch, so we wouldn't be able to tell which file triggered the + # error. Save output so it can be stored in the exception string along with context. + $output = tar -xf $file -C $extractDir 2>&1 + # Handle NZEC manually rather than using Exit-IfNZEC: we are in a background job, so we + # don't have access to the outer scope. + if ($LASTEXITCODE -ne 0) { + throw "Error extracting '$file': non-zero exit code ($LASTEXITCODE). Output: '$output'" + } + + Write-Host "Extracted to $extractDir" + } + } + + Receive-Job $jobs -Wait + } +} +catch { + Write-Host $_ + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ + ExitWithExitCode 1 +} diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1 index 1fe92711..3ac1d92b 100644 --- a/eng/common/sdl/init-sdl.ps1 +++ b/eng/common/sdl/init-sdl.ps1 @@ -10,7 +10,7 @@ Param( $ErrorActionPreference = 'Stop' Set-StrictMode -Version 2.0 $disableConfigureToolsetImport = $true -$LASTEXITCODE = 0 +$global:LASTEXITCODE = 0 # `tools.ps1` checks $ci to perform some actions. Since the SDL # scripts don't necessarily execute in the same agent that run the diff --git a/eng/common/sdl/run-sdl.ps1 b/eng/common/sdl/run-sdl.ps1 index fe95ab35..2eac8c78 100644 --- a/eng/common/sdl/run-sdl.ps1 +++ b/eng/common/sdl/run-sdl.ps1 @@ -1,19 +1,15 @@ Param( [string] $GuardianCliLocation, [string] $WorkingDirectory, - [string] $TargetDirectory, [string] $GdnFolder, - [string[]] $ToolsList, [string] $UpdateBaseline, - [string] $GuardianLoggerLevel='Standard', - [string[]] $CrScanAdditionalRunConfigParams, - [string[]] $PoliCheckAdditionalRunConfigParams + [string] $GuardianLoggerLevel='Standard' ) $ErrorActionPreference = 'Stop' Set-StrictMode -Version 2.0 $disableConfigureToolsetImport = $true -$LASTEXITCODE = 0 +$global:LASTEXITCODE = 0 try { # `tools.ps1` checks $ci to perform some actions. Since the SDL @@ -23,7 +19,6 @@ try { . $PSScriptRoot\..\tools.ps1 # We store config files in the r directory of .gdn - Write-Host $ToolsList $gdnConfigPath = Join-Path $GdnFolder 'r' $ValidPath = Test-Path $GuardianCliLocation @@ -33,37 +28,18 @@ try { ExitWithExitCode 1 } - $configParam = @('--config') - - foreach ($tool in $ToolsList) { - $gdnConfigFile = Join-Path $gdnConfigPath "$tool-configure.gdnconfig" - Write-Host $tool - # We have to manually configure tools that run on source to look at the source directory only - if ($tool -eq 'credscan') { - Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory < $TargetDirectory `" `" OutputType < pre `" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})" - & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory < $TargetDirectory " "OutputType < pre" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams}) - if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian configure for $tool failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE - } - } - if ($tool -eq 'policheck') { - Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target < $TargetDirectory `" $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})" - & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target < $TargetDirectory " $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams}) - if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian configure for $tool failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE - } - } - - $configParam+=$gdnConfigFile - } - - Write-Host "$GuardianCliLocation run --working-directory $WorkingDirectory --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam" - & $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam - if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian run for $ToolsList using $configParam failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE + $gdnConfigFiles = Get-ChildItem $gdnConfigPath -Recurse -Include '*.gdnconfig' + Write-Host "Discovered Guardian config files:" + $gdnConfigFiles | Out-String | Write-Host + + Exec-BlockVerbosely { + & $GuardianCliLocation run ` + --working-directory $WorkingDirectory ` + --baseline mainbaseline ` + --update-baseline $UpdateBaseline ` + --logger-level $GuardianLoggerLevel ` + --config @gdnConfigFiles + Exit-IfNZEC "Sdl" } } catch { diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index 53c10022..69eb6784 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -2,17 +2,41 @@ parameters: enable: 'false' # Whether the SDL validation job should execute or not overrideParameters: '' # Optional: to override values for parameters. additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")' + # Optional: if specified, restore and use this version of Guardian instead of the default. + overrideGuardianVersion: '' + # Optional: if true, publish the '.gdn' folder as a pipeline artifact. This can help with in-depth + # diagnosis of problems with specific tool configurations. + publishGuardianDirectoryToPipeline: false + # The script to run to execute all SDL tools. Use this if you want to use a script to define SDL + # parameters rather than relying on YAML. It may be better to use a local script, because you can + # reproduce results locally without piecing together a command based on the YAML. + executeAllSdlToolsScript: 'eng/common/sdl/execute-all-sdl-tools.ps1' # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named # 'continueOnError', the parameter value is not correctly picked up. # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter sdlContinueOnError: false # optional: determines whether to continue the build if the step errors; - downloadArtifacts: true # optional: determines if the artifacts should be dowloaded + # optional: determines if build artifacts should be downloaded. + downloadArtifacts: true + # optional: determines if this job should search the directory of downloaded artifacts for + # 'tar.gz' and 'zip' archive files and extract them before running SDL validation tasks. + extractArchiveArtifacts: false dependsOn: '' # Optional: dependencies of the job artifactNames: '' # Optional: patterns supplied to DownloadBuildArtifacts # Usage: # artifactNames: # - 'BlobArtifacts' # - 'Artifacts_Windows_NT_Release' + # Optional: download a list of pipeline artifacts. 'downloadArtifacts' controls build artifacts, + # not pipeline artifacts, so doesn't affect the use of this parameter. + pipelineArtifactNames: [] + # Optional: location and ID of the AzDO build that the build/pipeline artifacts should be + # downloaded from. By default, uses runtime expressions to decide based on the variables set by + # the 'setupMaestroVars' dependency. Overriding this parameter is necessary if SDL tasks are + # running without Maestro++/BAR involved, or to download artifacts from a specific existing build + # to iterate quickly on SDL changes. + AzDOProjectName: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + AzDOPipelineId: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + AzDOBuildId: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] jobs: - job: Run_SDL @@ -22,16 +46,29 @@ jobs: variables: - group: DotNet-VSTS-Bot - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + value: ${{ parameters.AzDOProjectName }} - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + value: ${{ parameters.AzDOPipelineId }} - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + value: ${{ parameters.AzDOBuildId }} + # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in + # sync with the packages.config file. + - name: DefaultGuardianVersion + value: 0.53.3 + - name: GuardianVersion + value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} + - name: GuardianPackagesConfigFile + value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config pool: - name: Hosted VS2017 + # To extract archives (.tar.gz, .zip), we need access to "tar", added in Windows 10/2019. + ${{ if eq(parameters.extractArchiveArtifacts, 'false') }}: + name: Hosted VS2017 + ${{ if ne(parameters.extractArchiveArtifacts, 'false') }}: + vmImage: windows-2019 steps: - checkout: self clean: true + - ${{ if ne(parameters.downloadArtifacts, 'false')}}: - ${{ if ne(parameters.artifactNames, '') }}: - ${{ each artifactName in parameters.artifactNames }}: @@ -45,6 +82,7 @@ jobs: buildId: $(AzDOBuildId) artifactName: ${{ artifactName }} downloadPath: $(Build.ArtifactStagingDirectory)\artifacts + checkDownloadedFiles: true - ${{ if eq(parameters.artifactNames, '') }}: - task: DownloadBuildArtifacts@0 displayName: Download Build Artifacts @@ -57,16 +95,52 @@ jobs: downloadType: specific files itemPattern: "**" downloadPath: $(Build.ArtifactStagingDirectory)\artifacts + checkDownloadedFiles: true + + - ${{ each artifactName in parameters.pipelineArtifactNames }}: + - task: DownloadPipelineArtifact@2 + displayName: Download Pipeline Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: ${{ artifactName }} + downloadPath: $(Build.ArtifactStagingDirectory)\artifacts + checkDownloadedFiles: true + - powershell: eng/common/sdl/extract-artifact-packages.ps1 -InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts displayName: Extract Blob Artifacts continueOnError: ${{ parameters.sdlContinueOnError }} + - powershell: eng/common/sdl/extract-artifact-packages.ps1 -InputPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts displayName: Extract Package Artifacts continueOnError: ${{ parameters.sdlContinueOnError }} + + - ${{ if ne(parameters.extractArchiveArtifacts, 'false') }}: + - powershell: eng/common/sdl/extract-artifact-archives.ps1 + -InputPath $(Build.ArtifactStagingDirectory)\artifacts + -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts + displayName: Extract Archive Artifacts + continueOnError: ${{ parameters.sdlContinueOnError }} + + - ${{ if ne(parameters.overrideGuardianVersion, '') }}: + - powershell: | + $content = Get-Content $(GuardianPackagesConfigFile) + + Write-Host "packages.config content was:`n$content" + + $content = $content.Replace('$(DefaultGuardianVersion)', '$(GuardianVersion)') + $content | Set-Content $(GuardianPackagesConfigFile) + + Write-Host "packages.config content updated to:`n$content" + displayName: Use overridden Guardian version ${{ parameters.overrideGuardianVersion }} + - task: NuGetToolInstaller@1 displayName: 'Install NuGet.exe' - task: NuGetCommand@2 @@ -77,15 +151,35 @@ jobs: nugetConfigPath: $(Build.SourcesDirectory)\eng\common\sdl\NuGet.config externalFeedCredentials: GuardianConnect restoreDirectory: $(Build.SourcesDirectory)\.packages + - ${{ if ne(parameters.overrideParameters, '') }}: - - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 ${{ parameters.overrideParameters }} + - powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }} displayName: Execute SDL continueOnError: ${{ parameters.sdlContinueOnError }} - ${{ if eq(parameters.overrideParameters, '') }}: - - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 - -GuardianPackageName Microsoft.Guardian.Cli.0.53.3 + - powershell: ${{ parameters.executeAllSdlToolsScript }} + -GuardianPackageName Microsoft.Guardian.Cli.$(GuardianVersion) -NugetPackageDirectory $(Build.SourcesDirectory)\.packages -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) ${{ parameters.additionalParameters }} displayName: Execute SDL continueOnError: ${{ parameters.sdlContinueOnError }} + + - ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}: + # We want to publish the Guardian results and configuration for easy diagnosis. However, the + # '.gdn' dir is a mix of configuration, results, extracted dependencies, and Guardian default + # tooling files. Some of these files are large and aren't useful during an investigation, so + # exclude them by simply deleting them before publishing. (As of writing, there is no documented + # way to selectively exclude a dir from the pipeline artifact publish task.) + - task: DeleteFiles@1 + displayName: Delete Guardian dependencies to avoid uploading + inputs: + SourceFolder: $(Agent.BuildDirectory)/.gdn + Contents: | + c + i + condition: succeededOrFailed() + - publish: $(Agent.BuildDirectory)/.gdn + artifact: GuardianConfiguration + displayName: Publish GuardianConfiguration + condition: succeededOrFailed() diff --git a/eng/common/templates/job/onelocbuild.yml b/eng/common/templates/job/onelocbuild.yml index 928a70cd..e8bc77d2 100644 --- a/eng/common/templates/job/onelocbuild.yml +++ b/eng/common/templates/job/onelocbuild.yml @@ -11,11 +11,17 @@ parameters: SourcesDirectory: $(Build.SourcesDirectory) CreatePr: true + AutoCompletePr: false + UseLfLineEndings: true UseCheckedInLocProjectJson: false LanguageSet: VS_Main_Languages LclSource: lclFilesInRepo LclPackageId: '' RepoType: gitHub + GitHubOrg: dotnet + MirrorRepo: '' + MirrorBranch: main + condition: '' jobs: - job: OneLocBuild @@ -43,23 +49,32 @@ jobs: filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1 arguments: $(_GenerateLocProjectArguments) displayName: Generate LocProject.json + condition: ${{ parameters.condition }} - task: OneLocBuild@2 displayName: OneLocBuild env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) inputs: - locProj: Localize/LocProject.json + locProj: eng/Localize/LocProject.json outDir: $(Build.ArtifactStagingDirectory) lclSource: ${{ parameters.LclSource }} lclPackageId: ${{ parameters.LclPackageId }} isCreatePrSelected: ${{ parameters.CreatePr }} + ${{ if eq(parameters.CreatePr, true) }}: + isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} + isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} packageSourceAuth: patAuth patVariable: ${{ parameters.CeapexPat }} ${{ if eq(parameters.RepoType, 'gitHub') }}: repoType: ${{ parameters.RepoType }} gitHubPatVariable: "${{ parameters.GithubPat }}" - condition: always() + ${{ if ne(parameters.MirrorRepo, '') }}: + isMirrorRepoSelected: true + gitHubOrganization: ${{ parameters.GitHubOrg }} + mirrorRepo: ${{ parameters.MirrorRepo }} + mirrorBranch: ${{ parameters.MirrorBranch }} + condition: ${{ parameters.condition }} - task: PublishBuildArtifacts@1 displayName: Publish Localization Files @@ -67,12 +82,12 @@ jobs: PathtoPublish: '$(Build.ArtifactStagingDirectory)/loc' PublishLocation: Container ArtifactName: Loc - condition: always() + condition: ${{ parameters.condition }} - task: PublishBuildArtifacts@1 displayName: Publish LocProject.json inputs: - PathtoPublish: '$(Build.SourcesDirectory)/Localize/' + PathtoPublish: '$(Build.SourcesDirectory)/eng/Localize/' PublishLocation: Container ArtifactName: Loc - condition: always() \ No newline at end of file + condition: ${{ parameters.condition }} \ No newline at end of file diff --git a/eng/common/templates/job/performance.yml b/eng/common/templates/job/performance.yml deleted file mode 100644 index f877fd7a..00000000 --- a/eng/common/templates/job/performance.yml +++ /dev/null @@ -1,95 +0,0 @@ -parameters: - steps: [] # optional -- any additional steps that need to happen before pulling down the performance repo and sending the performance benchmarks to helix (ie building your repo) - variables: [] # optional -- list of additional variables to send to the template - jobName: '' # required -- job name - displayName: '' # optional -- display name for the job. Will use jobName if not passed - pool: '' # required -- name of the Build pool - container: '' # required -- name of the container - osGroup: '' # required -- operating system for the job - extraSetupParameters: '' # optional -- extra arguments to pass to the setup script - frameworks: ['netcoreapp3.0'] # optional -- list of frameworks to run against - continueOnError: 'false' # optional -- determines whether to continue the build if the step errors - dependsOn: '' # optional -- dependencies of the job - timeoutInMinutes: 320 # optional -- timeout for the job - enableTelemetry: false # optional -- enable for telemetry - -jobs: -- template: ../jobs/jobs.yml - parameters: - dependsOn: ${{ parameters.dependsOn }} - enableTelemetry: ${{ parameters.enableTelemetry }} - enablePublishBuildArtifacts: true - continueOnError: ${{ parameters.continueOnError }} - - jobs: - - job: '${{ parameters.jobName }}' - - ${{ if ne(parameters.displayName, '') }}: - displayName: '${{ parameters.displayName }}' - ${{ if eq(parameters.displayName, '') }}: - displayName: '${{ parameters.jobName }}' - - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - - variables: - - - ${{ each variable in parameters.variables }}: - - ${{ if ne(variable.name, '') }}: - - name: ${{ variable.name }} - value: ${{ variable.value }} - - ${{ if ne(variable.group, '') }}: - - group: ${{ variable.group }} - - - IsInternal: '' - - HelixApiAccessToken: '' - - HelixPreCommand: '' - - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ${{ if eq( parameters.osGroup, 'Windows_NT') }}: - - HelixPreCommand: 'set "PERFLAB_UPLOAD_TOKEN=$(PerfCommandUploadToken)"' - - IsInternal: -Internal - - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - HelixPreCommand: 'export PERFLAB_UPLOAD_TOKEN="$(PerfCommandUploadTokenLinux)"' - - IsInternal: --internal - - - group: DotNet-HelixApi-Access - - group: dotnet-benchview - - workspace: - clean: all - pool: - ${{ parameters.pool }} - container: ${{ parameters.container }} - strategy: - matrix: - ${{ each framework in parameters.frameworks }}: - ${{ framework }}: - _Framework: ${{ framework }} - steps: - - checkout: self - clean: true - # Run all of the steps to setup repo - - ${{ each step in parameters.steps }}: - - ${{ step }} - - powershell: $(Build.SourcesDirectory)\eng\common\performance\performance-setup.ps1 $(IsInternal) -Framework $(_Framework) ${{ parameters.extraSetupParameters }} - displayName: Performance Setup (Windows) - condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - - script: $(Build.SourcesDirectory)/eng/common/performance/performance-setup.sh $(IsInternal) --framework $(_Framework) ${{ parameters.extraSetupParameters }} - displayName: Performance Setup (Unix) - condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - - script: $(Python) $(PerformanceDirectory)/scripts/ci_setup.py $(SetupArguments) - displayName: Run ci setup script - # Run perf testing in helix - - template: /eng/common/templates/steps/perf-send-to-helix.yml - parameters: - HelixSource: '$(HelixSourcePrefix)/$(Build.Repository.Name)/$(Build.SourceBranch)' # sources must start with pr/, official/, prodcon/, or agent/ - HelixType: 'test/performance/$(Kind)/$(_Framework)/$(Architecture)' - HelixAccessToken: $(HelixApiAccessToken) - HelixTargetQueues: $(Queue) - HelixPreCommands: $(HelixPreCommand) - Creator: $(Creator) - WorkItemTimeout: 4:00 # 4 hours - WorkItemDirectory: '$(WorkItemDirectory)' # WorkItemDirectory can not be empty, so we send it some docs to keep it happy - CorrelationPayloadDirectory: '$(PayloadDirectory)' # it gets checked out to a folder with shorter path than WorkItemDirectory so we can avoid file name too long exceptions \ No newline at end of file diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 66ef7364..3b9e2524 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -52,6 +52,7 @@ jobs: inputs: artifactName: AssetManifests downloadPath: '$(Build.StagingDirectory)/Download' + checkDownloadedFiles: true condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml index aad41464..5023d36d 100644 --- a/eng/common/templates/job/source-build.yml +++ b/eng/common/templates/job/source-build.yml @@ -15,6 +15,9 @@ parameters: # nonPortable: false # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than # linux-x64), and compiling against distro-provided packages rather than portable ones. + # skipPublishValidation: false + # Disables publishing validation. By default, a check is performed to ensure no packages are + # published by source-build. # container: '' # A container to use. Runs in docker. # pool: {} diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml index c002a2b1..b58d4236 100644 --- a/eng/common/templates/job/source-index-stage1.yml +++ b/eng/common/templates/job/source-index-stage1.yml @@ -1,15 +1,19 @@ parameters: runAsPublic: false - sourceIndexPackageVersion: 1.0.1-20210225.1 + sourceIndexPackageVersion: 1.0.1-20210614.1 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" preSteps: [] binlogPath: artifacts/log/Debug/Build.binlog pool: vmImage: vs2017-win2016 + condition: '' + dependsOn: '' jobs: - job: SourceIndexStage1 + dependsOn: ${{ parameters.dependsOn }} + condition: ${{ parameters.condition }} variables: - name: SourceIndexPackageVersion value: ${{ parameters.sourceIndexPackageVersion }} diff --git a/eng/common/templates/phases/publish-build-assets.yml b/eng/common/templates/phases/publish-build-assets.yml index a0a80742..4e51e472 100644 --- a/eng/common/templates/phases/publish-build-assets.yml +++ b/eng/common/templates/phases/publish-build-assets.yml @@ -20,6 +20,7 @@ phases: inputs: artifactName: AssetManifests downloadPath: '$(Build.StagingDirectory)/Download' + checkDownloadedFiles: true condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} - task: AzureKeyVault@1 diff --git a/eng/common/templates/post-build/channels/generic-internal-channel.yml b/eng/common/templates/post-build/channels/generic-internal-channel.yml index 7ae52559..8990dfc8 100644 --- a/eng/common/templates/post-build/channels/generic-internal-channel.yml +++ b/eng/common/templates/post-build/channels/generic-internal-channel.yml @@ -40,6 +40,9 @@ stages: pool: vmImage: 'windows-2019' steps: + - script: echo "##vso[task.logissue type=warning]Going forward, v2 Arcade publishing is no longer supported. Please read https://github.com/dotnet/arcade/blob/main/Documentation/CorePackages/Publishing.md for details, then contact dnceng if you have further questions." + displayName: Warn about v2 Arcade Publishing Usage + # This is necessary whenever we want to publish/restore to an AzDO private feed - task: NuGetAuthenticate@0 displayName: 'Authenticate to AzDO Feeds' @@ -58,6 +61,7 @@ stages: PdbArtifacts/** BlobArtifacts/** downloadPath: '$(Build.ArtifactStagingDirectory)' + checkDownloadedFiles: true # This is necessary whenever we want to publish/restore to an AzDO private feed # Since sdk-task.ps1 tries to restore packages we need to do this authentication here @@ -109,6 +113,9 @@ stages: pool: vmImage: 'windows-2019' steps: + - script: echo "##vso[task.logissue type=warning]Going forward, v2 Arcade publishing is no longer supported. Please read https://github.com/dotnet/arcade/blob/main/Documentation/CorePackages/Publishing.md for details, then contact dnceng if you have further questions." + displayName: Warn about v2 Arcade Publishing Usage + - task: DownloadBuildArtifacts@0 displayName: Download Build Assets continueOnError: true @@ -124,6 +131,7 @@ stages: BlobArtifacts/** AssetManifests/** downloadPath: '$(Build.ArtifactStagingDirectory)' + checkDownloadedFiles: true - task: NuGetToolInstaller@1 displayName: 'Install NuGet.exe' diff --git a/eng/common/templates/post-build/channels/generic-public-channel.yml b/eng/common/templates/post-build/channels/generic-public-channel.yml index 6cf39dbb..3220c6a4 100644 --- a/eng/common/templates/post-build/channels/generic-public-channel.yml +++ b/eng/common/templates/post-build/channels/generic-public-channel.yml @@ -42,6 +42,9 @@ stages: pool: vmImage: 'windows-2019' steps: + - script: echo "##vso[task.logissue type=warning]Going forward, v2 Arcade publishing is no longer supported. Please read https://github.com/dotnet/arcade/blob/main/Documentation/CorePackages/Publishing.md for details, then contact dnceng if you have further questions." + displayName: Warn about v2 Arcade Publishing Usage + - task: DownloadBuildArtifacts@0 displayName: Download Build Assets continueOnError: true @@ -56,6 +59,7 @@ stages: PdbArtifacts/** BlobArtifacts/** downloadPath: '$(Build.ArtifactStagingDirectory)' + checkDownloadedFiles: true # This is necessary whenever we want to publish/restore to an AzDO private feed # Since sdk-task.ps1 tries to restore packages we need to do this authentication here @@ -108,6 +112,9 @@ stages: pool: vmImage: 'windows-2019' steps: + - script: echo "##vso[task.logissue type=warning]Going forward, v2 Arcade publishing is no longer supported. Please read https://github.com/dotnet/arcade/blob/main/Documentation/CorePackages/Publishing.md for details, then contact dnceng if you have further questions." + displayName: Warn about v2 Arcade Publishing Usage + - task: DownloadBuildArtifacts@0 displayName: Download Build Assets continueOnError: true @@ -123,6 +130,7 @@ stages: BlobArtifacts/** AssetManifests/** downloadPath: '$(Build.ArtifactStagingDirectory)' + checkDownloadedFiles: true - task: NuGetToolInstaller@1 displayName: 'Install NuGet.exe' diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index c84ac55e..4f79cf0f 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -117,6 +117,7 @@ stages: pipeline: $(AzDOPipelineId) buildId: $(AzDOBuildId) artifactName: PackageArtifacts + checkDownloadedFiles: true - task: PowerShell@2 displayName: Validate @@ -149,6 +150,7 @@ stages: pipeline: $(AzDOPipelineId) buildId: $(AzDOBuildId) artifactName: PackageArtifacts + checkDownloadedFiles: true itemPattern: | ** !**/Microsoft.SourceBuild.Intermediate.*.nupkg @@ -205,6 +207,7 @@ stages: pipeline: $(AzDOPipelineId) buildId: $(AzDOBuildId) artifactName: BlobArtifacts + checkDownloadedFiles: true - task: PowerShell@2 displayName: Validate diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml index d0cbfb6c..4a22b2e6 100644 --- a/eng/common/templates/post-build/setup-maestro-vars.yml +++ b/eng/common/templates/post-build/setup-maestro-vars.yml @@ -18,6 +18,7 @@ jobs: inputs: buildType: current artifactName: ReleaseConfigs + checkDownloadedFiles: true - task: PowerShell@2 name: setReleaseVars diff --git a/eng/common/templates/steps/perf-send-to-helix.yml b/eng/common/templates/steps/perf-send-to-helix.yml deleted file mode 100644 index a468e92c..00000000 --- a/eng/common/templates/steps/perf-send-to-helix.yml +++ /dev/null @@ -1,50 +0,0 @@ -# Please remember to update the documentation if you make changes to these parameters! -parameters: - ProjectFile: '' # required -- project file that specifies the helix workitems - HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ - HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' - HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number - HelixTargetQueues: '' # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues - HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group - HelixPreCommands: '' # optional -- commands to run before Helix work item execution - HelixPostCommands: '' # optional -- commands to run after Helix work item execution - WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects - CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload - IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion - DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json - DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json - EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control - WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." - Creator: '' # optional -- if the build is external, use this to specify who is sending the job - DisplayNamePrefix: 'Send job to Helix' # optional -- rename the beginning of the displayName of the steps in AzDO - condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() - continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false - osGroup: '' # required -- operating system for the job - - -steps: -- template: /eng/pipelines/common/templates/runtimes/send-to-helix-inner-step.yml - parameters: - osGroup: ${{ parameters.osGroup }} - sendParams: $(Build.SourcesDirectory)/eng/common/performance/${{ parameters.ProjectFile }} /restore /t:Test /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/SendToHelix.binlog - displayName: ${{ parameters.DisplayNamePrefix }} - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} - environment: - BuildConfig: $(_BuildConfig) - HelixSource: ${{ parameters.HelixSource }} - HelixType: ${{ parameters.HelixType }} - HelixBuild: ${{ parameters.HelixBuild }} - HelixTargetQueues: ${{ parameters.HelixTargetQueues }} - HelixAccessToken: ${{ parameters.HelixAccessToken }} - HelixPreCommands: ${{ parameters.HelixPreCommands }} - HelixPostCommands: ${{ parameters.HelixPostCommands }} - WorkItemDirectory: ${{ parameters.WorkItemDirectory }} - CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} - IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} - DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} - DotNetCliVersion: ${{ parameters.DotNetCliVersion }} - EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} - WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - Creator: ${{ parameters.Creator }} - SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml index bb5f1a92..cd02ae16 100644 --- a/eng/common/templates/steps/send-to-helix.yml +++ b/eng/common/templates/steps/send-to-helix.yml @@ -18,8 +18,8 @@ parameters: XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion - DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases-index.json - DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases-index.json + DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json + DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml index 8e336b7d..705b7a1c 100644 --- a/eng/common/templates/steps/source-build.yml +++ b/eng/common/templates/steps/source-build.yml @@ -29,15 +29,26 @@ steps: officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)' fi + internalRuntimeDownloadArgs= + if [ '$(dotnetclimsrc-read-sas-token-base64)' != '$''(dotnetclimsrc-read-sas-token-base64)' ]; then + internalRuntimeDownloadArgs='--runtimesourcefeed https://dotnetclimsrc.blob.core.windows.net/dotnet --runtimesourcefeedkey $(dotnetclimsrc-read-sas-token-base64)' + fi + targetRidArgs= if [ '${{ parameters.platform.targetRID }}' != '' ]; then targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' fi + publishArgs= + if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then + publishArgs='--publish' + fi + ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ --configuration $buildConfig \ - --restore --build --pack --publish \ + --restore --build --pack $publishArgs -bl \ $officialBuildArgs \ + $internalRuntimeDownloadArgs \ $targetRidArgs \ /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ /p:ArcadeBuildFromSource=true diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index be5ce4a2..5d526c74 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -42,12 +42,15 @@ [bool]$useInstalledDotNetCli = if (Test-Path variable:useInstalledDotNetCli) { $useInstalledDotNetCli } else { $true } # Enable repos to use a particular version of the on-line dotnet-install scripts. -# default URL: https://dot.net/v1/dotnet-install.ps1 +# default URL: https://dotnet.microsoft.com/download/dotnet/scripts/v1/dotnet-install.ps1 [string]$dotnetInstallScriptVersion = if (Test-Path variable:dotnetInstallScriptVersion) { $dotnetInstallScriptVersion } else { 'v1' } # True to use global NuGet cache instead of restoring packages to repository-local directory. [bool]$useGlobalNuGetCache = if (Test-Path variable:useGlobalNuGetCache) { $useGlobalNuGetCache } else { !$ci } +# True to exclude prerelease versions Visual Studio during build +[bool]$excludePrereleaseVS = if (Test-Path variable:excludePrereleaseVS) { $excludePrereleaseVS } else { $false } + # An array of names of processes to stop on script exit if prepareMachine is true. $processesToStopOnExit = if (Test-Path variable:processesToStopOnExit) { $processesToStopOnExit } else { @('msbuild', 'dotnet', 'vbcscompiler') } @@ -103,6 +106,46 @@ function Exec-Process([string]$command, [string]$commandArgs) { } } +# Take the given block, print it, print what the block probably references from the current set of +# variables using low-effort string matching, then run the block. +# +# This is intended to replace the pattern of manually copy-pasting a command, wrapping it in quotes, +# and printing it using "Write-Host". The copy-paste method is more readable in build logs, but less +# maintainable and less reliable. It is easy to make a mistake and modify the command without +# properly updating the "Write-Host" line, resulting in misleading build logs. The probability of +# this mistake makes the pattern hard to trust when it shows up in build logs. Finding the bug in +# existing source code can also be difficult, because the strings are not aligned to each other and +# the line may be 300+ columns long. +# +# By removing the need to maintain two copies of the command, Exec-BlockVerbosely avoids the issues. +# +# In Bash (or any posix-like shell), "set -x" prints usable verbose output automatically. +# "Set-PSDebug" appears to be similar at first glance, but unfortunately, it isn't very useful: it +# doesn't print any info about the variables being used by the command, which is normally the +# interesting part to diagnose. +function Exec-BlockVerbosely([scriptblock] $block) { + Write-Host "--- Running script block:" + $blockString = $block.ToString().Trim() + Write-Host $blockString + + Write-Host "--- List of variables that might be used:" + # For each variable x in the environment, check the block for a reference to x via simple "$x" or + # "@x" syntax. This doesn't detect other ways to reference variables ("${x}" nor "$variable:x", + # among others). It only catches what this function was originally written for: simple + # command-line commands. + $variableTable = Get-Variable | + Where-Object { + $blockString.Contains("`$$($_.Name)") -or $blockString.Contains("@$($_.Name)") + } | + Format-Table -AutoSize -HideTableHeaders -Wrap | + Out-String + Write-Host $variableTable.Trim() + + Write-Host "--- Executing:" + & $block + Write-Host "--- Done running script block!" +} + # createSdkLocationFile parameter enables a file being generated under the toolset directory # which writes the sdk's location into. This is only necessary for cmd --> powershell invocations # as dot sourcing isn't possible. @@ -141,7 +184,7 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { # Use dotnet installation specified in DOTNET_INSTALL_DIR if it contains the required SDK version, # otherwise install the dotnet CLI and SDK to repo local .dotnet directory to avoid potential permission issues. - if ((-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -ne $null) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) { + if ((-not $globalJsonHasRuntimes) -and (-not [string]::IsNullOrEmpty($env:DOTNET_INSTALL_DIR)) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) { $dotnetRoot = $env:DOTNET_INSTALL_DIR } else { $dotnetRoot = Join-Path $RepoRoot '.dotnet' @@ -190,38 +233,42 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { return $global:_DotNetInstallDir = $dotnetRoot } +function Retry($downloadBlock, $maxRetries = 5) { + $retries = 1 + + while($true) { + try { + & $downloadBlock + break + } + catch { + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_ + } + + if (++$retries -le $maxRetries) { + $delayInSeconds = [math]::Pow(2, $retries) - 1 # Exponential backoff + Write-Host "Retrying. Waiting for $delayInSeconds seconds before next attempt ($retries of $maxRetries)." + Start-Sleep -Seconds $delayInSeconds + } + else { + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unable to download file in $maxRetries attempts." + break + } + + } +} + function GetDotNetInstallScript([string] $dotnetRoot) { $installScript = Join-Path $dotnetRoot 'dotnet-install.ps1' if (!(Test-Path $installScript)) { Create-Directory $dotnetRoot $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit + $uri = "https://dotnet.microsoft.com/download/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.ps1" - $maxRetries = 5 - $retries = 1 - - $uri = "https://dot.net/$dotnetInstallScriptVersion/dotnet-install.ps1" - - while($true) { - try { - Write-Host "GET $uri" - Invoke-WebRequest $uri -OutFile $installScript - break - } - catch { - Write-Host "Failed to download '$uri'" - Write-Error $_.Exception.Message -ErrorAction Continue - } - - if (++$retries -le $maxRetries) { - $delayInSeconds = [math]::Pow(2, $retries) - 1 # Exponential backoff - Write-Host "Retrying. Waiting for $delayInSeconds seconds before next attempt ($retries of $maxRetries)." - Start-Sleep -Seconds $delayInSeconds - } - else { - throw "Unable to download file in $maxRetries attempts." - } - - } + Retry({ + Write-Host "GET $uri" + Invoke-WebRequest $uri -OutFile $installScript + }) } return $installScript @@ -305,8 +352,8 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = # If the version of msbuild is going to be xcopied, # use this version. Version matches a package here: - # https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=16.8.0-preview3&view=overview - $defaultXCopyMSBuildVersion = '16.8.0-preview3' + # https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=16.10.0-preview2&view=overview + $defaultXCopyMSBuildVersion = '16.10.0-preview2' if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } $vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { $vsMinVersionReqdStr } @@ -371,7 +418,16 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = } $msbuildVersionDir = if ([int]$vsMajorVersion -lt 16) { "$vsMajorVersion.0" } else { "Current" } - return $global:_MSBuildExe = Join-Path $vsInstallDir "MSBuild\$msbuildVersionDir\Bin\msbuild.exe" + + $local:BinFolder = Join-Path $vsInstallDir "MSBuild\$msbuildVersionDir\Bin" + $local:Prefer64bit = if (Get-Member -InputObject $vsRequirements -Name 'Prefer64bit') { $vsRequirements.Prefer64bit } else { $false } + if ($local:Prefer64bit -and (Test-Path(Join-Path $local:BinFolder "amd64"))) { + $global:_MSBuildExe = Join-Path $local:BinFolder "amd64\msbuild.exe" + } else { + $global:_MSBuildExe = Join-Path $local:BinFolder "msbuild.exe" + } + + return $global:_MSBuildExe } function InitializeVisualStudioEnvironmentVariables([string] $vsInstallDir, [string] $vsMajorVersion) { @@ -400,9 +456,13 @@ function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { } Create-Directory $packageDir + Write-Host "Downloading $packageName $packageVersion" $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit - Invoke-WebRequest "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/flat2/$packageName/$packageVersion/$packageName.$packageVersion.nupkg" -OutFile $packagePath + Retry({ + Invoke-WebRequest "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/flat2/$packageName/$packageVersion/$packageName.$packageVersion.nupkg" -OutFile $packagePath + }) + Unzip $packagePath $packageDir } @@ -439,31 +499,17 @@ function LocateVisualStudio([object]$vsRequirements = $null){ if (!(Test-Path $vsWhereExe)) { Create-Directory $vsWhereDir Write-Host 'Downloading vswhere' - $maxRetries = 5 - $retries = 1 - - while($true) { - try { - Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe - break - } - catch{ - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_ - } - - if (++$retries -le $maxRetries) { - $delayInSeconds = [math]::Pow(2, $retries) - 1 # Exponential backoff - Write-Host "Retrying. Waiting for $delayInSeconds seconds before next attempt ($retries of $maxRetries)." - Start-Sleep -Seconds $delayInSeconds - } - else { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unable to download file in $maxRetries attempts." - } - } + Retry({ + Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe + }) } if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } - $args = @('-latest', '-prerelease', '-format', 'json', '-requires', 'Microsoft.Component.MSBuild', '-products', '*') + $args = @('-latest', '-format', 'json', '-requires', 'Microsoft.Component.MSBuild', '-products', '*') + + if (!$excludePrereleaseVS) { + $args += '-prerelease' + } if (Get-Member -InputObject $vsRequirements -Name 'version') { $args += '-version' @@ -489,7 +535,13 @@ function LocateVisualStudio([object]$vsRequirements = $null){ function InitializeBuildTool() { if (Test-Path variable:global:_BuildTool) { - return $global:_BuildTool + # If the requested msbuild parameters do not match, clear the cached variables. + if($global:_BuildTool.Contains('ExcludePrereleaseVS') -and $global:_BuildTool.ExcludePrereleaseVS -ne $excludePrereleaseVS) { + Remove-Item variable:global:_BuildTool + Remove-Item variable:global:_MSBuildExe + } else { + return $global:_BuildTool + } } if (-not $msbuildEngine) { @@ -517,7 +569,7 @@ function InitializeBuildTool() { ExitWithExitCode 1 } - $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472" } + $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472"; ExcludePrereleaseVS = $excludePrereleaseVS } } else { Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unexpected value of -msbuildEngine: '$msbuildEngine'." ExitWithExitCode 1 @@ -542,7 +594,7 @@ function GetDefaultMSBuildEngine() { function GetNuGetPackageCachePath() { if ($env:NUGET_PACKAGES -eq $null) { - # Use local cache on CI to ensure deterministic build. + # Use local cache on CI to ensure deterministic build. # Avoid using the http cache as workaround for https://github.com/NuGet/Home/issues/3116 # use global cache in dev builds to avoid cost of downloading packages. # For directory normalization, see also: https://github.com/NuGet/Home/issues/7968 @@ -620,6 +672,17 @@ function ExitWithExitCode([int] $exitCode) { exit $exitCode } +# Check if $LASTEXITCODE is a nonzero exit code (NZEC). If so, print a Azure Pipeline error for +# diagnostics, then exit the script with the $LASTEXITCODE. +function Exit-IfNZEC([string] $category = "General") { + Write-Host "Exit code $LASTEXITCODE" + if ($LASTEXITCODE -ne 0) { + $message = "Last command failed with exit code $LASTEXITCODE." + Write-PipelineTelemetryError -Force -Category $category -Message $message + ExitWithExitCode $LASTEXITCODE + } +} + function Stop-Processes() { Write-Host 'Killing running build processes...' foreach ($processName in $processesToStopOnExit) { @@ -699,7 +762,10 @@ function MSBuild-Core() { } foreach ($arg in $args) { - if ($arg -ne $null -and $arg.Trim() -ne "") { + if ($null -ne $arg -and $arg.Trim() -ne "") { + if ($arg.EndsWith('\')) { + $arg = $arg + "\" + } $cmdArgs += " `"$arg`"" } } @@ -771,7 +837,7 @@ function Get-Darc($version) { . $PSScriptRoot\pipeline-logging-functions.ps1 -$RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..') +$RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..\') $EngRoot = Resolve-Path (Join-Path $PSScriptRoot '..') $ArtifactsDir = Join-Path $RepoRoot 'artifacts' $ToolsetDir = Join-Path $ArtifactsDir 'toolset' diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 5fad1846..828119be 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -54,7 +54,7 @@ warn_as_error=${warn_as_error:-true} use_installed_dotnet_cli=${use_installed_dotnet_cli:-true} # Enable repos to use a particular version of the on-line dotnet-install scripts. -# default URL: https://dot.net/v1/dotnet-install.sh +# default URL: https://dotnet.microsoft.com/download/dotnet/scripts/v1/dotnet-install.sh dotnetInstallScriptVersion=${dotnetInstallScriptVersion:-'v1'} # True to use global NuGet cache instead of restoring packages to repository-local directory. @@ -262,7 +262,7 @@ function with_retries { function GetDotNetInstallScript { local root=$1 local install_script="$root/dotnet-install.sh" - local install_script_url="https://dot.net/$dotnetInstallScriptVersion/dotnet-install.sh" + local install_script_url="https://dotnet.microsoft.com/download/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.sh" if [[ ! -a "$install_script" ]]; then mkdir -p "$root" @@ -485,13 +485,14 @@ _script_dir=`dirname "$_ResolvePath"` eng_root=`cd -P "$_script_dir/.." && pwd` repo_root=`cd -P "$_script_dir/../.." && pwd` -artifacts_dir="$repo_root/artifacts" +repo_root="${repo_root}/" +artifacts_dir="${repo_root}artifacts" toolset_dir="$artifacts_dir/toolset" -tools_dir="$repo_root/.tools" +tools_dir="${repo_root}.tools" log_dir="$artifacts_dir/log/$configuration" temp_dir="$artifacts_dir/tmp/$configuration" -global_json_file="$repo_root/global.json" +global_json_file="${repo_root}global.json" # determine if global.json contains a "runtimes" entry global_json_has_runtimes=false if command -v jq &> /dev/null; then @@ -504,7 +505,7 @@ fi # HOME may not be defined in some scenarios, but it is required by NuGet if [[ -z $HOME ]]; then - export HOME="$repo_root/artifacts/.home/" + export HOME="${repo_root}artifacts/.home/" mkdir -p "$HOME" fi diff --git a/global.json b/global.json index 86995a1f..69cb8fb5 100644 --- a/global.json +++ b/global.json @@ -1,8 +1,8 @@ { "tools": { - "dotnet": "5.0.100" + "dotnet": "6.0.100-preview.6.21355.2" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21125.5" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21373.11" } } diff --git a/src/Microsoft.Build.StandardCI/Microsoft.Build.StandardCI.csproj b/src/Microsoft.Build.StandardCI/Microsoft.Build.StandardCI.csproj index 6f3f0405..7dd6f956 100644 --- a/src/Microsoft.Build.StandardCI/Microsoft.Build.StandardCI.csproj +++ b/src/Microsoft.Build.StandardCI/Microsoft.Build.StandardCI.csproj @@ -1,6 +1,6 @@  - net472;net5.0 + net472;net6.0 true true true diff --git a/src/Microsoft.Build.Tasks.Git.UnitTests/Microsoft.Build.Tasks.Git.UnitTests.csproj b/src/Microsoft.Build.Tasks.Git.UnitTests/Microsoft.Build.Tasks.Git.UnitTests.csproj index fdbe4a08..bb6a7da1 100644 --- a/src/Microsoft.Build.Tasks.Git.UnitTests/Microsoft.Build.Tasks.Git.UnitTests.csproj +++ b/src/Microsoft.Build.Tasks.Git.UnitTests/Microsoft.Build.Tasks.Git.UnitTests.csproj @@ -1,6 +1,6 @@  - net472;net5.0 + net472;net6.0 diff --git a/src/Microsoft.Build.Tasks.Git/Microsoft.Build.Tasks.Git.csproj b/src/Microsoft.Build.Tasks.Git/Microsoft.Build.Tasks.Git.csproj index 71b45dbb..3da4127e 100644 --- a/src/Microsoft.Build.Tasks.Git/Microsoft.Build.Tasks.Git.csproj +++ b/src/Microsoft.Build.Tasks.Git/Microsoft.Build.Tasks.Git.csproj @@ -1,6 +1,6 @@  - net472;net5.0 + net472;net6.0 true diff --git a/src/Microsoft.Build.Tasks.Git/build/Microsoft.Build.Tasks.Git.props b/src/Microsoft.Build.Tasks.Git/build/Microsoft.Build.Tasks.Git.props index 337242ea..308fd217 100644 --- a/src/Microsoft.Build.Tasks.Git/build/Microsoft.Build.Tasks.Git.props +++ b/src/Microsoft.Build.Tasks.Git/build/Microsoft.Build.Tasks.Git.props @@ -2,6 +2,6 @@ $(MSBuildThisFileDirectory)..\tools\net472\Microsoft.Build.Tasks.Git.dll - $(MSBuildThisFileDirectory)..\tools\net5.0\Microsoft.Build.Tasks.Git.dll + $(MSBuildThisFileDirectory)..\tools\net6.0\Microsoft.Build.Tasks.Git.dll diff --git a/src/SourceLink.AzureDevOpsServer.Git.UnitTests/Microsoft.SourceLink.AzureDevOpsServer.Git.UnitTests.csproj b/src/SourceLink.AzureDevOpsServer.Git.UnitTests/Microsoft.SourceLink.AzureDevOpsServer.Git.UnitTests.csproj index 9ff7a02d..610f1e75 100644 --- a/src/SourceLink.AzureDevOpsServer.Git.UnitTests/Microsoft.SourceLink.AzureDevOpsServer.Git.UnitTests.csproj +++ b/src/SourceLink.AzureDevOpsServer.Git.UnitTests/Microsoft.SourceLink.AzureDevOpsServer.Git.UnitTests.csproj @@ -1,6 +1,6 @@  - net472;net5.0 + net472;net6.0 diff --git a/src/SourceLink.AzureDevOpsServer.Git/Microsoft.SourceLink.AzureDevOpsServer.Git.csproj b/src/SourceLink.AzureDevOpsServer.Git/Microsoft.SourceLink.AzureDevOpsServer.Git.csproj index b905b8e5..c59c8b4b 100644 --- a/src/SourceLink.AzureDevOpsServer.Git/Microsoft.SourceLink.AzureDevOpsServer.Git.csproj +++ b/src/SourceLink.AzureDevOpsServer.Git/Microsoft.SourceLink.AzureDevOpsServer.Git.csproj @@ -1,6 +1,6 @@  - net472;net5.0 + net472;net6.0 true diff --git a/src/SourceLink.AzureDevOpsServer.Git/build/Microsoft.SourceLink.AzureDevOpsServer.Git.targets b/src/SourceLink.AzureDevOpsServer.Git/build/Microsoft.SourceLink.AzureDevOpsServer.Git.targets index 1c2b313b..8590b7d5 100644 --- a/src/SourceLink.AzureDevOpsServer.Git/build/Microsoft.SourceLink.AzureDevOpsServer.Git.targets +++ b/src/SourceLink.AzureDevOpsServer.Git/build/Microsoft.SourceLink.AzureDevOpsServer.Git.targets @@ -2,7 +2,7 @@ <_SourceLinkAzureDevOpsServerGitAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net472\Microsoft.SourceLink.AzureDevOpsServer.Git.dll - <_SourceLinkAzureDevOpsServerGitAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\net5.0\Microsoft.SourceLink.AzureDevOpsServer.Git.dll + <_SourceLinkAzureDevOpsServerGitAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\net6.0\Microsoft.SourceLink.AzureDevOpsServer.Git.dll diff --git a/src/SourceLink.AzureRepos.Git.UnitTests/Microsoft.SourceLink.AzureRepos.Git.UnitTests.csproj b/src/SourceLink.AzureRepos.Git.UnitTests/Microsoft.SourceLink.AzureRepos.Git.UnitTests.csproj index 180dd936..60d3a8ff 100644 --- a/src/SourceLink.AzureRepos.Git.UnitTests/Microsoft.SourceLink.AzureRepos.Git.UnitTests.csproj +++ b/src/SourceLink.AzureRepos.Git.UnitTests/Microsoft.SourceLink.AzureRepos.Git.UnitTests.csproj @@ -1,6 +1,6 @@  - net472;net5.0 + net472;net6.0 diff --git a/src/SourceLink.AzureRepos.Git/Microsoft.SourceLink.AzureRepos.Git.csproj b/src/SourceLink.AzureRepos.Git/Microsoft.SourceLink.AzureRepos.Git.csproj index 8a5a0bf3..191ca700 100644 --- a/src/SourceLink.AzureRepos.Git/Microsoft.SourceLink.AzureRepos.Git.csproj +++ b/src/SourceLink.AzureRepos.Git/Microsoft.SourceLink.AzureRepos.Git.csproj @@ -1,6 +1,6 @@  - net472;net5.0 + net472;net6.0 true diff --git a/src/SourceLink.AzureRepos.Git/build/Microsoft.SourceLink.AzureRepos.Git.targets b/src/SourceLink.AzureRepos.Git/build/Microsoft.SourceLink.AzureRepos.Git.targets index d49e459e..c07157a7 100644 --- a/src/SourceLink.AzureRepos.Git/build/Microsoft.SourceLink.AzureRepos.Git.targets +++ b/src/SourceLink.AzureRepos.Git/build/Microsoft.SourceLink.AzureRepos.Git.targets @@ -2,7 +2,7 @@ <_SourceLinkAzureReposGitAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net472\Microsoft.SourceLink.AzureRepos.Git.dll - <_SourceLinkAzureReposGitAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\net5.0\Microsoft.SourceLink.AzureRepos.Git.dll + <_SourceLinkAzureReposGitAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\net6.0\Microsoft.SourceLink.AzureRepos.Git.dll diff --git a/src/SourceLink.AzureRepos.Tfvc/build/Microsoft.SourceLink.AzureRepos.Tfvc.targets b/src/SourceLink.AzureRepos.Tfvc/build/Microsoft.SourceLink.AzureRepos.Tfvc.targets index f4045fb8..5b7be15d 100644 --- a/src/SourceLink.AzureRepos.Tfvc/build/Microsoft.SourceLink.AzureRepos.Tfvc.targets +++ b/src/SourceLink.AzureRepos.Tfvc/build/Microsoft.SourceLink.AzureRepos.Tfvc.targets @@ -2,7 +2,7 @@ <_SourceLinkAzureReposTfvcAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net472\Microsoft.SourceLink.AzureRepos.Tfvc.dll - <_SourceLinkAzureReposTfvcAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\net5.0\Microsoft.SourceLink.AzureRepos.Tfvc.dll + <_SourceLinkAzureReposTfvcAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\net6.0\Microsoft.SourceLink.AzureRepos.Tfvc.dll diff --git a/src/SourceLink.Bitbucket.Git.UnitTests/Microsoft.SourceLink.Bitbucket.Git.UnitTests.csproj b/src/SourceLink.Bitbucket.Git.UnitTests/Microsoft.SourceLink.Bitbucket.Git.UnitTests.csproj index a0c5edf4..c33cf00f 100644 --- a/src/SourceLink.Bitbucket.Git.UnitTests/Microsoft.SourceLink.Bitbucket.Git.UnitTests.csproj +++ b/src/SourceLink.Bitbucket.Git.UnitTests/Microsoft.SourceLink.Bitbucket.Git.UnitTests.csproj @@ -1,6 +1,6 @@  - net472;net5.0 + net472;net6.0 diff --git a/src/SourceLink.Bitbucket.Git/Microsoft.SourceLink.Bitbucket.Git.csproj b/src/SourceLink.Bitbucket.Git/Microsoft.SourceLink.Bitbucket.Git.csproj index 387b65c0..86fe6336 100644 --- a/src/SourceLink.Bitbucket.Git/Microsoft.SourceLink.Bitbucket.Git.csproj +++ b/src/SourceLink.Bitbucket.Git/Microsoft.SourceLink.Bitbucket.Git.csproj @@ -1,6 +1,6 @@  - net472;net5.0 + net472;net6.0 true diff --git a/src/SourceLink.Bitbucket.Git/build/Microsoft.SourceLink.Bitbucket.Git.targets b/src/SourceLink.Bitbucket.Git/build/Microsoft.SourceLink.Bitbucket.Git.targets index 0cb1491d..d826e799 100644 --- a/src/SourceLink.Bitbucket.Git/build/Microsoft.SourceLink.Bitbucket.Git.targets +++ b/src/SourceLink.Bitbucket.Git/build/Microsoft.SourceLink.Bitbucket.Git.targets @@ -2,7 +2,7 @@ <_SourceLinkBitbucketAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net472\Microsoft.SourceLink.Bitbucket.Git.dll - <_SourceLinkBitbucketAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\net5.0\Microsoft.SourceLink.Bitbucket.Git.dll + <_SourceLinkBitbucketAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\net6.0\Microsoft.SourceLink.Bitbucket.Git.dll diff --git a/src/SourceLink.Common.UnitTests/Microsoft.SourceLink.Common.UnitTests.csproj b/src/SourceLink.Common.UnitTests/Microsoft.SourceLink.Common.UnitTests.csproj index 4ca53283..45b74dcf 100644 --- a/src/SourceLink.Common.UnitTests/Microsoft.SourceLink.Common.UnitTests.csproj +++ b/src/SourceLink.Common.UnitTests/Microsoft.SourceLink.Common.UnitTests.csproj @@ -1,6 +1,6 @@  - net472;net5.0 + net472;net6.0 diff --git a/src/SourceLink.Common/Microsoft.SourceLink.Common.csproj b/src/SourceLink.Common/Microsoft.SourceLink.Common.csproj index b2205638..66636da6 100644 --- a/src/SourceLink.Common/Microsoft.SourceLink.Common.csproj +++ b/src/SourceLink.Common/Microsoft.SourceLink.Common.csproj @@ -1,6 +1,6 @@  - net472;net5.0 + net472;net6.0 true diff --git a/src/SourceLink.Common/build/Microsoft.SourceLink.Common.props b/src/SourceLink.Common/build/Microsoft.SourceLink.Common.props index ac63a8ae..8840a5e6 100644 --- a/src/SourceLink.Common/build/Microsoft.SourceLink.Common.props +++ b/src/SourceLink.Common/build/Microsoft.SourceLink.Common.props @@ -2,7 +2,7 @@ <_MicrosoftSourceLinkCommonAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net472\Microsoft.SourceLink.Common.dll - <_MicrosoftSourceLinkCommonAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\net5.0\Microsoft.SourceLink.Common.dll + <_MicrosoftSourceLinkCommonAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\net6.0\Microsoft.SourceLink.Common.dll diff --git a/src/SourceLink.Git.IntegrationTests/Microsoft.SourceLink.Git.IntegrationTests.csproj b/src/SourceLink.Git.IntegrationTests/Microsoft.SourceLink.Git.IntegrationTests.csproj index 12a748c5..00993826 100644 --- a/src/SourceLink.Git.IntegrationTests/Microsoft.SourceLink.Git.IntegrationTests.csproj +++ b/src/SourceLink.Git.IntegrationTests/Microsoft.SourceLink.Git.IntegrationTests.csproj @@ -1,6 +1,6 @@  - net472;net5.0 + net472;net6.0 diff --git a/src/SourceLink.GitHub.UnitTests/Microsoft.SourceLink.GitHub.UnitTests.csproj b/src/SourceLink.GitHub.UnitTests/Microsoft.SourceLink.GitHub.UnitTests.csproj index d229ac5d..e9f9bf20 100644 --- a/src/SourceLink.GitHub.UnitTests/Microsoft.SourceLink.GitHub.UnitTests.csproj +++ b/src/SourceLink.GitHub.UnitTests/Microsoft.SourceLink.GitHub.UnitTests.csproj @@ -1,6 +1,6 @@  - net472;net5.0 + net472;net6.0 diff --git a/src/SourceLink.GitHub/Microsoft.SourceLink.GitHub.csproj b/src/SourceLink.GitHub/Microsoft.SourceLink.GitHub.csproj index 45985c8e..aef83a7f 100644 --- a/src/SourceLink.GitHub/Microsoft.SourceLink.GitHub.csproj +++ b/src/SourceLink.GitHub/Microsoft.SourceLink.GitHub.csproj @@ -1,6 +1,6 @@  - net472;net5.0 + net472;net6.0 true diff --git a/src/SourceLink.GitHub/build/Microsoft.SourceLink.GitHub.targets b/src/SourceLink.GitHub/build/Microsoft.SourceLink.GitHub.targets index 2641835e..e016121b 100644 --- a/src/SourceLink.GitHub/build/Microsoft.SourceLink.GitHub.targets +++ b/src/SourceLink.GitHub/build/Microsoft.SourceLink.GitHub.targets @@ -2,7 +2,7 @@ <_SourceLinkGitHubAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net472\Microsoft.SourceLink.GitHub.dll - <_SourceLinkGitHubAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\net5.0\Microsoft.SourceLink.GitHub.dll + <_SourceLinkGitHubAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\net6.0\Microsoft.SourceLink.GitHub.dll diff --git a/src/SourceLink.GitLab.UnitTests/Microsoft.SourceLink.GitLab.UnitTests.csproj b/src/SourceLink.GitLab.UnitTests/Microsoft.SourceLink.GitLab.UnitTests.csproj index c31a3718..21a594f4 100644 --- a/src/SourceLink.GitLab.UnitTests/Microsoft.SourceLink.GitLab.UnitTests.csproj +++ b/src/SourceLink.GitLab.UnitTests/Microsoft.SourceLink.GitLab.UnitTests.csproj @@ -1,6 +1,6 @@  - net472;net5.0 + net472;net6.0 diff --git a/src/SourceLink.GitLab/Microsoft.SourceLink.GitLab.csproj b/src/SourceLink.GitLab/Microsoft.SourceLink.GitLab.csproj index 18300cef..f74e5f49 100644 --- a/src/SourceLink.GitLab/Microsoft.SourceLink.GitLab.csproj +++ b/src/SourceLink.GitLab/Microsoft.SourceLink.GitLab.csproj @@ -1,6 +1,6 @@  - net472;net5.0 + net472;net6.0 true diff --git a/src/SourceLink.GitLab/build/Microsoft.SourceLink.GitLab.targets b/src/SourceLink.GitLab/build/Microsoft.SourceLink.GitLab.targets index ffc5140d..724c0075 100644 --- a/src/SourceLink.GitLab/build/Microsoft.SourceLink.GitLab.targets +++ b/src/SourceLink.GitLab/build/Microsoft.SourceLink.GitLab.targets @@ -2,7 +2,7 @@ <_SourceLinkGitLabAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net472\Microsoft.SourceLink.GitLab.dll - <_SourceLinkGitLabAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\net5.0\Microsoft.SourceLink.GitLab.dll + <_SourceLinkGitLabAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\net6.0\Microsoft.SourceLink.GitLab.dll diff --git a/src/SourceLink.GitWeb.UnitTests/Microsoft.SourceLink.GitWeb.UnitTests.csproj b/src/SourceLink.GitWeb.UnitTests/Microsoft.SourceLink.GitWeb.UnitTests.csproj index 5a9bcb86..0a1d6a86 100644 --- a/src/SourceLink.GitWeb.UnitTests/Microsoft.SourceLink.GitWeb.UnitTests.csproj +++ b/src/SourceLink.GitWeb.UnitTests/Microsoft.SourceLink.GitWeb.UnitTests.csproj @@ -1,6 +1,6 @@  - net472;net5.0 + net472;net6.0 diff --git a/src/SourceLink.GitWeb/Microsoft.SourceLink.GitWeb.csproj b/src/SourceLink.GitWeb/Microsoft.SourceLink.GitWeb.csproj index 1991db75..e643e97d 100644 --- a/src/SourceLink.GitWeb/Microsoft.SourceLink.GitWeb.csproj +++ b/src/SourceLink.GitWeb/Microsoft.SourceLink.GitWeb.csproj @@ -1,6 +1,6 @@  - net472;net5.0 + net472;net6.0 true diff --git a/src/SourceLink.GitWeb/build/Microsoft.SourceLink.GitWeb.targets b/src/SourceLink.GitWeb/build/Microsoft.SourceLink.GitWeb.targets index 9f21d62b..e3f61434 100644 --- a/src/SourceLink.GitWeb/build/Microsoft.SourceLink.GitWeb.targets +++ b/src/SourceLink.GitWeb/build/Microsoft.SourceLink.GitWeb.targets @@ -2,7 +2,7 @@ <_SourceLinkGitWebAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net472\Microsoft.SourceLink.GitWeb.dll - <_SourceLinkGitWebAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\net5.0\Microsoft.SourceLink.GitWeb.dll + <_SourceLinkGitWebAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\net6.0\Microsoft.SourceLink.GitWeb.dll diff --git a/src/SourceLink.Gitea.UnitTests/Microsoft.SourceLink.Gitea.UnitTests.csproj b/src/SourceLink.Gitea.UnitTests/Microsoft.SourceLink.Gitea.UnitTests.csproj index 12bdc2d6..513d2842 100644 --- a/src/SourceLink.Gitea.UnitTests/Microsoft.SourceLink.Gitea.UnitTests.csproj +++ b/src/SourceLink.Gitea.UnitTests/Microsoft.SourceLink.Gitea.UnitTests.csproj @@ -1,6 +1,6 @@  - net472;net5.0 + net472;net6.0 diff --git a/src/SourceLink.Gitea/Microsoft.SourceLink.Gitea.csproj b/src/SourceLink.Gitea/Microsoft.SourceLink.Gitea.csproj index b5bbc93d..c4d0b140 100644 --- a/src/SourceLink.Gitea/Microsoft.SourceLink.Gitea.csproj +++ b/src/SourceLink.Gitea/Microsoft.SourceLink.Gitea.csproj @@ -1,6 +1,6 @@  - net472;net5.0 + net472;net6.0 true diff --git a/src/SourceLink.Gitea/build/Microsoft.SourceLink.Gitea.targets b/src/SourceLink.Gitea/build/Microsoft.SourceLink.Gitea.targets index 5b6765e5..d86a5d33 100644 --- a/src/SourceLink.Gitea/build/Microsoft.SourceLink.Gitea.targets +++ b/src/SourceLink.Gitea/build/Microsoft.SourceLink.Gitea.targets @@ -2,7 +2,7 @@ <_SourceLinkGiteaAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net472\Microsoft.SourceLink.Gitea.dll - <_SourceLinkGiteaAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\net5.0\Microsoft.SourceLink.Gitea.dll + <_SourceLinkGiteaAssemblyFile Condition="'$(MSBuildRuntimeType)' == 'Core'">$(MSBuildThisFileDirectory)..\tools\net6.0\Microsoft.SourceLink.Gitea.dll diff --git a/src/SourceLink.Tools.UnitTests/Microsoft.SourceLink.Tools.UnitTests.csproj b/src/SourceLink.Tools.UnitTests/Microsoft.SourceLink.Tools.UnitTests.csproj index 8d211b84..5122b50e 100644 --- a/src/SourceLink.Tools.UnitTests/Microsoft.SourceLink.Tools.UnitTests.csproj +++ b/src/SourceLink.Tools.UnitTests/Microsoft.SourceLink.Tools.UnitTests.csproj @@ -1,6 +1,6 @@  - net472;net5.0 + net472;net6.0 diff --git a/src/SourceLink.Tools/Microsoft.SourceLink.Tools.Package.csproj b/src/SourceLink.Tools/Microsoft.SourceLink.Tools.Package.csproj index 053f1245..6f6c6b3b 100644 --- a/src/SourceLink.Tools/Microsoft.SourceLink.Tools.Package.csproj +++ b/src/SourceLink.Tools/Microsoft.SourceLink.Tools.Package.csproj @@ -2,7 +2,7 @@ - net5.0;net472 + net6.0;net472 false none false diff --git a/src/TestUtilities/TestUtilities.csproj b/src/TestUtilities/TestUtilities.csproj index f5588696..b5572c7c 100644 --- a/src/TestUtilities/TestUtilities.csproj +++ b/src/TestUtilities/TestUtilities.csproj @@ -1,6 +1,6 @@  - net472;net5.0 + net472;net6.0 false true From 040e4c0d50456ddfd57754ddd9a1c8c7d641d756 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Matou=C5=A1ek?= Date: Tue, 27 Jul 2021 13:08:59 -0700 Subject: [PATCH 135/161] Add netstandard2.0 target to tools package (#702) * Add netcoreapp3.1 target to tools package * netstandard2.0 --- src/SourceLink.Tools/Microsoft.SourceLink.Tools.Package.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SourceLink.Tools/Microsoft.SourceLink.Tools.Package.csproj b/src/SourceLink.Tools/Microsoft.SourceLink.Tools.Package.csproj index 6f6c6b3b..baea22f8 100644 --- a/src/SourceLink.Tools/Microsoft.SourceLink.Tools.Package.csproj +++ b/src/SourceLink.Tools/Microsoft.SourceLink.Tools.Package.csproj @@ -2,7 +2,7 @@ - net6.0;net472 + net6.0;netstandard2.0 false none false From 266451c6979a3598a6b7216684f118696e6efcd0 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 2 Aug 2021 13:17:33 +0000 Subject: [PATCH 136/161] Update dependencies from https://github.com/dotnet/arcade build 20210729.2 (#728) [main] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- eng/common/internal/Tools.csproj | 2 +- global.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 4f6463c0..0b89c2ef 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - c6a28c81f96d196338b3ea520bc1e6dc7c440ee2 + 62a8aafffd4c68ef887680f6837abdff906a662c https://github.com/dotnet/source-build-reference-packages diff --git a/eng/common/internal/Tools.csproj b/eng/common/internal/Tools.csproj index f46d5efe..2067b8df 100644 --- a/eng/common/internal/Tools.csproj +++ b/eng/common/internal/Tools.csproj @@ -1,5 +1,5 @@ + - net472 diff --git a/global.json b/global.json index 69cb8fb5..dfc9bf3c 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "6.0.100-preview.6.21355.2" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21373.11" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21379.2" } } From 4e754679f37dac182184c10813dc77814caea05b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Matou=C5=A1ek?= Date: Tue, 10 Aug 2021 10:09:56 -0700 Subject: [PATCH 137/161] Update license to MIT (#730) --- .editorconfig | 5 +- License.txt | 215 ++---------------- eng/BuildTask.targets | 4 +- eng/SourceBuild.props | 2 + eng/Versions.props | 3 +- src/Common/AzureDevOpsUrlParser.cs | 4 +- src/Common/GetSourceLinkUrlGitTask.cs | 4 +- src/Common/GitUrlMappingTask.cs | 4 +- src/Common/Names.cs | 4 +- src/Common/NullableAttributes.cs | 4 +- src/Common/PathUtilities.cs | 6 +- src/Common/SequenceComparer.cs | 4 +- src/Common/TranslateRepositoryUrlGitTask.cs | 5 +- src/Common/UriUtilities.cs | 4 +- src/Common/ValueTuple.cs | 6 +- src/Directory.Build.props | 5 +- src/Directory.Build.targets | 3 +- .../Microsoft.Build.StandardCI.csproj | 4 +- .../build/Microsoft.Build.StandardCI.props | 3 +- .../GitConfigTests.cs | 4 +- .../GitDataTests.cs | 4 +- .../GitEnvironmentTests.cs | 4 +- .../GitIgnoreTests.cs | 4 +- .../GitOperationsTests.cs | 4 +- .../GitReferenceResolverTests.cs | 4 +- .../GitRepositoryTests.cs | 4 +- .../GlobTests.cs | 4 +- ...Microsoft.Build.Tasks.Git.UnitTests.csproj | 4 +- .../TestUtilities.cs | 4 +- .../AssemblyResolver.cs | 4 +- .../GetUntrackedFiles.cs | 4 +- .../GitDataReader/CharUtils.cs | 4 +- .../GitDataReader/GitConfig.Reader.cs | 4 +- .../GitDataReader/GitConfig.cs | 4 +- .../GitDataReader/GitEnvironment.cs | 4 +- .../GitDataReader/GitIgnore.Matcher.cs | 4 +- .../GitDataReader/GitIgnore.cs | 4 +- .../GitDataReader/GitReferenceResolver.cs | 4 +- .../GitDataReader/GitRepository.cs | 4 +- .../GitDataReader/GitRepositoryLocation.cs | 4 +- .../GitDataReader/GitSubmodule.cs | 4 +- .../GitDataReader/GitVariableName.cs | 4 +- .../GitDataReader/Glob.cs | 6 +- .../GitDataReader/PathUtils.cs | 4 +- .../GitOperations.cs | 4 +- .../LocateRepository.cs | 4 +- .../Microsoft.Build.Tasks.Git.csproj | 4 +- .../RepositoryTask.cs | 4 +- .../build/Microsoft.Build.Tasks.Git.props | 3 +- .../build/Microsoft.Build.Tasks.Git.targets | 3 +- .../Microsoft.Build.Tasks.Git.props | 3 +- .../Microsoft.Build.Tasks.Git.targets | 3 +- .../GetRepositoryUrl.cs | 4 +- .../GetSourceRevisionId.cs | 4 +- .../GetSourceRoots.cs | 4 +- .../GetUntrackedSourceFiles.cs | 4 +- .../LocateRepository.cs | 4 +- .../Microsoft.Build.Tasks.Tfvc.csproj | 4 +- .../RepositoryTask.cs | 4 +- .../build/Microsoft.Build.Tasks.Tfvc.props | 5 +- .../build/Microsoft.Build.Tasks.Tfvc.targets | 3 +- .../Microsoft.Build.Tasks.Tfvc.props | 3 +- .../Microsoft.Build.Tasks.Tfvc.targets | 3 +- .../AzureDevOpsUrlParserOnPremTests.cs | 4 +- .../GetSourceLinkUrlTests.cs | 4 +- ...ink.AzureDevOpsServer.Git.UnitTests.csproj | 4 +- .../TranslateRepositoryUrlsTests.cs | 4 +- .../GetSourceLinkUrl.cs | 4 +- ...ft.SourceLink.AzureDevOpsServer.Git.csproj | 4 +- .../TranslateRepositoryUrls.cs | 4 +- ...t.SourceLink.AzureDevOpsServer.Git.targets | 5 +- ...t.SourceLink.AzureDevOpsServer.Git.targets | 3 +- .../AzureDevOpsUrlParserHostedTests.cs | 4 +- .../GetSourceLinkUrlTests.cs | 4 +- ...SourceLink.AzureRepos.Git.UnitTests.csproj | 4 +- .../TranslateRepositoryUrlsTests.cs | 4 +- .../GetSourceLinkUrl.cs | 4 +- ...Microsoft.SourceLink.AzureRepos.Git.csproj | 4 +- .../TranslateRepositoryUrls.cs | 4 +- .../Microsoft.SourceLink.AzureRepos.Git.props | 3 +- ...icrosoft.SourceLink.AzureRepos.Git.targets | 5 +- .../Microsoft.SourceLink.AzureRepos.Git.props | 3 +- ...icrosoft.SourceLink.AzureRepos.Git.targets | 3 +- .../GetSourceLinkUrl.cs | 4 +- ...icrosoft.SourceLink.AzureRepos.Tfvc.csproj | 4 +- ...crosoft.SourceLink.AzureRepos.Tfvc.targets | 5 +- ...crosoft.SourceLink.AzureRepos.Tfvc.targets | 3 +- .../GetSourceLinkUrlTests.cs | 4 +- ....SourceLink.Bitbucket.Git.UnitTests.csproj | 4 +- .../TranslateRepositoryUrlsTests.cs | 4 +- .../GetSourceLinkUrl.cs | 4 +- .../Microsoft.SourceLink.Bitbucket.Git.csproj | 4 +- .../TranslateRepositoryUrls.cs | 4 +- .../Microsoft.SourceLink.Bitbucket.Git.props | 3 +- ...Microsoft.SourceLink.Bitbucket.Git.targets | 3 +- .../Microsoft.SourceLink.Bitbucket.Git.props | 3 +- ...Microsoft.SourceLink.Bitbucket.Git.targets | 3 +- .../GenerateSourceLinkFileTests.cs | 4 +- .../GetSourceLinkUrlTests.cs | 4 +- ...crosoft.SourceLink.Common.UnitTests.csproj | 4 +- .../Mocks/MockGetSourceLinkUrlGitTask.cs | 4 +- .../PathUtilitiesTests.cs | 4 +- .../TranslateRepositoryUrlsTests.cs | 4 +- .../UriUtilitiesTests.cs | 4 +- .../GenerateSourceLinkFile.cs | 4 +- .../Microsoft.SourceLink.Common.csproj | 4 +- .../SourceLinkHasSingleProvider.cs | 4 +- ...InitializeSourceControlInformation.targets | 1 + .../build/Microsoft.SourceLink.Common.props | 3 +- .../build/Microsoft.SourceLink.Common.targets | 3 +- .../Microsoft.SourceLink.Common.props | 3 +- .../Microsoft.SourceLink.Common.targets | 3 +- .../AzureDevOpsServerTests.cs | 4 +- .../AzureReposAndGitHubTests.cs | 4 +- .../AzureReposTests.cs | 4 +- .../BitbucketGitTests.cs | 4 +- .../GitHubTests.cs | 4 +- .../GitLabTests.cs | 4 +- .../GitWebTests.cs | 5 +- .../GiteaTests.cs | 4 +- ...oft.SourceLink.Git.IntegrationTests.csproj | 4 +- .../TargetTests.cs | 4 +- .../Utilities/GitUtilities.cs | 4 +- .../Utilities/TestUtilities.cs | 4 +- .../GetSourceLinkUrlTests.cs | 4 +- ...crosoft.SourceLink.GitHub.UnitTests.csproj | 4 +- .../TranslateRepositoryUrlsTests.cs | 4 +- src/SourceLink.GitHub/GetSourceLinkUrl.cs | 4 +- .../Microsoft.SourceLink.GitHub.csproj | 4 +- .../TranslateRepositoryUrls.cs | 4 +- .../build/Microsoft.SourceLink.GitHub.props | 3 +- .../build/Microsoft.SourceLink.GitHub.targets | 3 +- .../Microsoft.SourceLink.GitHub.props | 3 +- .../Microsoft.SourceLink.GitHub.targets | 3 +- .../GetSourceLinkUrlTests.cs | 4 +- ...crosoft.SourceLink.GitLab.UnitTests.csproj | 4 +- .../TranslateRepositoryUrlsTests.cs | 4 +- src/SourceLink.GitLab/GetSourceLinkUrl.cs | 4 +- .../Microsoft.SourceLink.GitLab.csproj | 4 +- .../TranslateRepositoryUrls.cs | 4 +- .../build/Microsoft.SourceLink.GitLab.targets | 3 +- .../Microsoft.SourceLink.GitLab.targets | 3 +- .../GetSourceLinkUrlTests.cs | 5 +- ...crosoft.SourceLink.GitWeb.UnitTests.csproj | 4 +- .../TranslateRepositoryUrlsTests.cs | 5 +- src/SourceLink.GitWeb/GetSourceLinkUrl.cs | 5 +- .../Microsoft.SourceLink.GitWeb.csproj | 4 +- .../TranslateRepositoryUrls.cs | 5 +- .../build/Microsoft.SourceLink.GitWeb.targets | 3 +- .../Microsoft.SourceLink.GitWeb.targets | 3 +- .../GetSourceLinkUrlTests.cs | 4 +- ...icrosoft.SourceLink.Gitea.UnitTests.csproj | 4 +- .../TranslateRepositoryUrlsTests.cs | 4 +- src/SourceLink.Gitea/GetSourceLinkUrl.cs | 4 +- .../Microsoft.SourceLink.Gitea.csproj | 4 +- .../TranslateRepositoryUrls.cs | 4 +- .../build/Microsoft.SourceLink.Gitea.targets | 3 +- .../Microsoft.SourceLink.Gitea.targets | 3 +- ...icrosoft.SourceLink.Tools.UnitTests.csproj | 4 +- .../SourceLinkMapTests.cs | 4 +- .../Microsoft.SourceLink.Tools.Package.csproj | 4 +- src/SourceLink.Tools/SourceLinkMap.cs | 6 +- src/TestUtilities/AssertEx.cs | 4 +- src/TestUtilities/ConditionalFactAttribute.cs | 4 +- src/TestUtilities/DiffUtil.cs | 4 +- .../DotNetSdk/BuildInfoAttribute.cs | 4 +- .../DotNetSdk/DotNetSdkTestBase.cs | 4 +- src/TestUtilities/EnumerableExtensions.cs | 4 +- src/TestUtilities/KeyValuePairUtils.cs | 4 +- src/TestUtilities/MockEngine.cs | 5 +- src/TestUtilities/MockItem.cs | 5 +- src/TestUtilities/ProcessResult.cs | 6 +- src/TestUtilities/ProcessUtilities.cs | 4 +- .../TempFiles/DisposableDirectory.cs | 4 +- src/TestUtilities/TempFiles/DisposableFile.cs | 4 +- src/TestUtilities/TempFiles/TempDirectory.cs | 4 +- src/TestUtilities/TempFiles/TempFile.cs | 4 +- src/TestUtilities/TempFiles/TempRoot.cs | 4 +- src/TestUtilities/TestUtilities.csproj | 4 +- 179 files changed, 519 insertions(+), 394 deletions(-) diff --git a/.editorconfig b/.editorconfig index 226f7d30..eb0d1eaa 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,4 +1,4 @@ -# EditorConfig is awesome: +# EditorConfig is awesome: http://EditorConfig.org # top-most EditorConfig file @@ -153,3 +153,6 @@ csharp_space_between_square_brackets = false csharp_preserve_single_line_blocks = true csharp_preserve_single_line_statements = true + +[*.{cs}] +file_header_template = Licensed to the.NET Foundation under one or more agreements.\nThe.NET Foundation licenses this file to you under the MIT license.\nSee the License.txt file in the project root for more information. diff --git a/License.txt b/License.txt index d6456956..a616ed18 100644 --- a/License.txt +++ b/License.txt @@ -1,202 +1,23 @@ +The MIT License (MIT) - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ +Copyright (c) .NET Foundation and Contributors - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION +All rights reserved. - 1. Definitions. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/eng/BuildTask.targets b/eng/BuildTask.targets index 3a9de4d9..868c8666 100644 --- a/eng/BuildTask.targets +++ b/eng/BuildTask.targets @@ -1,5 +1,5 @@ - - + + true diff --git a/eng/SourceBuild.props b/eng/SourceBuild.props index 4cd76267..c9492022 100644 --- a/eng/SourceBuild.props +++ b/eng/SourceBuild.props @@ -1,3 +1,5 @@ + + diff --git a/eng/Versions.props b/eng/Versions.props index 4ea51a89..4afc3c3e 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,4 +1,5 @@ - + + diff --git a/src/Common/AzureDevOpsUrlParser.cs b/src/Common/AzureDevOpsUrlParser.cs index 4f212f5e..0bbf0b06 100644 --- a/src/Common/AzureDevOpsUrlParser.cs +++ b/src/Common/AzureDevOpsUrlParser.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Diagnostics; diff --git a/src/Common/GetSourceLinkUrlGitTask.cs b/src/Common/GetSourceLinkUrlGitTask.cs index 2962db3c..203fb7f7 100644 --- a/src/Common/GetSourceLinkUrlGitTask.cs +++ b/src/Common/GetSourceLinkUrlGitTask.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Collections.Generic; diff --git a/src/Common/GitUrlMappingTask.cs b/src/Common/GitUrlMappingTask.cs index abe54ef6..a31b6ad9 100644 --- a/src/Common/GitUrlMappingTask.cs +++ b/src/Common/GitUrlMappingTask.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Collections.Generic; diff --git a/src/Common/Names.cs b/src/Common/Names.cs index 59a3f726..e0515cb3 100644 --- a/src/Common/Names.cs +++ b/src/Common/Names.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. namespace Microsoft.Build.Tasks.SourceControl { diff --git a/src/Common/NullableAttributes.cs b/src/Common/NullableAttributes.cs index 8bce5d69..9c45cc65 100644 --- a/src/Common/NullableAttributes.cs +++ b/src/Common/NullableAttributes.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System.Diagnostics; using System.Diagnostics.CodeAnalysis; diff --git a/src/Common/PathUtilities.cs b/src/Common/PathUtilities.cs index 9d30da69..aa20eb1f 100644 --- a/src/Common/PathUtilities.cs +++ b/src/Common/PathUtilities.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Collections.Generic; @@ -48,4 +50,4 @@ public static string EndWithSeparator(this string path) public static string EndWithSeparator(this string path, char separator) => path.EndsWithSeparator() ? path : path + separator; } -} \ No newline at end of file +} diff --git a/src/Common/SequenceComparer.cs b/src/Common/SequenceComparer.cs index c280573c..9ec6e0f6 100644 --- a/src/Common/SequenceComparer.cs +++ b/src/Common/SequenceComparer.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Collections.Generic; diff --git a/src/Common/TranslateRepositoryUrlGitTask.cs b/src/Common/TranslateRepositoryUrlGitTask.cs index 02c0e032..f09822ef 100644 --- a/src/Common/TranslateRepositoryUrlGitTask.cs +++ b/src/Common/TranslateRepositoryUrlGitTask.cs @@ -1,5 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See -// License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Collections.Generic; diff --git a/src/Common/UriUtilities.cs b/src/Common/UriUtilities.cs index b490faf7..981c9c13 100644 --- a/src/Common/UriUtilities.cs +++ b/src/Common/UriUtilities.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Linq; diff --git a/src/Common/ValueTuple.cs b/src/Common/ValueTuple.cs index 074e9f60..05c44232 100644 --- a/src/Common/ValueTuple.cs +++ b/src/Common/ValueTuple.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. #if NET461 @@ -46,4 +48,4 @@ public TupleElementNamesAttribute(string[] transformNames) } } -#endif \ No newline at end of file +#endif diff --git a/src/Directory.Build.props b/src/Directory.Build.props index 01bbff1b..5283e917 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -1,11 +1,12 @@ - + + Preview enable - Apache-2.0 + MIT true $(RepositoryEngineeringDir)runtimeconfig.template.json diff --git a/src/Directory.Build.targets b/src/Directory.Build.targets index a4b4d2c1..f175fd49 100644 --- a/src/Directory.Build.targets +++ b/src/Directory.Build.targets @@ -1,4 +1,5 @@ - + + diff --git a/src/Microsoft.Build.StandardCI/Microsoft.Build.StandardCI.csproj b/src/Microsoft.Build.StandardCI/Microsoft.Build.StandardCI.csproj index 7dd6f956..2e74e6b7 100644 --- a/src/Microsoft.Build.StandardCI/Microsoft.Build.StandardCI.csproj +++ b/src/Microsoft.Build.StandardCI/Microsoft.Build.StandardCI.csproj @@ -1,4 +1,6 @@ - + + + net472;net6.0 true diff --git a/src/Microsoft.Build.StandardCI/build/Microsoft.Build.StandardCI.props b/src/Microsoft.Build.StandardCI/build/Microsoft.Build.StandardCI.props index 353bd638..69d79020 100644 --- a/src/Microsoft.Build.StandardCI/build/Microsoft.Build.StandardCI.props +++ b/src/Microsoft.Build.StandardCI/build/Microsoft.Build.StandardCI.props @@ -1,4 +1,5 @@ - + + + net472;net6.0 diff --git a/src/Microsoft.Build.Tasks.Git.UnitTests/TestUtilities.cs b/src/Microsoft.Build.Tasks.Git.UnitTests/TestUtilities.cs index 6497485f..9e22db39 100644 --- a/src/Microsoft.Build.Tasks.Git.UnitTests/TestUtilities.cs +++ b/src/Microsoft.Build.Tasks.Git.UnitTests/TestUtilities.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using Microsoft.Build.Framework; diff --git a/src/Microsoft.Build.Tasks.Git/AssemblyResolver.cs b/src/Microsoft.Build.Tasks.Git/AssemblyResolver.cs index 9889bb54..e0c77070 100644 --- a/src/Microsoft.Build.Tasks.Git/AssemblyResolver.cs +++ b/src/Microsoft.Build.Tasks.Git/AssemblyResolver.cs @@ -1,6 +1,6 @@ -// Licensed to the .NET Foundation under one or more agreements. +// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. +// See the License.txt file in the project root for more information. #if NET461 diff --git a/src/Microsoft.Build.Tasks.Git/GetUntrackedFiles.cs b/src/Microsoft.Build.Tasks.Git/GetUntrackedFiles.cs index 7465313b..0be4a3d6 100644 --- a/src/Microsoft.Build.Tasks.Git/GetUntrackedFiles.cs +++ b/src/Microsoft.Build.Tasks.Git/GetUntrackedFiles.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System.Diagnostics.CodeAnalysis; using Microsoft.Build.Framework; diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/CharUtils.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/CharUtils.cs index 4baf9bd5..66ce48f5 100644 --- a/src/Microsoft.Build.Tasks.Git/GitDataReader/CharUtils.cs +++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/CharUtils.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitConfig.Reader.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitConfig.Reader.cs index e01f1954..6530d46b 100644 --- a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitConfig.Reader.cs +++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitConfig.Reader.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Collections.Generic; diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitConfig.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitConfig.cs index d55cd02d..5d57c7b6 100644 --- a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitConfig.cs +++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitConfig.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Collections.Generic; diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitEnvironment.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitEnvironment.cs index 6d7be601..b0a74c08 100644 --- a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitEnvironment.cs +++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitEnvironment.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Diagnostics; diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitIgnore.Matcher.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitIgnore.Matcher.cs index 5196ac9b..ba784e07 100644 --- a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitIgnore.Matcher.cs +++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitIgnore.Matcher.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Collections.Generic; diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitIgnore.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitIgnore.cs index 47f75dd7..5757f60c 100644 --- a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitIgnore.cs +++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitIgnore.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Collections.Generic; diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitReferenceResolver.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitReferenceResolver.cs index e62bd4ed..cec393af 100644 --- a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitReferenceResolver.cs +++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitReferenceResolver.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Collections.Generic; diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs index 9f1648f8..30ee72bc 100644 --- a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs +++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Collections.Generic; diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepositoryLocation.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepositoryLocation.cs index 6d1d96f3..0f44a401 100644 --- a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepositoryLocation.cs +++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepositoryLocation.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitSubmodule.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitSubmodule.cs index a953773e..43f8611f 100644 --- a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitSubmodule.cs +++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitSubmodule.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitVariableName.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitVariableName.cs index b4eefb68..ddddbc35 100644 --- a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitVariableName.cs +++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitVariableName.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Diagnostics; diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/Glob.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/Glob.cs index 8c9f62e9..605409e7 100644 --- a/src/Microsoft.Build.Tasks.Git/GitDataReader/Glob.cs +++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/Glob.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. // Implementation based on documentation: // - https://git-scm.com/docs/gitignore @@ -248,4 +250,4 @@ private static bool IsRangeMatch(string pattern, ref int patternIndex, char path return false; } } -} \ No newline at end of file +} diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/PathUtils.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/PathUtils.cs index 9593432c..8ff2dd1d 100644 --- a/src/Microsoft.Build.Tasks.Git/GitDataReader/PathUtils.cs +++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/PathUtils.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Diagnostics; diff --git a/src/Microsoft.Build.Tasks.Git/GitOperations.cs b/src/Microsoft.Build.Tasks.Git/GitOperations.cs index 2d029de4..ed9d36a7 100644 --- a/src/Microsoft.Build.Tasks.Git/GitOperations.cs +++ b/src/Microsoft.Build.Tasks.Git/GitOperations.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Collections.Generic; diff --git a/src/Microsoft.Build.Tasks.Git/LocateRepository.cs b/src/Microsoft.Build.Tasks.Git/LocateRepository.cs index ac6bfb79..a592df01 100644 --- a/src/Microsoft.Build.Tasks.Git/LocateRepository.cs +++ b/src/Microsoft.Build.Tasks.Git/LocateRepository.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Diagnostics.CodeAnalysis; diff --git a/src/Microsoft.Build.Tasks.Git/Microsoft.Build.Tasks.Git.csproj b/src/Microsoft.Build.Tasks.Git/Microsoft.Build.Tasks.Git.csproj index 3da4127e..3fa8c71d 100644 --- a/src/Microsoft.Build.Tasks.Git/Microsoft.Build.Tasks.Git.csproj +++ b/src/Microsoft.Build.Tasks.Git/Microsoft.Build.Tasks.Git.csproj @@ -1,4 +1,6 @@ - + + + net472;net6.0 true diff --git a/src/Microsoft.Build.Tasks.Git/RepositoryTask.cs b/src/Microsoft.Build.Tasks.Git/RepositoryTask.cs index e4b1a000..62aa8cf2 100644 --- a/src/Microsoft.Build.Tasks.Git/RepositoryTask.cs +++ b/src/Microsoft.Build.Tasks.Git/RepositoryTask.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Diagnostics.CodeAnalysis; diff --git a/src/Microsoft.Build.Tasks.Git/build/Microsoft.Build.Tasks.Git.props b/src/Microsoft.Build.Tasks.Git/build/Microsoft.Build.Tasks.Git.props index 308fd217..16ef7e3c 100644 --- a/src/Microsoft.Build.Tasks.Git/build/Microsoft.Build.Tasks.Git.props +++ b/src/Microsoft.Build.Tasks.Git/build/Microsoft.Build.Tasks.Git.props @@ -1,4 +1,5 @@ - + + $(MSBuildThisFileDirectory)..\tools\net472\Microsoft.Build.Tasks.Git.dll diff --git a/src/Microsoft.Build.Tasks.Git/build/Microsoft.Build.Tasks.Git.targets b/src/Microsoft.Build.Tasks.Git/build/Microsoft.Build.Tasks.Git.targets index 213daf5e..1e26dec0 100644 --- a/src/Microsoft.Build.Tasks.Git/build/Microsoft.Build.Tasks.Git.targets +++ b/src/Microsoft.Build.Tasks.Git/build/Microsoft.Build.Tasks.Git.targets @@ -1,4 +1,5 @@ - + + diff --git a/src/Microsoft.Build.Tasks.Git/buildMultiTargeting/Microsoft.Build.Tasks.Git.props b/src/Microsoft.Build.Tasks.Git/buildMultiTargeting/Microsoft.Build.Tasks.Git.props index 9c26c672..afadd0e1 100644 --- a/src/Microsoft.Build.Tasks.Git/buildMultiTargeting/Microsoft.Build.Tasks.Git.props +++ b/src/Microsoft.Build.Tasks.Git/buildMultiTargeting/Microsoft.Build.Tasks.Git.props @@ -1,4 +1,5 @@ - + + diff --git a/src/Microsoft.Build.Tasks.Git/buildMultiTargeting/Microsoft.Build.Tasks.Git.targets b/src/Microsoft.Build.Tasks.Git/buildMultiTargeting/Microsoft.Build.Tasks.Git.targets index 84afbc37..c72d0e0c 100644 --- a/src/Microsoft.Build.Tasks.Git/buildMultiTargeting/Microsoft.Build.Tasks.Git.targets +++ b/src/Microsoft.Build.Tasks.Git/buildMultiTargeting/Microsoft.Build.Tasks.Git.targets @@ -1,4 +1,5 @@ - + + diff --git a/src/Microsoft.Build.Tasks.Tfvc/GetRepositoryUrl.cs b/src/Microsoft.Build.Tasks.Tfvc/GetRepositoryUrl.cs index 3f476dca..5da4b3e6 100644 --- a/src/Microsoft.Build.Tasks.Tfvc/GetRepositoryUrl.cs +++ b/src/Microsoft.Build.Tasks.Tfvc/GetRepositoryUrl.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.IO; diff --git a/src/Microsoft.Build.Tasks.Tfvc/GetSourceRevisionId.cs b/src/Microsoft.Build.Tasks.Tfvc/GetSourceRevisionId.cs index 253ae8e9..a31421a3 100644 --- a/src/Microsoft.Build.Tasks.Tfvc/GetSourceRevisionId.cs +++ b/src/Microsoft.Build.Tasks.Tfvc/GetSourceRevisionId.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using Microsoft.Build.Framework; using Microsoft.TeamFoundation.Client; diff --git a/src/Microsoft.Build.Tasks.Tfvc/GetSourceRoots.cs b/src/Microsoft.Build.Tasks.Tfvc/GetSourceRoots.cs index b35f74c6..6df2edff 100644 --- a/src/Microsoft.Build.Tasks.Tfvc/GetSourceRoots.cs +++ b/src/Microsoft.Build.Tasks.Tfvc/GetSourceRoots.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Collections.Generic; diff --git a/src/Microsoft.Build.Tasks.Tfvc/GetUntrackedSourceFiles.cs b/src/Microsoft.Build.Tasks.Tfvc/GetUntrackedSourceFiles.cs index 21469d67..d049007f 100644 --- a/src/Microsoft.Build.Tasks.Tfvc/GetUntrackedSourceFiles.cs +++ b/src/Microsoft.Build.Tasks.Tfvc/GetUntrackedSourceFiles.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Collections.Generic; diff --git a/src/Microsoft.Build.Tasks.Tfvc/LocateRepository.cs b/src/Microsoft.Build.Tasks.Tfvc/LocateRepository.cs index 1c44ee09..a7cc99ab 100644 --- a/src/Microsoft.Build.Tasks.Tfvc/LocateRepository.cs +++ b/src/Microsoft.Build.Tasks.Tfvc/LocateRepository.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System.Diagnostics.CodeAnalysis; using Microsoft.Build.Framework; diff --git a/src/Microsoft.Build.Tasks.Tfvc/Microsoft.Build.Tasks.Tfvc.csproj b/src/Microsoft.Build.Tasks.Tfvc/Microsoft.Build.Tasks.Tfvc.csproj index 2c6b3458..93b8bf35 100644 --- a/src/Microsoft.Build.Tasks.Tfvc/Microsoft.Build.Tasks.Tfvc.csproj +++ b/src/Microsoft.Build.Tasks.Tfvc/Microsoft.Build.Tasks.Tfvc.csproj @@ -1,4 +1,6 @@ - + + + net472 true diff --git a/src/Microsoft.Build.Tasks.Tfvc/RepositoryTask.cs b/src/Microsoft.Build.Tasks.Tfvc/RepositoryTask.cs index 385cf5cd..3ad7e48a 100644 --- a/src/Microsoft.Build.Tasks.Tfvc/RepositoryTask.cs +++ b/src/Microsoft.Build.Tasks.Tfvc/RepositoryTask.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System.Diagnostics.CodeAnalysis; using Microsoft.Build.Framework; diff --git a/src/Microsoft.Build.Tasks.Tfvc/build/Microsoft.Build.Tasks.Tfvc.props b/src/Microsoft.Build.Tasks.Tfvc/build/Microsoft.Build.Tasks.Tfvc.props index aee1e765..64301247 100644 --- a/src/Microsoft.Build.Tasks.Tfvc/build/Microsoft.Build.Tasks.Tfvc.props +++ b/src/Microsoft.Build.Tasks.Tfvc/build/Microsoft.Build.Tasks.Tfvc.props @@ -1,6 +1,7 @@ - + + $(MSBuildThisFileDirectory)..\tools\net472\Microsoft.Build.Tasks.Tfvc.dll - \ No newline at end of file + diff --git a/src/Microsoft.Build.Tasks.Tfvc/build/Microsoft.Build.Tasks.Tfvc.targets b/src/Microsoft.Build.Tasks.Tfvc/build/Microsoft.Build.Tasks.Tfvc.targets index fe9a936f..0bc8d3ae 100644 --- a/src/Microsoft.Build.Tasks.Tfvc/build/Microsoft.Build.Tasks.Tfvc.targets +++ b/src/Microsoft.Build.Tasks.Tfvc/build/Microsoft.Build.Tasks.Tfvc.targets @@ -1,4 +1,5 @@ - + + diff --git a/src/Microsoft.Build.Tasks.Tfvc/buildMultiTargeting/Microsoft.Build.Tasks.Tfvc.props b/src/Microsoft.Build.Tasks.Tfvc/buildMultiTargeting/Microsoft.Build.Tasks.Tfvc.props index 9c26c672..afadd0e1 100644 --- a/src/Microsoft.Build.Tasks.Tfvc/buildMultiTargeting/Microsoft.Build.Tasks.Tfvc.props +++ b/src/Microsoft.Build.Tasks.Tfvc/buildMultiTargeting/Microsoft.Build.Tasks.Tfvc.props @@ -1,4 +1,5 @@ - + + diff --git a/src/Microsoft.Build.Tasks.Tfvc/buildMultiTargeting/Microsoft.Build.Tasks.Tfvc.targets b/src/Microsoft.Build.Tasks.Tfvc/buildMultiTargeting/Microsoft.Build.Tasks.Tfvc.targets index 84afbc37..c72d0e0c 100644 --- a/src/Microsoft.Build.Tasks.Tfvc/buildMultiTargeting/Microsoft.Build.Tasks.Tfvc.targets +++ b/src/Microsoft.Build.Tasks.Tfvc/buildMultiTargeting/Microsoft.Build.Tasks.Tfvc.targets @@ -1,4 +1,5 @@ - + + diff --git a/src/SourceLink.AzureDevOpsServer.Git.UnitTests/AzureDevOpsUrlParserOnPremTests.cs b/src/SourceLink.AzureDevOpsServer.Git.UnitTests/AzureDevOpsUrlParserOnPremTests.cs index 8577b23d..95e5bc49 100644 --- a/src/SourceLink.AzureDevOpsServer.Git.UnitTests/AzureDevOpsUrlParserOnPremTests.cs +++ b/src/SourceLink.AzureDevOpsServer.Git.UnitTests/AzureDevOpsUrlParserOnPremTests.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using Xunit; diff --git a/src/SourceLink.AzureDevOpsServer.Git.UnitTests/GetSourceLinkUrlTests.cs b/src/SourceLink.AzureDevOpsServer.Git.UnitTests/GetSourceLinkUrlTests.cs index 661c4561..bb5fec9a 100644 --- a/src/SourceLink.AzureDevOpsServer.Git.UnitTests/GetSourceLinkUrlTests.cs +++ b/src/SourceLink.AzureDevOpsServer.Git.UnitTests/GetSourceLinkUrlTests.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using Microsoft.Build.Tasks.SourceControl; using TestUtilities; diff --git a/src/SourceLink.AzureDevOpsServer.Git.UnitTests/Microsoft.SourceLink.AzureDevOpsServer.Git.UnitTests.csproj b/src/SourceLink.AzureDevOpsServer.Git.UnitTests/Microsoft.SourceLink.AzureDevOpsServer.Git.UnitTests.csproj index 610f1e75..8754e0ac 100644 --- a/src/SourceLink.AzureDevOpsServer.Git.UnitTests/Microsoft.SourceLink.AzureDevOpsServer.Git.UnitTests.csproj +++ b/src/SourceLink.AzureDevOpsServer.Git.UnitTests/Microsoft.SourceLink.AzureDevOpsServer.Git.UnitTests.csproj @@ -1,4 +1,6 @@ - + + + net472;net6.0 diff --git a/src/SourceLink.AzureDevOpsServer.Git.UnitTests/TranslateRepositoryUrlsTests.cs b/src/SourceLink.AzureDevOpsServer.Git.UnitTests/TranslateRepositoryUrlsTests.cs index 81ef6a32..8e9b3f5f 100644 --- a/src/SourceLink.AzureDevOpsServer.Git.UnitTests/TranslateRepositoryUrlsTests.cs +++ b/src/SourceLink.AzureDevOpsServer.Git.UnitTests/TranslateRepositoryUrlsTests.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System.Linq; using TestUtilities; using Xunit; diff --git a/src/SourceLink.AzureDevOpsServer.Git/GetSourceLinkUrl.cs b/src/SourceLink.AzureDevOpsServer.Git/GetSourceLinkUrl.cs index 5a6d516e..c70f4465 100644 --- a/src/SourceLink.AzureDevOpsServer.Git/GetSourceLinkUrl.cs +++ b/src/SourceLink.AzureDevOpsServer.Git/GetSourceLinkUrl.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using Microsoft.Build.Framework; diff --git a/src/SourceLink.AzureDevOpsServer.Git/Microsoft.SourceLink.AzureDevOpsServer.Git.csproj b/src/SourceLink.AzureDevOpsServer.Git/Microsoft.SourceLink.AzureDevOpsServer.Git.csproj index c59c8b4b..ceec252b 100644 --- a/src/SourceLink.AzureDevOpsServer.Git/Microsoft.SourceLink.AzureDevOpsServer.Git.csproj +++ b/src/SourceLink.AzureDevOpsServer.Git/Microsoft.SourceLink.AzureDevOpsServer.Git.csproj @@ -1,4 +1,6 @@ - + + + net472;net6.0 true diff --git a/src/SourceLink.AzureDevOpsServer.Git/TranslateRepositoryUrls.cs b/src/SourceLink.AzureDevOpsServer.Git/TranslateRepositoryUrls.cs index 9b12b949..2994302f 100644 --- a/src/SourceLink.AzureDevOpsServer.Git/TranslateRepositoryUrls.cs +++ b/src/SourceLink.AzureDevOpsServer.Git/TranslateRepositoryUrls.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using Microsoft.Build.Tasks.SourceControl; diff --git a/src/SourceLink.AzureDevOpsServer.Git/build/Microsoft.SourceLink.AzureDevOpsServer.Git.targets b/src/SourceLink.AzureDevOpsServer.Git/build/Microsoft.SourceLink.AzureDevOpsServer.Git.targets index 8590b7d5..2e0e3cb3 100644 --- a/src/SourceLink.AzureDevOpsServer.Git/build/Microsoft.SourceLink.AzureDevOpsServer.Git.targets +++ b/src/SourceLink.AzureDevOpsServer.Git/build/Microsoft.SourceLink.AzureDevOpsServer.Git.targets @@ -1,4 +1,5 @@ - + + <_SourceLinkAzureDevOpsServerGitAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net472\Microsoft.SourceLink.AzureDevOpsServer.Git.dll @@ -68,4 +69,4 @@ - \ No newline at end of file + diff --git a/src/SourceLink.AzureDevOpsServer.Git/buildMultiTargeting/Microsoft.SourceLink.AzureDevOpsServer.Git.targets b/src/SourceLink.AzureDevOpsServer.Git/buildMultiTargeting/Microsoft.SourceLink.AzureDevOpsServer.Git.targets index 84afbc37..c72d0e0c 100644 --- a/src/SourceLink.AzureDevOpsServer.Git/buildMultiTargeting/Microsoft.SourceLink.AzureDevOpsServer.Git.targets +++ b/src/SourceLink.AzureDevOpsServer.Git/buildMultiTargeting/Microsoft.SourceLink.AzureDevOpsServer.Git.targets @@ -1,4 +1,5 @@ - + + diff --git a/src/SourceLink.AzureRepos.Git.UnitTests/AzureDevOpsUrlParserHostedTests.cs b/src/SourceLink.AzureRepos.Git.UnitTests/AzureDevOpsUrlParserHostedTests.cs index cbdac190..2284ff1f 100644 --- a/src/SourceLink.AzureRepos.Git.UnitTests/AzureDevOpsUrlParserHostedTests.cs +++ b/src/SourceLink.AzureRepos.Git.UnitTests/AzureDevOpsUrlParserHostedTests.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using Xunit; diff --git a/src/SourceLink.AzureRepos.Git.UnitTests/GetSourceLinkUrlTests.cs b/src/SourceLink.AzureRepos.Git.UnitTests/GetSourceLinkUrlTests.cs index 46854350..a84eb067 100644 --- a/src/SourceLink.AzureRepos.Git.UnitTests/GetSourceLinkUrlTests.cs +++ b/src/SourceLink.AzureRepos.Git.UnitTests/GetSourceLinkUrlTests.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using Microsoft.Build.Tasks.SourceControl; using TestUtilities; diff --git a/src/SourceLink.AzureRepos.Git.UnitTests/Microsoft.SourceLink.AzureRepos.Git.UnitTests.csproj b/src/SourceLink.AzureRepos.Git.UnitTests/Microsoft.SourceLink.AzureRepos.Git.UnitTests.csproj index 60d3a8ff..f8cc548b 100644 --- a/src/SourceLink.AzureRepos.Git.UnitTests/Microsoft.SourceLink.AzureRepos.Git.UnitTests.csproj +++ b/src/SourceLink.AzureRepos.Git.UnitTests/Microsoft.SourceLink.AzureRepos.Git.UnitTests.csproj @@ -1,4 +1,6 @@ - + + + net472;net6.0 diff --git a/src/SourceLink.AzureRepos.Git.UnitTests/TranslateRepositoryUrlsTests.cs b/src/SourceLink.AzureRepos.Git.UnitTests/TranslateRepositoryUrlsTests.cs index adbb8c0a..5ad83c5d 100644 --- a/src/SourceLink.AzureRepos.Git.UnitTests/TranslateRepositoryUrlsTests.cs +++ b/src/SourceLink.AzureRepos.Git.UnitTests/TranslateRepositoryUrlsTests.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System.Linq; using TestUtilities; using Xunit; diff --git a/src/SourceLink.AzureRepos.Git/GetSourceLinkUrl.cs b/src/SourceLink.AzureRepos.Git/GetSourceLinkUrl.cs index 5a74a384..f7116fa3 100644 --- a/src/SourceLink.AzureRepos.Git/GetSourceLinkUrl.cs +++ b/src/SourceLink.AzureRepos.Git/GetSourceLinkUrl.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Collections.Generic; diff --git a/src/SourceLink.AzureRepos.Git/Microsoft.SourceLink.AzureRepos.Git.csproj b/src/SourceLink.AzureRepos.Git/Microsoft.SourceLink.AzureRepos.Git.csproj index 191ca700..bdb048e8 100644 --- a/src/SourceLink.AzureRepos.Git/Microsoft.SourceLink.AzureRepos.Git.csproj +++ b/src/SourceLink.AzureRepos.Git/Microsoft.SourceLink.AzureRepos.Git.csproj @@ -1,4 +1,6 @@ - + + + net472;net6.0 true diff --git a/src/SourceLink.AzureRepos.Git/TranslateRepositoryUrls.cs b/src/SourceLink.AzureRepos.Git/TranslateRepositoryUrls.cs index 73452eec..4dbdf60b 100644 --- a/src/SourceLink.AzureRepos.Git/TranslateRepositoryUrls.cs +++ b/src/SourceLink.AzureRepos.Git/TranslateRepositoryUrls.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using Microsoft.Build.Tasks.SourceControl; diff --git a/src/SourceLink.AzureRepos.Git/build/Microsoft.SourceLink.AzureRepos.Git.props b/src/SourceLink.AzureRepos.Git/build/Microsoft.SourceLink.AzureRepos.Git.props index c258402d..ca9cbc82 100644 --- a/src/SourceLink.AzureRepos.Git/build/Microsoft.SourceLink.AzureRepos.Git.props +++ b/src/SourceLink.AzureRepos.Git/build/Microsoft.SourceLink.AzureRepos.Git.props @@ -1,4 +1,5 @@ - + + diff --git a/src/SourceLink.AzureRepos.Git/build/Microsoft.SourceLink.AzureRepos.Git.targets b/src/SourceLink.AzureRepos.Git/build/Microsoft.SourceLink.AzureRepos.Git.targets index c07157a7..d431e138 100644 --- a/src/SourceLink.AzureRepos.Git/build/Microsoft.SourceLink.AzureRepos.Git.targets +++ b/src/SourceLink.AzureRepos.Git/build/Microsoft.SourceLink.AzureRepos.Git.targets @@ -1,4 +1,5 @@ - + + <_SourceLinkAzureReposGitAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net472\Microsoft.SourceLink.AzureRepos.Git.dll @@ -55,4 +56,4 @@ - \ No newline at end of file + diff --git a/src/SourceLink.AzureRepos.Git/buildMultiTargeting/Microsoft.SourceLink.AzureRepos.Git.props b/src/SourceLink.AzureRepos.Git/buildMultiTargeting/Microsoft.SourceLink.AzureRepos.Git.props index 9c26c672..afadd0e1 100644 --- a/src/SourceLink.AzureRepos.Git/buildMultiTargeting/Microsoft.SourceLink.AzureRepos.Git.props +++ b/src/SourceLink.AzureRepos.Git/buildMultiTargeting/Microsoft.SourceLink.AzureRepos.Git.props @@ -1,4 +1,5 @@ - + + diff --git a/src/SourceLink.AzureRepos.Git/buildMultiTargeting/Microsoft.SourceLink.AzureRepos.Git.targets b/src/SourceLink.AzureRepos.Git/buildMultiTargeting/Microsoft.SourceLink.AzureRepos.Git.targets index 84afbc37..c72d0e0c 100644 --- a/src/SourceLink.AzureRepos.Git/buildMultiTargeting/Microsoft.SourceLink.AzureRepos.Git.targets +++ b/src/SourceLink.AzureRepos.Git/buildMultiTargeting/Microsoft.SourceLink.AzureRepos.Git.targets @@ -1,4 +1,5 @@ - + + diff --git a/src/SourceLink.AzureRepos.Tfvc/GetSourceLinkUrl.cs b/src/SourceLink.AzureRepos.Tfvc/GetSourceLinkUrl.cs index 59794c42..92528bca 100644 --- a/src/SourceLink.AzureRepos.Tfvc/GetSourceLinkUrl.cs +++ b/src/SourceLink.AzureRepos.Tfvc/GetSourceLinkUrl.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Diagnostics.CodeAnalysis; diff --git a/src/SourceLink.AzureRepos.Tfvc/Microsoft.SourceLink.AzureRepos.Tfvc.csproj b/src/SourceLink.AzureRepos.Tfvc/Microsoft.SourceLink.AzureRepos.Tfvc.csproj index 91af51fe..fddc9662 100644 --- a/src/SourceLink.AzureRepos.Tfvc/Microsoft.SourceLink.AzureRepos.Tfvc.csproj +++ b/src/SourceLink.AzureRepos.Tfvc/Microsoft.SourceLink.AzureRepos.Tfvc.csproj @@ -1,4 +1,6 @@ - + + + net472 true diff --git a/src/SourceLink.AzureRepos.Tfvc/build/Microsoft.SourceLink.AzureRepos.Tfvc.targets b/src/SourceLink.AzureRepos.Tfvc/build/Microsoft.SourceLink.AzureRepos.Tfvc.targets index 5b7be15d..27fb8054 100644 --- a/src/SourceLink.AzureRepos.Tfvc/build/Microsoft.SourceLink.AzureRepos.Tfvc.targets +++ b/src/SourceLink.AzureRepos.Tfvc/build/Microsoft.SourceLink.AzureRepos.Tfvc.targets @@ -1,4 +1,5 @@ - + + <_SourceLinkAzureReposTfvcAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net472\Microsoft.SourceLink.AzureRepos.Tfvc.dll @@ -22,4 +23,4 @@ - \ No newline at end of file + diff --git a/src/SourceLink.AzureRepos.Tfvc/buildMultiTargeting/Microsoft.SourceLink.AzureRepos.Tfvc.targets b/src/SourceLink.AzureRepos.Tfvc/buildMultiTargeting/Microsoft.SourceLink.AzureRepos.Tfvc.targets index 84afbc37..c72d0e0c 100644 --- a/src/SourceLink.AzureRepos.Tfvc/buildMultiTargeting/Microsoft.SourceLink.AzureRepos.Tfvc.targets +++ b/src/SourceLink.AzureRepos.Tfvc/buildMultiTargeting/Microsoft.SourceLink.AzureRepos.Tfvc.targets @@ -1,4 +1,5 @@ - + + diff --git a/src/SourceLink.Bitbucket.Git.UnitTests/GetSourceLinkUrlTests.cs b/src/SourceLink.Bitbucket.Git.UnitTests/GetSourceLinkUrlTests.cs index 1d7f45e9..10cb201a 100644 --- a/src/SourceLink.Bitbucket.Git.UnitTests/GetSourceLinkUrlTests.cs +++ b/src/SourceLink.Bitbucket.Git.UnitTests/GetSourceLinkUrlTests.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using Microsoft.Build.Evaluation; using Microsoft.Build.Tasks.SourceControl; diff --git a/src/SourceLink.Bitbucket.Git.UnitTests/Microsoft.SourceLink.Bitbucket.Git.UnitTests.csproj b/src/SourceLink.Bitbucket.Git.UnitTests/Microsoft.SourceLink.Bitbucket.Git.UnitTests.csproj index c33cf00f..f20a90b2 100644 --- a/src/SourceLink.Bitbucket.Git.UnitTests/Microsoft.SourceLink.Bitbucket.Git.UnitTests.csproj +++ b/src/SourceLink.Bitbucket.Git.UnitTests/Microsoft.SourceLink.Bitbucket.Git.UnitTests.csproj @@ -1,4 +1,6 @@ - + + + net472;net6.0 diff --git a/src/SourceLink.Bitbucket.Git.UnitTests/TranslateRepositoryUrlsTests.cs b/src/SourceLink.Bitbucket.Git.UnitTests/TranslateRepositoryUrlsTests.cs index 68af468d..4d3f9bfd 100644 --- a/src/SourceLink.Bitbucket.Git.UnitTests/TranslateRepositoryUrlsTests.cs +++ b/src/SourceLink.Bitbucket.Git.UnitTests/TranslateRepositoryUrlsTests.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System.Linq; using TestUtilities; using Xunit; diff --git a/src/SourceLink.Bitbucket.Git/GetSourceLinkUrl.cs b/src/SourceLink.Bitbucket.Git/GetSourceLinkUrl.cs index 66c66349..eea57757 100644 --- a/src/SourceLink.Bitbucket.Git/GetSourceLinkUrl.cs +++ b/src/SourceLink.Bitbucket.Git/GetSourceLinkUrl.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Diagnostics; diff --git a/src/SourceLink.Bitbucket.Git/Microsoft.SourceLink.Bitbucket.Git.csproj b/src/SourceLink.Bitbucket.Git/Microsoft.SourceLink.Bitbucket.Git.csproj index 86fe6336..bf27b3fc 100644 --- a/src/SourceLink.Bitbucket.Git/Microsoft.SourceLink.Bitbucket.Git.csproj +++ b/src/SourceLink.Bitbucket.Git/Microsoft.SourceLink.Bitbucket.Git.csproj @@ -1,4 +1,6 @@ - + + + net472;net6.0 true diff --git a/src/SourceLink.Bitbucket.Git/TranslateRepositoryUrls.cs b/src/SourceLink.Bitbucket.Git/TranslateRepositoryUrls.cs index 75189e9e..0892dda5 100644 --- a/src/SourceLink.Bitbucket.Git/TranslateRepositoryUrls.cs +++ b/src/SourceLink.Bitbucket.Git/TranslateRepositoryUrls.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using Microsoft.Build.Tasks.SourceControl; diff --git a/src/SourceLink.Bitbucket.Git/build/Microsoft.SourceLink.Bitbucket.Git.props b/src/SourceLink.Bitbucket.Git/build/Microsoft.SourceLink.Bitbucket.Git.props index 8710d6aa..5c99acf2 100644 --- a/src/SourceLink.Bitbucket.Git/build/Microsoft.SourceLink.Bitbucket.Git.props +++ b/src/SourceLink.Bitbucket.Git/build/Microsoft.SourceLink.Bitbucket.Git.props @@ -1,4 +1,5 @@ - + + diff --git a/src/SourceLink.Bitbucket.Git/build/Microsoft.SourceLink.Bitbucket.Git.targets b/src/SourceLink.Bitbucket.Git/build/Microsoft.SourceLink.Bitbucket.Git.targets index d826e799..7a6d4eeb 100644 --- a/src/SourceLink.Bitbucket.Git/build/Microsoft.SourceLink.Bitbucket.Git.targets +++ b/src/SourceLink.Bitbucket.Git/build/Microsoft.SourceLink.Bitbucket.Git.targets @@ -1,4 +1,5 @@ - + + <_SourceLinkBitbucketAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net472\Microsoft.SourceLink.Bitbucket.Git.dll diff --git a/src/SourceLink.Bitbucket.Git/buildMultiTargeting/Microsoft.SourceLink.Bitbucket.Git.props b/src/SourceLink.Bitbucket.Git/buildMultiTargeting/Microsoft.SourceLink.Bitbucket.Git.props index 9c26c672..afadd0e1 100644 --- a/src/SourceLink.Bitbucket.Git/buildMultiTargeting/Microsoft.SourceLink.Bitbucket.Git.props +++ b/src/SourceLink.Bitbucket.Git/buildMultiTargeting/Microsoft.SourceLink.Bitbucket.Git.props @@ -1,4 +1,5 @@ - + + diff --git a/src/SourceLink.Bitbucket.Git/buildMultiTargeting/Microsoft.SourceLink.Bitbucket.Git.targets b/src/SourceLink.Bitbucket.Git/buildMultiTargeting/Microsoft.SourceLink.Bitbucket.Git.targets index 84afbc37..c72d0e0c 100644 --- a/src/SourceLink.Bitbucket.Git/buildMultiTargeting/Microsoft.SourceLink.Bitbucket.Git.targets +++ b/src/SourceLink.Bitbucket.Git/buildMultiTargeting/Microsoft.SourceLink.Bitbucket.Git.targets @@ -1,4 +1,5 @@ - + + diff --git a/src/SourceLink.Common.UnitTests/GenerateSourceLinkFileTests.cs b/src/SourceLink.Common.UnitTests/GenerateSourceLinkFileTests.cs index 6c04671a..07fa14be 100644 --- a/src/SourceLink.Common.UnitTests/GenerateSourceLinkFileTests.cs +++ b/src/SourceLink.Common.UnitTests/GenerateSourceLinkFileTests.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.IO; using Xunit; diff --git a/src/SourceLink.Common.UnitTests/GetSourceLinkUrlTests.cs b/src/SourceLink.Common.UnitTests/GetSourceLinkUrlTests.cs index 96a7dfc5..59ae3b31 100644 --- a/src/SourceLink.Common.UnitTests/GetSourceLinkUrlTests.cs +++ b/src/SourceLink.Common.UnitTests/GetSourceLinkUrlTests.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using Microsoft.Build.Tasks.SourceControl; using TestUtilities; diff --git a/src/SourceLink.Common.UnitTests/Microsoft.SourceLink.Common.UnitTests.csproj b/src/SourceLink.Common.UnitTests/Microsoft.SourceLink.Common.UnitTests.csproj index 45b74dcf..483cab5c 100644 --- a/src/SourceLink.Common.UnitTests/Microsoft.SourceLink.Common.UnitTests.csproj +++ b/src/SourceLink.Common.UnitTests/Microsoft.SourceLink.Common.UnitTests.csproj @@ -1,4 +1,6 @@ - + + + net472;net6.0 diff --git a/src/SourceLink.Common.UnitTests/Mocks/MockGetSourceLinkUrlGitTask.cs b/src/SourceLink.Common.UnitTests/Mocks/MockGetSourceLinkUrlGitTask.cs index 34e1e869..a3707255 100644 --- a/src/SourceLink.Common.UnitTests/Mocks/MockGetSourceLinkUrlGitTask.cs +++ b/src/SourceLink.Common.UnitTests/Mocks/MockGetSourceLinkUrlGitTask.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.IO; using Xunit; diff --git a/src/SourceLink.Common.UnitTests/PathUtilitiesTests.cs b/src/SourceLink.Common.UnitTests/PathUtilitiesTests.cs index 431ac354..d9cb5914 100644 --- a/src/SourceLink.Common.UnitTests/PathUtilitiesTests.cs +++ b/src/SourceLink.Common.UnitTests/PathUtilitiesTests.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using TestUtilities; using Microsoft.Build.Tasks.SourceControl; using Xunit; diff --git a/src/SourceLink.Common.UnitTests/TranslateRepositoryUrlsTests.cs b/src/SourceLink.Common.UnitTests/TranslateRepositoryUrlsTests.cs index 9c77a292..a02060bf 100644 --- a/src/SourceLink.Common.UnitTests/TranslateRepositoryUrlsTests.cs +++ b/src/SourceLink.Common.UnitTests/TranslateRepositoryUrlsTests.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System.Linq; using Microsoft.Build.Tasks.SourceControl; using TestUtilities; diff --git a/src/SourceLink.Common.UnitTests/UriUtilitiesTests.cs b/src/SourceLink.Common.UnitTests/UriUtilitiesTests.cs index 9ab1dbb2..ebd428ae 100644 --- a/src/SourceLink.Common.UnitTests/UriUtilitiesTests.cs +++ b/src/SourceLink.Common.UnitTests/UriUtilitiesTests.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using TestUtilities; using Microsoft.Build.Tasks.SourceControl; using Xunit; diff --git a/src/SourceLink.Common/GenerateSourceLinkFile.cs b/src/SourceLink.Common/GenerateSourceLinkFile.cs index b3d763b2..124c937a 100644 --- a/src/SourceLink.Common/GenerateSourceLinkFile.cs +++ b/src/SourceLink.Common/GenerateSourceLinkFile.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Diagnostics.CodeAnalysis; diff --git a/src/SourceLink.Common/Microsoft.SourceLink.Common.csproj b/src/SourceLink.Common/Microsoft.SourceLink.Common.csproj index 66636da6..09ac8ec5 100644 --- a/src/SourceLink.Common/Microsoft.SourceLink.Common.csproj +++ b/src/SourceLink.Common/Microsoft.SourceLink.Common.csproj @@ -1,4 +1,6 @@ - + + + net472;net6.0 true diff --git a/src/SourceLink.Common/SourceLinkHasSingleProvider.cs b/src/SourceLink.Common/SourceLinkHasSingleProvider.cs index c08290e0..98a69084 100644 --- a/src/SourceLink.Common/SourceLinkHasSingleProvider.cs +++ b/src/SourceLink.Common/SourceLinkHasSingleProvider.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using Microsoft.Build.Framework; diff --git a/src/SourceLink.Common/build/InitializeSourceControlInformation.targets b/src/SourceLink.Common/build/InitializeSourceControlInformation.targets index 27233b9b..81703fcb 100644 --- a/src/SourceLink.Common/build/InitializeSourceControlInformation.targets +++ b/src/SourceLink.Common/build/InitializeSourceControlInformation.targets @@ -1,4 +1,5 @@  + diff --git a/src/SourceLink.Common/build/Microsoft.SourceLink.Common.props b/src/SourceLink.Common/build/Microsoft.SourceLink.Common.props index 8840a5e6..2b056fe9 100644 --- a/src/SourceLink.Common/build/Microsoft.SourceLink.Common.props +++ b/src/SourceLink.Common/build/Microsoft.SourceLink.Common.props @@ -1,4 +1,5 @@ - + + <_MicrosoftSourceLinkCommonAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net472\Microsoft.SourceLink.Common.dll diff --git a/src/SourceLink.Common/build/Microsoft.SourceLink.Common.targets b/src/SourceLink.Common/build/Microsoft.SourceLink.Common.targets index 8b84623a..9a7f4d59 100644 --- a/src/SourceLink.Common/build/Microsoft.SourceLink.Common.targets +++ b/src/SourceLink.Common/build/Microsoft.SourceLink.Common.targets @@ -1,4 +1,5 @@ - + + diff --git a/src/SourceLink.Common/buildMultiTargeting/Microsoft.SourceLink.Common.props b/src/SourceLink.Common/buildMultiTargeting/Microsoft.SourceLink.Common.props index 9c26c672..afadd0e1 100644 --- a/src/SourceLink.Common/buildMultiTargeting/Microsoft.SourceLink.Common.props +++ b/src/SourceLink.Common/buildMultiTargeting/Microsoft.SourceLink.Common.props @@ -1,4 +1,5 @@ - + + diff --git a/src/SourceLink.Common/buildMultiTargeting/Microsoft.SourceLink.Common.targets b/src/SourceLink.Common/buildMultiTargeting/Microsoft.SourceLink.Common.targets index 8ff10e2f..8e8e3be3 100644 --- a/src/SourceLink.Common/buildMultiTargeting/Microsoft.SourceLink.Common.targets +++ b/src/SourceLink.Common/buildMultiTargeting/Microsoft.SourceLink.Common.targets @@ -1,4 +1,5 @@ - + + diff --git a/src/SourceLink.Git.IntegrationTests/AzureDevOpsServerTests.cs b/src/SourceLink.Git.IntegrationTests/AzureDevOpsServerTests.cs index 0f9fa83d..a3772897 100644 --- a/src/SourceLink.Git.IntegrationTests/AzureDevOpsServerTests.cs +++ b/src/SourceLink.Git.IntegrationTests/AzureDevOpsServerTests.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System.IO; using TestUtilities; diff --git a/src/SourceLink.Git.IntegrationTests/AzureReposAndGitHubTests.cs b/src/SourceLink.Git.IntegrationTests/AzureReposAndGitHubTests.cs index 21d2b6b1..fb043223 100644 --- a/src/SourceLink.Git.IntegrationTests/AzureReposAndGitHubTests.cs +++ b/src/SourceLink.Git.IntegrationTests/AzureReposAndGitHubTests.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System.IO; using Microsoft.SourceLink.Common; diff --git a/src/SourceLink.Git.IntegrationTests/AzureReposTests.cs b/src/SourceLink.Git.IntegrationTests/AzureReposTests.cs index 6a488d16..b8542707 100644 --- a/src/SourceLink.Git.IntegrationTests/AzureReposTests.cs +++ b/src/SourceLink.Git.IntegrationTests/AzureReposTests.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System.IO; using TestUtilities; diff --git a/src/SourceLink.Git.IntegrationTests/BitbucketGitTests.cs b/src/SourceLink.Git.IntegrationTests/BitbucketGitTests.cs index efefa8e4..6a37e7b2 100644 --- a/src/SourceLink.Git.IntegrationTests/BitbucketGitTests.cs +++ b/src/SourceLink.Git.IntegrationTests/BitbucketGitTests.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System.IO; using TestUtilities; diff --git a/src/SourceLink.Git.IntegrationTests/GitHubTests.cs b/src/SourceLink.Git.IntegrationTests/GitHubTests.cs index 324d4322..81cd9714 100644 --- a/src/SourceLink.Git.IntegrationTests/GitHubTests.cs +++ b/src/SourceLink.Git.IntegrationTests/GitHubTests.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.IO; diff --git a/src/SourceLink.Git.IntegrationTests/GitLabTests.cs b/src/SourceLink.Git.IntegrationTests/GitLabTests.cs index 5185cc49..5707f8d2 100644 --- a/src/SourceLink.Git.IntegrationTests/GitLabTests.cs +++ b/src/SourceLink.Git.IntegrationTests/GitLabTests.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System.IO; using TestUtilities; diff --git a/src/SourceLink.Git.IntegrationTests/GitWebTests.cs b/src/SourceLink.Git.IntegrationTests/GitWebTests.cs index 22080050..5d37d787 100644 --- a/src/SourceLink.Git.IntegrationTests/GitWebTests.cs +++ b/src/SourceLink.Git.IntegrationTests/GitWebTests.cs @@ -1,5 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See -// License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using Microsoft.Build.Tasks.SourceControl; using System; diff --git a/src/SourceLink.Git.IntegrationTests/GiteaTests.cs b/src/SourceLink.Git.IntegrationTests/GiteaTests.cs index d0c06e9c..7839062b 100644 --- a/src/SourceLink.Git.IntegrationTests/GiteaTests.cs +++ b/src/SourceLink.Git.IntegrationTests/GiteaTests.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System.IO; using TestUtilities; diff --git a/src/SourceLink.Git.IntegrationTests/Microsoft.SourceLink.Git.IntegrationTests.csproj b/src/SourceLink.Git.IntegrationTests/Microsoft.SourceLink.Git.IntegrationTests.csproj index 00993826..8a036556 100644 --- a/src/SourceLink.Git.IntegrationTests/Microsoft.SourceLink.Git.IntegrationTests.csproj +++ b/src/SourceLink.Git.IntegrationTests/Microsoft.SourceLink.Git.IntegrationTests.csproj @@ -1,4 +1,6 @@ - + + + net472;net6.0 diff --git a/src/SourceLink.Git.IntegrationTests/TargetTests.cs b/src/SourceLink.Git.IntegrationTests/TargetTests.cs index 68b63dd0..8b452a39 100644 --- a/src/SourceLink.Git.IntegrationTests/TargetTests.cs +++ b/src/SourceLink.Git.IntegrationTests/TargetTests.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.IO; diff --git a/src/SourceLink.Git.IntegrationTests/Utilities/GitUtilities.cs b/src/SourceLink.Git.IntegrationTests/Utilities/GitUtilities.cs index 0e53153d..021cdc37 100644 --- a/src/SourceLink.Git.IntegrationTests/Utilities/GitUtilities.cs +++ b/src/SourceLink.Git.IntegrationTests/Utilities/GitUtilities.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.IO; diff --git a/src/SourceLink.Git.IntegrationTests/Utilities/TestUtilities.cs b/src/SourceLink.Git.IntegrationTests/Utilities/TestUtilities.cs index 0ca5030c..208cd2f6 100644 --- a/src/SourceLink.Git.IntegrationTests/Utilities/TestUtilities.cs +++ b/src/SourceLink.Git.IntegrationTests/Utilities/TestUtilities.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System.IO; using System.IO.Compression; diff --git a/src/SourceLink.GitHub.UnitTests/GetSourceLinkUrlTests.cs b/src/SourceLink.GitHub.UnitTests/GetSourceLinkUrlTests.cs index 8ad36929..d91ad538 100644 --- a/src/SourceLink.GitHub.UnitTests/GetSourceLinkUrlTests.cs +++ b/src/SourceLink.GitHub.UnitTests/GetSourceLinkUrlTests.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using Microsoft.Build.Tasks.SourceControl; using TestUtilities; using Xunit; diff --git a/src/SourceLink.GitHub.UnitTests/Microsoft.SourceLink.GitHub.UnitTests.csproj b/src/SourceLink.GitHub.UnitTests/Microsoft.SourceLink.GitHub.UnitTests.csproj index e9f9bf20..af24de0a 100644 --- a/src/SourceLink.GitHub.UnitTests/Microsoft.SourceLink.GitHub.UnitTests.csproj +++ b/src/SourceLink.GitHub.UnitTests/Microsoft.SourceLink.GitHub.UnitTests.csproj @@ -1,4 +1,6 @@ - + + + net472;net6.0 diff --git a/src/SourceLink.GitHub.UnitTests/TranslateRepositoryUrlsTests.cs b/src/SourceLink.GitHub.UnitTests/TranslateRepositoryUrlsTests.cs index 3310cbd3..85aac344 100644 --- a/src/SourceLink.GitHub.UnitTests/TranslateRepositoryUrlsTests.cs +++ b/src/SourceLink.GitHub.UnitTests/TranslateRepositoryUrlsTests.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System.Linq; using TestUtilities; using Xunit; diff --git a/src/SourceLink.GitHub/GetSourceLinkUrl.cs b/src/SourceLink.GitHub/GetSourceLinkUrl.cs index db632e7a..f7a6c601 100644 --- a/src/SourceLink.GitHub/GetSourceLinkUrl.cs +++ b/src/SourceLink.GitHub/GetSourceLinkUrl.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using Microsoft.Build.Framework; diff --git a/src/SourceLink.GitHub/Microsoft.SourceLink.GitHub.csproj b/src/SourceLink.GitHub/Microsoft.SourceLink.GitHub.csproj index aef83a7f..626b9cc5 100644 --- a/src/SourceLink.GitHub/Microsoft.SourceLink.GitHub.csproj +++ b/src/SourceLink.GitHub/Microsoft.SourceLink.GitHub.csproj @@ -1,4 +1,6 @@ - + + + net472;net6.0 true diff --git a/src/SourceLink.GitHub/TranslateRepositoryUrls.cs b/src/SourceLink.GitHub/TranslateRepositoryUrls.cs index fa1b2c6b..61e91713 100644 --- a/src/SourceLink.GitHub/TranslateRepositoryUrls.cs +++ b/src/SourceLink.GitHub/TranslateRepositoryUrls.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using Microsoft.Build.Tasks.SourceControl; diff --git a/src/SourceLink.GitHub/build/Microsoft.SourceLink.GitHub.props b/src/SourceLink.GitHub/build/Microsoft.SourceLink.GitHub.props index 54fdc53c..434ac4c7 100644 --- a/src/SourceLink.GitHub/build/Microsoft.SourceLink.GitHub.props +++ b/src/SourceLink.GitHub/build/Microsoft.SourceLink.GitHub.props @@ -1,4 +1,5 @@ - + + diff --git a/src/SourceLink.GitHub/build/Microsoft.SourceLink.GitHub.targets b/src/SourceLink.GitHub/build/Microsoft.SourceLink.GitHub.targets index e016121b..67ce1b10 100644 --- a/src/SourceLink.GitHub/build/Microsoft.SourceLink.GitHub.targets +++ b/src/SourceLink.GitHub/build/Microsoft.SourceLink.GitHub.targets @@ -1,4 +1,5 @@ - + + <_SourceLinkGitHubAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net472\Microsoft.SourceLink.GitHub.dll diff --git a/src/SourceLink.GitHub/buildMultiTargeting/Microsoft.SourceLink.GitHub.props b/src/SourceLink.GitHub/buildMultiTargeting/Microsoft.SourceLink.GitHub.props index 9c26c672..afadd0e1 100644 --- a/src/SourceLink.GitHub/buildMultiTargeting/Microsoft.SourceLink.GitHub.props +++ b/src/SourceLink.GitHub/buildMultiTargeting/Microsoft.SourceLink.GitHub.props @@ -1,4 +1,5 @@ - + + diff --git a/src/SourceLink.GitHub/buildMultiTargeting/Microsoft.SourceLink.GitHub.targets b/src/SourceLink.GitHub/buildMultiTargeting/Microsoft.SourceLink.GitHub.targets index 84afbc37..c72d0e0c 100644 --- a/src/SourceLink.GitHub/buildMultiTargeting/Microsoft.SourceLink.GitHub.targets +++ b/src/SourceLink.GitHub/buildMultiTargeting/Microsoft.SourceLink.GitHub.targets @@ -1,4 +1,5 @@ - + + diff --git a/src/SourceLink.GitLab.UnitTests/GetSourceLinkUrlTests.cs b/src/SourceLink.GitLab.UnitTests/GetSourceLinkUrlTests.cs index dadaccda..e412a6b8 100644 --- a/src/SourceLink.GitLab.UnitTests/GetSourceLinkUrlTests.cs +++ b/src/SourceLink.GitLab.UnitTests/GetSourceLinkUrlTests.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using Microsoft.Build.Tasks.SourceControl; using TestUtilities; using Xunit; diff --git a/src/SourceLink.GitLab.UnitTests/Microsoft.SourceLink.GitLab.UnitTests.csproj b/src/SourceLink.GitLab.UnitTests/Microsoft.SourceLink.GitLab.UnitTests.csproj index 21a594f4..6e3c417f 100644 --- a/src/SourceLink.GitLab.UnitTests/Microsoft.SourceLink.GitLab.UnitTests.csproj +++ b/src/SourceLink.GitLab.UnitTests/Microsoft.SourceLink.GitLab.UnitTests.csproj @@ -1,4 +1,6 @@ - + + + net472;net6.0 diff --git a/src/SourceLink.GitLab.UnitTests/TranslateRepositoryUrlsTests.cs b/src/SourceLink.GitLab.UnitTests/TranslateRepositoryUrlsTests.cs index e536cf20..75caf89d 100644 --- a/src/SourceLink.GitLab.UnitTests/TranslateRepositoryUrlsTests.cs +++ b/src/SourceLink.GitLab.UnitTests/TranslateRepositoryUrlsTests.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System.Linq; using TestUtilities; using Xunit; diff --git a/src/SourceLink.GitLab/GetSourceLinkUrl.cs b/src/SourceLink.GitLab/GetSourceLinkUrl.cs index 2edd3917..5c2f6d5a 100644 --- a/src/SourceLink.GitLab/GetSourceLinkUrl.cs +++ b/src/SourceLink.GitLab/GetSourceLinkUrl.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using Microsoft.Build.Framework; diff --git a/src/SourceLink.GitLab/Microsoft.SourceLink.GitLab.csproj b/src/SourceLink.GitLab/Microsoft.SourceLink.GitLab.csproj index f74e5f49..4a1d0af7 100644 --- a/src/SourceLink.GitLab/Microsoft.SourceLink.GitLab.csproj +++ b/src/SourceLink.GitLab/Microsoft.SourceLink.GitLab.csproj @@ -1,4 +1,6 @@ - + + + net472;net6.0 true diff --git a/src/SourceLink.GitLab/TranslateRepositoryUrls.cs b/src/SourceLink.GitLab/TranslateRepositoryUrls.cs index d56d7f65..78dafccc 100644 --- a/src/SourceLink.GitLab/TranslateRepositoryUrls.cs +++ b/src/SourceLink.GitLab/TranslateRepositoryUrls.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using Microsoft.Build.Tasks.SourceControl; diff --git a/src/SourceLink.GitLab/build/Microsoft.SourceLink.GitLab.targets b/src/SourceLink.GitLab/build/Microsoft.SourceLink.GitLab.targets index 724c0075..20ee5654 100644 --- a/src/SourceLink.GitLab/build/Microsoft.SourceLink.GitLab.targets +++ b/src/SourceLink.GitLab/build/Microsoft.SourceLink.GitLab.targets @@ -1,4 +1,5 @@ - + + <_SourceLinkGitLabAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net472\Microsoft.SourceLink.GitLab.dll diff --git a/src/SourceLink.GitLab/buildMultiTargeting/Microsoft.SourceLink.GitLab.targets b/src/SourceLink.GitLab/buildMultiTargeting/Microsoft.SourceLink.GitLab.targets index 84afbc37..c72d0e0c 100644 --- a/src/SourceLink.GitLab/buildMultiTargeting/Microsoft.SourceLink.GitLab.targets +++ b/src/SourceLink.GitLab/buildMultiTargeting/Microsoft.SourceLink.GitLab.targets @@ -1,4 +1,5 @@ - + + diff --git a/src/SourceLink.GitWeb.UnitTests/GetSourceLinkUrlTests.cs b/src/SourceLink.GitWeb.UnitTests/GetSourceLinkUrlTests.cs index b215d1a0..67000094 100644 --- a/src/SourceLink.GitWeb.UnitTests/GetSourceLinkUrlTests.cs +++ b/src/SourceLink.GitWeb.UnitTests/GetSourceLinkUrlTests.cs @@ -1,5 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See -// License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using Microsoft.Build.Tasks.SourceControl; using TestUtilities; using Xunit; diff --git a/src/SourceLink.GitWeb.UnitTests/Microsoft.SourceLink.GitWeb.UnitTests.csproj b/src/SourceLink.GitWeb.UnitTests/Microsoft.SourceLink.GitWeb.UnitTests.csproj index 0a1d6a86..b4b92f6c 100644 --- a/src/SourceLink.GitWeb.UnitTests/Microsoft.SourceLink.GitWeb.UnitTests.csproj +++ b/src/SourceLink.GitWeb.UnitTests/Microsoft.SourceLink.GitWeb.UnitTests.csproj @@ -1,4 +1,6 @@ - + + + net472;net6.0 diff --git a/src/SourceLink.GitWeb.UnitTests/TranslateRepositoryUrlsTests.cs b/src/SourceLink.GitWeb.UnitTests/TranslateRepositoryUrlsTests.cs index 91ca1976..ac62ba4f 100644 --- a/src/SourceLink.GitWeb.UnitTests/TranslateRepositoryUrlsTests.cs +++ b/src/SourceLink.GitWeb.UnitTests/TranslateRepositoryUrlsTests.cs @@ -1,5 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See -// License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System.Linq; using TestUtilities; using Xunit; diff --git a/src/SourceLink.GitWeb/GetSourceLinkUrl.cs b/src/SourceLink.GitWeb/GetSourceLinkUrl.cs index da5df319..fea2d2f7 100644 --- a/src/SourceLink.GitWeb/GetSourceLinkUrl.cs +++ b/src/SourceLink.GitWeb/GetSourceLinkUrl.cs @@ -1,5 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See -// License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using Microsoft.Build.Framework; diff --git a/src/SourceLink.GitWeb/Microsoft.SourceLink.GitWeb.csproj b/src/SourceLink.GitWeb/Microsoft.SourceLink.GitWeb.csproj index e643e97d..19f92a8d 100644 --- a/src/SourceLink.GitWeb/Microsoft.SourceLink.GitWeb.csproj +++ b/src/SourceLink.GitWeb/Microsoft.SourceLink.GitWeb.csproj @@ -1,4 +1,6 @@ - + + + net472;net6.0 true diff --git a/src/SourceLink.GitWeb/TranslateRepositoryUrls.cs b/src/SourceLink.GitWeb/TranslateRepositoryUrls.cs index 591cd7d4..ccb83a0e 100644 --- a/src/SourceLink.GitWeb/TranslateRepositoryUrls.cs +++ b/src/SourceLink.GitWeb/TranslateRepositoryUrls.cs @@ -1,5 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See -// License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using Microsoft.Build.Tasks.SourceControl; using System; diff --git a/src/SourceLink.GitWeb/build/Microsoft.SourceLink.GitWeb.targets b/src/SourceLink.GitWeb/build/Microsoft.SourceLink.GitWeb.targets index e3f61434..9140d6d4 100644 --- a/src/SourceLink.GitWeb/build/Microsoft.SourceLink.GitWeb.targets +++ b/src/SourceLink.GitWeb/build/Microsoft.SourceLink.GitWeb.targets @@ -1,4 +1,5 @@ - + + <_SourceLinkGitWebAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net472\Microsoft.SourceLink.GitWeb.dll diff --git a/src/SourceLink.GitWeb/buildMultiTargeting/Microsoft.SourceLink.GitWeb.targets b/src/SourceLink.GitWeb/buildMultiTargeting/Microsoft.SourceLink.GitWeb.targets index 84afbc37..c72d0e0c 100644 --- a/src/SourceLink.GitWeb/buildMultiTargeting/Microsoft.SourceLink.GitWeb.targets +++ b/src/SourceLink.GitWeb/buildMultiTargeting/Microsoft.SourceLink.GitWeb.targets @@ -1,4 +1,5 @@ - + + diff --git a/src/SourceLink.Gitea.UnitTests/GetSourceLinkUrlTests.cs b/src/SourceLink.Gitea.UnitTests/GetSourceLinkUrlTests.cs index 06e75407..98cab915 100644 --- a/src/SourceLink.Gitea.UnitTests/GetSourceLinkUrlTests.cs +++ b/src/SourceLink.Gitea.UnitTests/GetSourceLinkUrlTests.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using Microsoft.Build.Tasks.SourceControl; using TestUtilities; using Xunit; diff --git a/src/SourceLink.Gitea.UnitTests/Microsoft.SourceLink.Gitea.UnitTests.csproj b/src/SourceLink.Gitea.UnitTests/Microsoft.SourceLink.Gitea.UnitTests.csproj index 513d2842..6da282a2 100644 --- a/src/SourceLink.Gitea.UnitTests/Microsoft.SourceLink.Gitea.UnitTests.csproj +++ b/src/SourceLink.Gitea.UnitTests/Microsoft.SourceLink.Gitea.UnitTests.csproj @@ -1,4 +1,6 @@ - + + + net472;net6.0 diff --git a/src/SourceLink.Gitea.UnitTests/TranslateRepositoryUrlsTests.cs b/src/SourceLink.Gitea.UnitTests/TranslateRepositoryUrlsTests.cs index 3949583d..6578d097 100644 --- a/src/SourceLink.Gitea.UnitTests/TranslateRepositoryUrlsTests.cs +++ b/src/SourceLink.Gitea.UnitTests/TranslateRepositoryUrlsTests.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System.Linq; using TestUtilities; using Xunit; diff --git a/src/SourceLink.Gitea/GetSourceLinkUrl.cs b/src/SourceLink.Gitea/GetSourceLinkUrl.cs index 5d5526e7..22cd3d82 100644 --- a/src/SourceLink.Gitea/GetSourceLinkUrl.cs +++ b/src/SourceLink.Gitea/GetSourceLinkUrl.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using Microsoft.Build.Framework; diff --git a/src/SourceLink.Gitea/Microsoft.SourceLink.Gitea.csproj b/src/SourceLink.Gitea/Microsoft.SourceLink.Gitea.csproj index c4d0b140..fa9e33e4 100644 --- a/src/SourceLink.Gitea/Microsoft.SourceLink.Gitea.csproj +++ b/src/SourceLink.Gitea/Microsoft.SourceLink.Gitea.csproj @@ -1,4 +1,6 @@ - + + + net472;net6.0 true diff --git a/src/SourceLink.Gitea/TranslateRepositoryUrls.cs b/src/SourceLink.Gitea/TranslateRepositoryUrls.cs index 1bc949a7..b3e5d9c3 100644 --- a/src/SourceLink.Gitea/TranslateRepositoryUrls.cs +++ b/src/SourceLink.Gitea/TranslateRepositoryUrls.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using Microsoft.Build.Tasks.SourceControl; diff --git a/src/SourceLink.Gitea/build/Microsoft.SourceLink.Gitea.targets b/src/SourceLink.Gitea/build/Microsoft.SourceLink.Gitea.targets index d86a5d33..a7882e5e 100644 --- a/src/SourceLink.Gitea/build/Microsoft.SourceLink.Gitea.targets +++ b/src/SourceLink.Gitea/build/Microsoft.SourceLink.Gitea.targets @@ -1,4 +1,5 @@ - + + <_SourceLinkGiteaAssemblyFile Condition="'$(MSBuildRuntimeType)' != 'Core'">$(MSBuildThisFileDirectory)..\tools\net472\Microsoft.SourceLink.Gitea.dll diff --git a/src/SourceLink.Gitea/buildMultiTargeting/Microsoft.SourceLink.Gitea.targets b/src/SourceLink.Gitea/buildMultiTargeting/Microsoft.SourceLink.Gitea.targets index 84afbc37..c72d0e0c 100644 --- a/src/SourceLink.Gitea/buildMultiTargeting/Microsoft.SourceLink.Gitea.targets +++ b/src/SourceLink.Gitea/buildMultiTargeting/Microsoft.SourceLink.Gitea.targets @@ -1,4 +1,5 @@ - + + diff --git a/src/SourceLink.Tools.UnitTests/Microsoft.SourceLink.Tools.UnitTests.csproj b/src/SourceLink.Tools.UnitTests/Microsoft.SourceLink.Tools.UnitTests.csproj index 5122b50e..c56375f7 100644 --- a/src/SourceLink.Tools.UnitTests/Microsoft.SourceLink.Tools.UnitTests.csproj +++ b/src/SourceLink.Tools.UnitTests/Microsoft.SourceLink.Tools.UnitTests.csproj @@ -1,4 +1,6 @@ - + + + net472;net6.0 diff --git a/src/SourceLink.Tools.UnitTests/SourceLinkMapTests.cs b/src/SourceLink.Tools.UnitTests/SourceLinkMapTests.cs index db134dfa..db6cb9f5 100644 --- a/src/SourceLink.Tools.UnitTests/SourceLinkMapTests.cs +++ b/src/SourceLink.Tools.UnitTests/SourceLinkMapTests.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Collections.Generic; diff --git a/src/SourceLink.Tools/Microsoft.SourceLink.Tools.Package.csproj b/src/SourceLink.Tools/Microsoft.SourceLink.Tools.Package.csproj index baea22f8..5d17262c 100644 --- a/src/SourceLink.Tools/Microsoft.SourceLink.Tools.Package.csproj +++ b/src/SourceLink.Tools/Microsoft.SourceLink.Tools.Package.csproj @@ -1,5 +1,5 @@  - + net6.0;netstandard2.0 @@ -21,4 +21,4 @@ - \ No newline at end of file + diff --git a/src/SourceLink.Tools/SourceLinkMap.cs b/src/SourceLink.Tools/SourceLinkMap.cs index 2aff6b81..1218c9c4 100644 --- a/src/SourceLink.Tools/SourceLinkMap.cs +++ b/src/SourceLink.Tools/SourceLinkMap.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. #nullable enable @@ -226,4 +228,4 @@ public bool TryGetUri( return false; } } -} \ No newline at end of file +} diff --git a/src/TestUtilities/AssertEx.cs b/src/TestUtilities/AssertEx.cs index 82cc2eb1..804ac4d7 100644 --- a/src/TestUtilities/AssertEx.cs +++ b/src/TestUtilities/AssertEx.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Collections; diff --git a/src/TestUtilities/ConditionalFactAttribute.cs b/src/TestUtilities/ConditionalFactAttribute.cs index 8a013c7b..720b84c7 100644 --- a/src/TestUtilities/ConditionalFactAttribute.cs +++ b/src/TestUtilities/ConditionalFactAttribute.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.IO; diff --git a/src/TestUtilities/DiffUtil.cs b/src/TestUtilities/DiffUtil.cs index 5bcbc94c..2cbf156f 100644 --- a/src/TestUtilities/DiffUtil.cs +++ b/src/TestUtilities/DiffUtil.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Collections.Generic; diff --git a/src/TestUtilities/DotNetSdk/BuildInfoAttribute.cs b/src/TestUtilities/DotNetSdk/BuildInfoAttribute.cs index fc9ba9fc..d4830392 100644 --- a/src/TestUtilities/DotNetSdk/BuildInfoAttribute.cs +++ b/src/TestUtilities/DotNetSdk/BuildInfoAttribute.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; namespace TestUtilities diff --git a/src/TestUtilities/DotNetSdk/DotNetSdkTestBase.cs b/src/TestUtilities/DotNetSdk/DotNetSdkTestBase.cs index a9936150..8608bf80 100644 --- a/src/TestUtilities/DotNetSdk/DotNetSdkTestBase.cs +++ b/src/TestUtilities/DotNetSdk/DotNetSdkTestBase.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Collections.Generic; using System.Diagnostics; diff --git a/src/TestUtilities/EnumerableExtensions.cs b/src/TestUtilities/EnumerableExtensions.cs index c86b6cdc..e4c90136 100644 --- a/src/TestUtilities/EnumerableExtensions.cs +++ b/src/TestUtilities/EnumerableExtensions.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Collections.Generic; diff --git a/src/TestUtilities/KeyValuePairUtils.cs b/src/TestUtilities/KeyValuePairUtils.cs index a452e9ff..2a3abc5e 100644 --- a/src/TestUtilities/KeyValuePairUtils.cs +++ b/src/TestUtilities/KeyValuePairUtils.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System.Collections.Generic; diff --git a/src/TestUtilities/MockEngine.cs b/src/TestUtilities/MockEngine.cs index d29fe2be..3fcb3ad1 100644 --- a/src/TestUtilities/MockEngine.cs +++ b/src/TestUtilities/MockEngine.cs @@ -1,5 +1,6 @@ -// Copyright (c) Microsoft. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Collections; diff --git a/src/TestUtilities/MockItem.cs b/src/TestUtilities/MockItem.cs index 0b33105c..e2b0e8ef 100644 --- a/src/TestUtilities/MockItem.cs +++ b/src/TestUtilities/MockItem.cs @@ -1,5 +1,6 @@ -// Copyright (c) Microsoft. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Collections; diff --git a/src/TestUtilities/ProcessResult.cs b/src/TestUtilities/ProcessResult.cs index ef41d145..072e56a0 100644 --- a/src/TestUtilities/ProcessResult.cs +++ b/src/TestUtilities/ProcessResult.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; @@ -39,4 +41,4 @@ public bool ContainsErrors get { return this.ExitCode != 0 || !string.IsNullOrEmpty(this.Errors); } } } -} \ No newline at end of file +} diff --git a/src/TestUtilities/ProcessUtilities.cs b/src/TestUtilities/ProcessUtilities.cs index 445b54cb..dc35fdea 100644 --- a/src/TestUtilities/ProcessUtilities.cs +++ b/src/TestUtilities/ProcessUtilities.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Collections.Generic; diff --git a/src/TestUtilities/TempFiles/DisposableDirectory.cs b/src/TestUtilities/TempFiles/DisposableDirectory.cs index abf9af44..ef5536de 100644 --- a/src/TestUtilities/TempFiles/DisposableDirectory.cs +++ b/src/TestUtilities/TempFiles/DisposableDirectory.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.IO; diff --git a/src/TestUtilities/TempFiles/DisposableFile.cs b/src/TestUtilities/TempFiles/DisposableFile.cs index cd7669d9..35784a5a 100644 --- a/src/TestUtilities/TempFiles/DisposableFile.cs +++ b/src/TestUtilities/TempFiles/DisposableFile.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.IO; diff --git a/src/TestUtilities/TempFiles/TempDirectory.cs b/src/TestUtilities/TempFiles/TempDirectory.cs index e6860d82..f3be0afd 100644 --- a/src/TestUtilities/TempFiles/TempDirectory.cs +++ b/src/TestUtilities/TempFiles/TempDirectory.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Diagnostics; diff --git a/src/TestUtilities/TempFiles/TempFile.cs b/src/TestUtilities/TempFiles/TempFile.cs index 0bd9dc32..3b921b6c 100644 --- a/src/TestUtilities/TempFiles/TempFile.cs +++ b/src/TestUtilities/TempFiles/TempFile.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Collections.Immutable; diff --git a/src/TestUtilities/TempFiles/TempRoot.cs b/src/TestUtilities/TempFiles/TempRoot.cs index 5e5083ea..69202a53 100644 --- a/src/TestUtilities/TempFiles/TempRoot.cs +++ b/src/TestUtilities/TempFiles/TempRoot.cs @@ -1,4 +1,6 @@ -// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the License.txt file in the project root for more information. using System; using System.Collections.Concurrent; diff --git a/src/TestUtilities/TestUtilities.csproj b/src/TestUtilities/TestUtilities.csproj index b5572c7c..95630830 100644 --- a/src/TestUtilities/TestUtilities.csproj +++ b/src/TestUtilities/TestUtilities.csproj @@ -1,4 +1,6 @@ - + + + net472;net6.0 false From 0f5a44c5ffcc375c5c4304385dbed5e3af3484c3 Mon Sep 17 00:00:00 2001 From: Jon Fortescue Date: Wed, 11 Aug 2021 15:26:29 -0700 Subject: [PATCH 138/161] Turn on localization PRs from OneLocBuild (#731) --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8eea90b0..9030c7d4 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -29,10 +29,10 @@ stages: - stage: build displayName: Build jobs: - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/main')) }}: - template: /eng/common/templates/job/onelocbuild.yml parameters: - CreatePr: false + MirrorRepo: sourcelink LclSource: lclFilesfromPackage LclPackageId: 'LCL-JUNO-PROD-SOURCELINK' - template: /eng/common/templates/jobs/jobs.yml From 67b28841adcd09bd7076248c6bcdcb691c8ce9f0 Mon Sep 17 00:00:00 2001 From: dotnet bot Date: Thu, 12 Aug 2021 09:43:54 -0700 Subject: [PATCH 139/161] Localized file check-in by OneLocBuild Task (#732) --- src/Common/xlf/CommonResources.zh-Hans.xlf | 2 +- src/Common/xlf/CommonResources.zh-Hant.xlf | 2 +- src/Microsoft.Build.Tasks.Git/xlf/Resources.zh-Hans.xlf | 2 +- src/Microsoft.Build.Tasks.Git/xlf/Resources.zh-Hant.xlf | 2 +- src/SourceLink.AzureDevOpsServer.Git/xlf/Resources.zh-Hans.xlf | 2 +- src/SourceLink.AzureDevOpsServer.Git/xlf/Resources.zh-Hant.xlf | 2 +- src/SourceLink.AzureRepos.Git/xlf/Resources.zh-Hans.xlf | 2 +- src/SourceLink.AzureRepos.Git/xlf/Resources.zh-Hant.xlf | 2 +- src/SourceLink.Bitbucket.Git/xlf/Resources.zh-Hans.xlf | 2 +- src/SourceLink.Bitbucket.Git/xlf/Resources.zh-Hant.xlf | 2 +- src/SourceLink.Common/xlf/Resources.zh-Hans.xlf | 2 +- src/SourceLink.Common/xlf/Resources.zh-Hant.xlf | 2 +- src/SourceLink.GitHub/xlf/Resources.zh-Hans.xlf | 2 +- src/SourceLink.GitHub/xlf/Resources.zh-Hant.xlf | 2 +- src/SourceLink.GitLab/xlf/Resources.zh-Hans.xlf | 2 +- src/SourceLink.GitLab/xlf/Resources.zh-Hant.xlf | 2 +- src/SourceLink.GitWeb/xlf/Resources.zh-Hans.xlf | 2 +- src/SourceLink.GitWeb/xlf/Resources.zh-Hant.xlf | 2 +- src/SourceLink.Gitea/xlf/Resources.zh-Hans.xlf | 2 +- src/SourceLink.Gitea/xlf/Resources.zh-Hant.xlf | 2 +- 20 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/Common/xlf/CommonResources.zh-Hans.xlf b/src/Common/xlf/CommonResources.zh-Hans.xlf index 74dc1582..ad49a889 100644 --- a/src/Common/xlf/CommonResources.zh-Hans.xlf +++ b/src/Common/xlf/CommonResources.zh-Hans.xlf @@ -1,6 +1,6 @@  - + {0} item group is empty. At least one {1} repository host is required in order to generate SourceLink. diff --git a/src/Common/xlf/CommonResources.zh-Hant.xlf b/src/Common/xlf/CommonResources.zh-Hant.xlf index 623c7110..93892606 100644 --- a/src/Common/xlf/CommonResources.zh-Hant.xlf +++ b/src/Common/xlf/CommonResources.zh-Hant.xlf @@ -1,6 +1,6 @@  - + {0} item group is empty. At least one {1} repository host is required in order to generate SourceLink. diff --git a/src/Microsoft.Build.Tasks.Git/xlf/Resources.zh-Hans.xlf b/src/Microsoft.Build.Tasks.Git/xlf/Resources.zh-Hans.xlf index 536f892f..e358863e 100644 --- a/src/Microsoft.Build.Tasks.Git/xlf/Resources.zh-Hans.xlf +++ b/src/Microsoft.Build.Tasks.Git/xlf/Resources.zh-Hans.xlf @@ -1,6 +1,6 @@  - + Configuration file recursion exceeded maximum allowed depth of {0}. diff --git a/src/Microsoft.Build.Tasks.Git/xlf/Resources.zh-Hant.xlf b/src/Microsoft.Build.Tasks.Git/xlf/Resources.zh-Hant.xlf index 466629fe..30966089 100644 --- a/src/Microsoft.Build.Tasks.Git/xlf/Resources.zh-Hant.xlf +++ b/src/Microsoft.Build.Tasks.Git/xlf/Resources.zh-Hant.xlf @@ -1,6 +1,6 @@  - + Configuration file recursion exceeded maximum allowed depth of {0}. diff --git a/src/SourceLink.AzureDevOpsServer.Git/xlf/Resources.zh-Hans.xlf b/src/SourceLink.AzureDevOpsServer.Git/xlf/Resources.zh-Hans.xlf index 05d2ad3b..bc963ae6 100644 --- a/src/SourceLink.AzureDevOpsServer.Git/xlf/Resources.zh-Hans.xlf +++ b/src/SourceLink.AzureDevOpsServer.Git/xlf/Resources.zh-Hans.xlf @@ -1,6 +1,6 @@  - + The value of environment variable {0} is not a well-formed list of URL pairs: '{1}'" diff --git a/src/SourceLink.AzureDevOpsServer.Git/xlf/Resources.zh-Hant.xlf b/src/SourceLink.AzureDevOpsServer.Git/xlf/Resources.zh-Hant.xlf index b5bb875c..177fe576 100644 --- a/src/SourceLink.AzureDevOpsServer.Git/xlf/Resources.zh-Hant.xlf +++ b/src/SourceLink.AzureDevOpsServer.Git/xlf/Resources.zh-Hant.xlf @@ -1,6 +1,6 @@  - + The value of environment variable {0} is not a well-formed list of URL pairs: '{1}'" diff --git a/src/SourceLink.AzureRepos.Git/xlf/Resources.zh-Hans.xlf b/src/SourceLink.AzureRepos.Git/xlf/Resources.zh-Hans.xlf index 05d2ad3b..bc963ae6 100644 --- a/src/SourceLink.AzureRepos.Git/xlf/Resources.zh-Hans.xlf +++ b/src/SourceLink.AzureRepos.Git/xlf/Resources.zh-Hans.xlf @@ -1,6 +1,6 @@  - + The value of environment variable {0} is not a well-formed list of URL pairs: '{1}'" diff --git a/src/SourceLink.AzureRepos.Git/xlf/Resources.zh-Hant.xlf b/src/SourceLink.AzureRepos.Git/xlf/Resources.zh-Hant.xlf index b5bb875c..177fe576 100644 --- a/src/SourceLink.AzureRepos.Git/xlf/Resources.zh-Hant.xlf +++ b/src/SourceLink.AzureRepos.Git/xlf/Resources.zh-Hant.xlf @@ -1,6 +1,6 @@  - + The value of environment variable {0} is not a well-formed list of URL pairs: '{1}'" diff --git a/src/SourceLink.Bitbucket.Git/xlf/Resources.zh-Hans.xlf b/src/SourceLink.Bitbucket.Git/xlf/Resources.zh-Hans.xlf index b649950f..d831e742 100644 --- a/src/SourceLink.Bitbucket.Git/xlf/Resources.zh-Hans.xlf +++ b/src/SourceLink.Bitbucket.Git/xlf/Resources.zh-Hans.xlf @@ -1,6 +1,6 @@  - + \ No newline at end of file diff --git a/src/SourceLink.Bitbucket.Git/xlf/Resources.zh-Hant.xlf b/src/SourceLink.Bitbucket.Git/xlf/Resources.zh-Hant.xlf index 1856dda6..a0c04874 100644 --- a/src/SourceLink.Bitbucket.Git/xlf/Resources.zh-Hant.xlf +++ b/src/SourceLink.Bitbucket.Git/xlf/Resources.zh-Hant.xlf @@ -1,6 +1,6 @@  - + \ No newline at end of file diff --git a/src/SourceLink.Common/xlf/Resources.zh-Hans.xlf b/src/SourceLink.Common/xlf/Resources.zh-Hans.xlf index d3e4be08..f64d8b75 100644 --- a/src/SourceLink.Common/xlf/Resources.zh-Hans.xlf +++ b/src/SourceLink.Common/xlf/Resources.zh-Hans.xlf @@ -1,6 +1,6 @@  - + {0} must end with a directory separator: '{1}' diff --git a/src/SourceLink.Common/xlf/Resources.zh-Hant.xlf b/src/SourceLink.Common/xlf/Resources.zh-Hant.xlf index d2c13727..97978234 100644 --- a/src/SourceLink.Common/xlf/Resources.zh-Hant.xlf +++ b/src/SourceLink.Common/xlf/Resources.zh-Hant.xlf @@ -1,6 +1,6 @@  - + {0} must end with a directory separator: '{1}' diff --git a/src/SourceLink.GitHub/xlf/Resources.zh-Hans.xlf b/src/SourceLink.GitHub/xlf/Resources.zh-Hans.xlf index b649950f..d831e742 100644 --- a/src/SourceLink.GitHub/xlf/Resources.zh-Hans.xlf +++ b/src/SourceLink.GitHub/xlf/Resources.zh-Hans.xlf @@ -1,6 +1,6 @@  - + \ No newline at end of file diff --git a/src/SourceLink.GitHub/xlf/Resources.zh-Hant.xlf b/src/SourceLink.GitHub/xlf/Resources.zh-Hant.xlf index 1856dda6..a0c04874 100644 --- a/src/SourceLink.GitHub/xlf/Resources.zh-Hant.xlf +++ b/src/SourceLink.GitHub/xlf/Resources.zh-Hant.xlf @@ -1,6 +1,6 @@  - + \ No newline at end of file diff --git a/src/SourceLink.GitLab/xlf/Resources.zh-Hans.xlf b/src/SourceLink.GitLab/xlf/Resources.zh-Hans.xlf index e4a4010d..da2cf8fc 100644 --- a/src/SourceLink.GitLab/xlf/Resources.zh-Hans.xlf +++ b/src/SourceLink.GitLab/xlf/Resources.zh-Hans.xlf @@ -1,6 +1,6 @@  - + {0} item group is empty. At least one {1} repository host is required in order to generate SourceLink. diff --git a/src/SourceLink.GitLab/xlf/Resources.zh-Hant.xlf b/src/SourceLink.GitLab/xlf/Resources.zh-Hant.xlf index ab6c29b8..b77078ea 100644 --- a/src/SourceLink.GitLab/xlf/Resources.zh-Hant.xlf +++ b/src/SourceLink.GitLab/xlf/Resources.zh-Hant.xlf @@ -1,6 +1,6 @@  - + {0} item group is empty. At least one {1} repository host is required in order to generate SourceLink. diff --git a/src/SourceLink.GitWeb/xlf/Resources.zh-Hans.xlf b/src/SourceLink.GitWeb/xlf/Resources.zh-Hans.xlf index 2d6553a8..8472a671 100644 --- a/src/SourceLink.GitWeb/xlf/Resources.zh-Hans.xlf +++ b/src/SourceLink.GitWeb/xlf/Resources.zh-Hans.xlf @@ -1,6 +1,6 @@  - + The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. diff --git a/src/SourceLink.GitWeb/xlf/Resources.zh-Hant.xlf b/src/SourceLink.GitWeb/xlf/Resources.zh-Hant.xlf index efa8ded5..0ae63676 100644 --- a/src/SourceLink.GitWeb/xlf/Resources.zh-Hant.xlf +++ b/src/SourceLink.GitWeb/xlf/Resources.zh-Hant.xlf @@ -1,6 +1,6 @@  - + The Microsoft.SourceLink.GitWeb provider does not support translating the '{0}' protocol to content URLs. diff --git a/src/SourceLink.Gitea/xlf/Resources.zh-Hans.xlf b/src/SourceLink.Gitea/xlf/Resources.zh-Hans.xlf index e4a4010d..da2cf8fc 100644 --- a/src/SourceLink.Gitea/xlf/Resources.zh-Hans.xlf +++ b/src/SourceLink.Gitea/xlf/Resources.zh-Hans.xlf @@ -1,6 +1,6 @@  - + {0} item group is empty. At least one {1} repository host is required in order to generate SourceLink. diff --git a/src/SourceLink.Gitea/xlf/Resources.zh-Hant.xlf b/src/SourceLink.Gitea/xlf/Resources.zh-Hant.xlf index ab6c29b8..b77078ea 100644 --- a/src/SourceLink.Gitea/xlf/Resources.zh-Hant.xlf +++ b/src/SourceLink.Gitea/xlf/Resources.zh-Hant.xlf @@ -1,6 +1,6 @@  - + {0} item group is empty. At least one {1} repository host is required in order to generate SourceLink. From 9006950900cd6a0783ae864c4ca27b8e6b753723 Mon Sep 17 00:00:00 2001 From: dotnet bot Date: Thu, 12 Aug 2021 09:44:14 -0700 Subject: [PATCH 140/161] Localized file check-in by OneLocBuild Task (#733) From 5e2c4f1d59413df38059baed8ddf73350fb5139a Mon Sep 17 00:00:00 2001 From: Chris Rummel Date: Thu, 12 Aug 2021 13:29:16 -0500 Subject: [PATCH 141/161] Allow using .git directory instead of gitdir redirect in submodules. (#653) * Allow using .git directory instead of gitdir redirect in submodules. * Address review comments and add test for old-style submodule behavior. * Use .git dir as the common directory as well. * Add more details to the comments about how this case can happen. * Address code review comments. * Allow empty dir Co-authored-by: tmat --- .../GitRepositoryTests.cs | 23 +++++++++++++++++-- .../GitDataReader/GitRepository.cs | 8 ++++++- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.Build.Tasks.Git.UnitTests/GitRepositoryTests.cs b/src/Microsoft.Build.Tasks.Git.UnitTests/GitRepositoryTests.cs index c0e502af..f1fe9428 100644 --- a/src/Microsoft.Build.Tasks.Git.UnitTests/GitRepositoryTests.cs +++ b/src/Microsoft.Build.Tasks.Git.UnitTests/GitRepositoryTests.cs @@ -297,6 +297,7 @@ public void Submodules_Errors() { "S10: 'sub10' 'http://github.com'", "S11: 'sub11' 'http://github.com'", + "S6: 'sub6' 'http://github.com'", "S9: 'sub9' 'http://github.com'" }, submodules.Select(s => $"{s.Name}: '{s.WorkingDirectoryRelativePath}' '{s.Url}'")); @@ -313,8 +314,6 @@ public void Submodules_Errors() TestUtilities.GetExceptionMessage(() => File.ReadAllText(Path.Combine(workingDir.Path, "sub4", ".git"))), // Could not find a part of the path 'sub5\.git'. TestUtilities.GetExceptionMessage(() => File.ReadAllText(Path.Combine(workingDir.Path, "sub5", ".git"))), - // Access to the path 'sub6\.git' is denied - TestUtilities.GetExceptionMessage(() => File.ReadAllText(Path.Combine(workingDir.Path, "sub6", ".git"))), // The format of the file 'sub7\.git' is invalid. string.Format(Resources.FormatOfFileIsInvalid, Path.Combine(workingDir.Path, "sub7", ".git")), // Path specified in file 'sub8\.git' is invalid. @@ -358,5 +357,25 @@ public void GetSubmoduleHeadCommitSha() var repository = new GitRepository(GitEnvironment.Empty, GitConfig.Empty, gitDir.Path, gitDir.Path, workingDir.Path); Assert.Equal("0000000000000000000000000000000000000000", repository.ReadSubmoduleHeadCommitSha(submoduleWorkingDir.Path)); } + + [Fact] + public void GetOldStyleSubmoduleHeadCommitSha() + { + using var temp = new TempRoot(); + + var gitDir = temp.CreateDirectory(); + var workingDir = temp.CreateDirectory(); + + // this is a unusual but legal case which can occur with older versions of Git or other tools. + // see https://git-scm.com/docs/gitsubmodules#_forms for more details. + var oldStyleSubmoduleWorkingDir = workingDir.CreateDirectory("old-style-submodule"); + var oldStyleSubmoduleGitDir = oldStyleSubmoduleWorkingDir.CreateDirectory(".git"); + var oldStyleSubmoduleRefsHeadDir = oldStyleSubmoduleGitDir.CreateDirectory("refs").CreateDirectory("heads"); + oldStyleSubmoduleRefsHeadDir.CreateFile("branch1").WriteAllText("1111111111111111111111111111111111111111"); + oldStyleSubmoduleGitDir.CreateFile("HEAD").WriteAllText("ref: refs/heads/branch1"); + + var repository = new GitRepository(GitEnvironment.Empty, GitConfig.Empty, gitDir.Path, gitDir.Path, workingDir.Path); + Assert.Equal("1111111111111111111111111111111111111111", repository.ReadSubmoduleHeadCommitSha(oldStyleSubmoduleWorkingDir.Path)); + } } } diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs index 30ee72bc..7b77bbc7 100644 --- a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs +++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs @@ -211,7 +211,13 @@ public static GitRepository OpenRepository(GitRepositoryLocation location, GitEn /// Null if the HEAD tip reference can't be resolved. internal string? ReadSubmoduleHeadCommitSha(string submoduleWorkingDirectoryFullPath) { - var gitDirectory = ReadDotGitFile(Path.Combine(submoduleWorkingDirectoryFullPath, GitDirName)); + // Submodules don't usually have their own .git directories but this is still legal. + // This can occur with older versions of Git or other tools, or when a user clones one + // repo into another's source tree (but it was not yet registered as a submodule). + // See https://git-scm.com/docs/gitsubmodules#_forms for more details. + var dotGitPath = Path.Combine(submoduleWorkingDirectoryFullPath, GitDirName); + + var gitDirectory = Directory.Exists(dotGitPath) ? dotGitPath : ReadDotGitFile(dotGitPath); if (!IsGitDirectory(gitDirectory, out var commonDirectory)) { return null; From 48e002e14a97800c0d6acf24a55357699c9d72a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Matou=C5=A1ek?= Date: Thu, 12 Aug 2021 15:28:59 -0700 Subject: [PATCH 142/161] Fix discovery of working directory for worktrees (#734) SourceLink used gitdir to determine the directory (and incorrectly interpreting the content) but turns out git doesn't use it at all for this purpose. --- .../GitRepositoryTests.cs | 107 +++++++++++++++++- .../GitDataReader/GitRepository.cs | 44 +------ .../GitOperations.cs | 2 +- 3 files changed, 110 insertions(+), 43 deletions(-) diff --git a/src/Microsoft.Build.Tasks.Git.UnitTests/GitRepositoryTests.cs b/src/Microsoft.Build.Tasks.Git.UnitTests/GitRepositoryTests.cs index f1fe9428..b27949cf 100644 --- a/src/Microsoft.Build.Tasks.Git.UnitTests/GitRepositoryTests.cs +++ b/src/Microsoft.Build.Tasks.Git.UnitTests/GitRepositoryTests.cs @@ -95,19 +95,33 @@ public void TryFindRepository_Worktree_Realistic() Assert.Equal(mainGitDir.Path, location.CommonDirectory); Assert.Null(location.WorkingDirectory); + var repository = GitRepository.OpenRepository(location, GitEnvironment.Empty); + Assert.Equal(location.GitDirectory, repository.GitDirectory); + Assert.Equal(location.CommonDirectory, repository.CommonDirectory); + Assert.Null(repository.WorkingDirectory); + // start under worktree directory: Assert.True(GitRepository.TryFindRepository(worktreeSubDir.Path, out location)); - Assert.Equal(worktreeGitDir.Path, location.GitDirectory); Assert.Equal(mainGitDir.Path, location.CommonDirectory); Assert.Equal(worktreeDir.Path, location.WorkingDirectory); + repository = GitRepository.OpenRepository(location, GitEnvironment.Empty); + Assert.Equal(location.GitDirectory, repository.GitDirectory); + Assert.Equal(location.WorkingDirectory, repository.WorkingDirectory); + Assert.Equal(location.CommonDirectory, repository.CommonDirectory); + // start under worktree git directory (git config works from this dir, but git status requires work dir): Assert.True(GitRepository.TryFindRepository(worktreeGitSubDir.Path, out location)); Assert.Equal(worktreeGitDir.Path, location.GitDirectory); Assert.Equal(mainGitDir.Path, location.CommonDirectory); Assert.Null(location.WorkingDirectory); + + repository = GitRepository.OpenRepository(location, GitEnvironment.Empty); + Assert.Equal(location.GitDirectory, repository.GitDirectory); + Assert.Equal(location.CommonDirectory, repository.CommonDirectory); + Assert.Null(repository.WorkingDirectory); } [Fact] @@ -170,6 +184,31 @@ public void OpenRepository() Assert.Equal("0000000000000000000000000000000000000000", repository.GetHeadCommitSha()); } + [Fact] + public void OpenRepository_WorkingDirectorySpecifiedInConfig() + { + using var temp = new TempRoot(); + + var homeDir = temp.CreateDirectory(); + + var workingDir = temp.CreateDirectory(); + var workingDir2 = temp.CreateDirectory(); + var gitDir = workingDir.CreateDirectory(".git"); + + gitDir.CreateFile("HEAD"); + gitDir.CreateFile("config").WriteAllText("[core]worktree = " + workingDir2.Path.Replace(@"\", @"\\")); + + Assert.True(GitRepository.TryFindRepository(gitDir.Path, out var location)); + Assert.Equal(gitDir.Path, location.CommonDirectory); + Assert.Equal(gitDir.Path, location.GitDirectory); + Assert.Null(location.WorkingDirectory); + + var repository = GitRepository.OpenRepository(location, GitEnvironment.Empty); + Assert.Equal(gitDir.Path, repository.CommonDirectory); + Assert.Equal(gitDir.Path, repository.GitDirectory); + Assert.Equal(workingDir2.Path, repository.WorkingDirectory); + } + [Fact] public void OpenRepository_VersionNotSupported() { @@ -191,6 +230,72 @@ public void OpenRepository_VersionNotSupported() Assert.Throws(() => GitRepository.OpenRepository(src.Path, new GitEnvironment(homeDir.Path))); } + [Fact] + public void OpenRepository_Worktree_GitdirFileMissing() + { + using var temp = new TempRoot(); + + var mainWorkingDir = temp.CreateDirectory(); + var mainGitDir = mainWorkingDir.CreateDirectory(".git"); + mainGitDir.CreateFile("HEAD"); + + var worktreesDir = mainGitDir.CreateDirectory("worktrees"); + var worktreeGitDir = worktreesDir.CreateDirectory("myworktree"); + var worktreeDir = temp.CreateDirectory(); + var worktreeGitFile = worktreeDir.CreateFile(".git").WriteAllText("gitdir: " + worktreeGitDir + " \r\n\t\v"); + + worktreeGitDir.CreateFile("HEAD"); + worktreeGitDir.CreateFile("commondir").WriteAllText("../..\n"); + // gitdir file that links back to the worktree working directory is missing from worktreeGitDir + + Assert.True(GitRepository.TryFindRepository(worktreeDir.Path, out var location)); + Assert.Equal(worktreeGitDir.Path, location.GitDirectory); + Assert.Equal(mainGitDir.Path, location.CommonDirectory); + Assert.Equal(worktreeDir.Path, location.WorkingDirectory); + + var repository = GitRepository.OpenRepository(location, GitEnvironment.Empty); + Assert.Equal(repository.GitDirectory, location.GitDirectory); + Assert.Equal(repository.CommonDirectory, location.CommonDirectory); + Assert.Equal(repository.WorkingDirectory, location.WorkingDirectory); + } + + /// + /// The directory in gitdir file is ignored for the purposes of determining repository working directory. + /// + [Fact] + public void OpenRepository_Worktree_GitdirFileDifferentPath() + { + using var temp = new TempRoot(); + + var mainWorkingDir = temp.CreateDirectory(); + var mainGitDir = mainWorkingDir.CreateDirectory(".git"); + mainGitDir.CreateFile("HEAD"); + + var worktreesDir = mainGitDir.CreateDirectory("worktrees"); + var worktreeGitDir = worktreesDir.CreateDirectory("myworktree"); + var worktreeDir = temp.CreateDirectory(); + var worktreeGitFile = worktreeDir.CreateFile(".git").WriteAllText("gitdir: " + worktreeGitDir + " \r\n\t\v"); + + var worktreeDir2 = temp.CreateDirectory(); + var worktreeGitFile2 = worktreeDir2.CreateFile(".git").WriteAllText("gitdir: " + worktreeGitDir + " \r\n\t\v"); + + worktreeGitDir.CreateFile("HEAD"); + worktreeGitDir.CreateFile("commondir").WriteAllText("../..\n"); + worktreeGitDir.CreateFile("gitdir").WriteAllText(worktreeGitFile2.Path + " \r\n\t\v"); + + Assert.True(GitRepository.TryFindRepository(worktreeDir.Path, out var location)); + Assert.Equal(worktreeGitDir.Path, location.GitDirectory); + Assert.Equal(mainGitDir.Path, location.CommonDirectory); + Assert.Equal(worktreeDir.Path, location.WorkingDirectory); + + var repository = GitRepository.OpenRepository(location, GitEnvironment.Empty); + Assert.Equal(repository.GitDirectory, location.GitDirectory); + Assert.Equal(repository.CommonDirectory, location.CommonDirectory); + + // actual working dir is not affected: + Assert.Equal(worktreeDir.Path, location.WorkingDirectory); + } + [Fact] public void Submodules() { diff --git a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs index 7b77bbc7..612e9f65 100644 --- a/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs +++ b/src/Microsoft.Build.Tasks.Git/GitDataReader/GitRepository.cs @@ -5,7 +5,6 @@ using System; using System.Collections.Generic; using System.Collections.Immutable; -using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; @@ -19,7 +18,6 @@ internal sealed class GitRepository private const string CommonDirFileName = "commondir"; private const string GitDirName = ".git"; private const string GitDirPrefix = "gitdir: "; - private const string GitDirFileName = "gitdir"; // See https://git-scm.com/docs/gitrepository-layout#Documentation/gitrepository-layout.txt-HEAD internal const string GitHeadFileName = "HEAD"; @@ -131,46 +129,10 @@ public static GitRepository OpenRepository(GitRepositoryLocation location, GitEn private static string? GetWorkingDirectory(GitConfig config, GitRepositoryLocation location) { - // Working trees cannot have the same common directory and git directory. - // 'gitdir' file in a git directory indicates a working tree. - - var gitdirFilePath = Path.Combine(location.GitDirectory, GitDirFileName); - - var isLinkedWorkingTree = PathUtils.ToPosixDirectoryPath(location.CommonDirectory) != PathUtils.ToPosixDirectoryPath(location.GitDirectory) && - File.Exists(gitdirFilePath); - - if (isLinkedWorkingTree) - { - // https://git-scm.com/docs/gitrepository-layout#Documentation/gitrepository-layout.txt-worktreesltidgtgitdir - - string workingDirectory; - try - { - workingDirectory = File.ReadAllText(gitdirFilePath); - } - catch (Exception e) when (!(e is IOException)) - { - throw new IOException(e.Message, e); - } - - workingDirectory = workingDirectory.TrimEnd(CharUtils.AsciiWhitespace); - - // Path in gitdir file must be absolute. - if (!PathUtils.IsAbsolute(workingDirectory)) - { - throw new InvalidDataException(string.Format(Resources.PathSpecifiedInFileIsNotAbsolute, gitdirFilePath, workingDirectory)); - } - - try - { - return Path.GetFullPath(workingDirectory); - } - catch - { - throw new InvalidDataException(string.Format(Resources.PathSpecifiedInFileIsInvalid, gitdirFilePath, workingDirectory)); - } - } + // TODO (https://github.com/dotnet/sourcelink/issues/301): + // GIT_WORK_TREE environment variable can also override working directory. + // Working directory can be overridden by a config option. // See https://git-scm.com/docs/git-config#Documentation/git-config.txt-coreworktree string? value = config.GetVariableValue("core", "worktree"); if (value != null) diff --git a/src/Microsoft.Build.Tasks.Git/GitOperations.cs b/src/Microsoft.Build.Tasks.Git/GitOperations.cs index ed9d36a7..4cf550f4 100644 --- a/src/Microsoft.Build.Tasks.Git/GitOperations.cs +++ b/src/Microsoft.Build.Tasks.Git/GitOperations.cs @@ -357,7 +357,7 @@ internal sealed class DirectoryNode public readonly string Name; public readonly List OrderedChildren; - // set on nodes that represent submodule working directory: + // set on nodes that represent working directory of the repository or a submodule: public Lazy? Matcher; public DirectoryNode(string name, List orderedChildren) From 1ca09f079b12727cd4b1e49278a5e7e893d0ea1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Matou=C5=A1ek?= Date: Fri, 13 Aug 2021 09:34:09 -0700 Subject: [PATCH 143/161] Update README.md (#735) --- docs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/README.md b/docs/README.md index 084786f9..943a53cc 100644 --- a/docs/README.md +++ b/docs/README.md @@ -140,7 +140,7 @@ The default value is `true`. Set to `false` to suppress publishing `SourceRevisi ### SourceRevisionId -Set by target `SetSourceRevisionId` and consumed by NuGet `Pack` target and `GenerateAssemblyInfo` target. +Set by target `InitializeSourceControlInformationFromSourceControlManager` and consumed by NuGet `Pack` target and `GenerateAssemblyInfo` target. May be used by custom targets that need this information. ### EnableSourceLink From 9dab77882ae68f36e14bf869fa631fa80a5d91cd Mon Sep 17 00:00:00 2001 From: Rui Lopes Date: Mon, 16 Aug 2021 19:53:41 +0100 Subject: [PATCH 144/161] Add support for the new GitLab raw url (#713) The url changed from, e.g.: https://gitlab.example.com/raw/example/example-dotnet-source-link/538b7a2b0d086146a6534bfe6f98a379191ea529/ExampleApplication/Program.cs To: https://gitlab.example.com/example/example-dotnet-source-link/-/raw/538b7a2b0d086146a6534bfe6f98a379191ea529/ExampleApplication/Program.cs See https://gitlab.com/gitlab-org/gitlab/-/issues/28848 See #670 --- .../GetSourceLinkUrlTests.cs | 42 ++++++++++++++++--- src/SourceLink.GitLab/GetSourceLinkUrl.cs | 28 ++++++++++++- 2 files changed, 63 insertions(+), 7 deletions(-) diff --git a/src/SourceLink.GitLab.UnitTests/GetSourceLinkUrlTests.cs b/src/SourceLink.GitLab.UnitTests/GetSourceLinkUrlTests.cs index e412a6b8..1ef882ed 100644 --- a/src/SourceLink.GitLab.UnitTests/GetSourceLinkUrlTests.cs +++ b/src/SourceLink.GitLab.UnitTests/GetSourceLinkUrlTests.cs @@ -30,11 +30,15 @@ public void EmptyHosts() } [Theory] - [InlineData("", "")] - [InlineData("", "/")] - [InlineData("/", "")] - [InlineData("/", "/")] - public void BuildSourceLinkUrl(string s1, string s2) + [InlineData("", "", null)] + [InlineData("", "/", null)] + [InlineData("/", "", null)] + [InlineData("/", "/", null)] + [InlineData("", "", "12.0")] + [InlineData("", "/", "12.0")] + [InlineData("/", "", "12.0")] + [InlineData("/", "/", "12.0")] + public void BuildSourceLinkUrl(string s1, string s2, string version) { var engine = new MockEngine(); @@ -44,7 +48,33 @@ public void BuildSourceLinkUrl(string s1, string s2) SourceRoot = new MockItem("/src/", KVP("RepositoryUrl", "http://subdomain.mygitlab.com:100/a/b" + s1), KVP("SourceControl", "git"), KVP("RevisionId", "0123456789abcdefABCDEF000000000000000000")), Hosts = new[] { - new MockItem("mygitlab.com", KVP("ContentUrl", "https://domain.com/x/y" + s2)), + version == null + ? new MockItem("mygitlab.com", KVP("ContentUrl", "https://domain.com/x/y" + s2)) + : new MockItem("mygitlab.com", KVP("ContentUrl", "https://domain.com/x/y" + s2), KVP("Version", version)), + } + }; + + bool result = task.Execute(); + AssertEx.AssertEqualToleratingWhitespaceDifferences("", engine.Log); + AssertEx.AreEqual("https://domain.com/x/y/a/b/-/raw/0123456789abcdefABCDEF000000000000000000/*", task.SourceLinkUrl); + Assert.True(result); + } + + [Theory] + [InlineData("", "/", "11.0")] + [InlineData("/", "", "11.0")] + [InlineData("/", "/", "11.0")] + public void BuildSourceLinkUrl_DeprecatedVersion(string s1, string s2, string version) + { + var engine = new MockEngine(); + + var task = new GetSourceLinkUrl() + { + BuildEngine = engine, + SourceRoot = new MockItem("/src/", KVP("RepositoryUrl", "http://subdomain.mygitlab.com:100/a/b" + s1), KVP("SourceControl", "git"), KVP("RevisionId", "0123456789abcdefABCDEF000000000000000000")), + Hosts = new[] + { + new MockItem("mygitlab.com", KVP("ContentUrl", "https://domain.com/x/y" + s2), KVP("Version", version)), } }; diff --git a/src/SourceLink.GitLab/GetSourceLinkUrl.cs b/src/SourceLink.GitLab/GetSourceLinkUrl.cs index 5c2f6d5a..33325efe 100644 --- a/src/SourceLink.GitLab/GetSourceLinkUrl.cs +++ b/src/SourceLink.GitLab/GetSourceLinkUrl.cs @@ -18,7 +18,33 @@ public sealed class GetSourceLinkUrl : GetSourceLinkUrlGitTask protected override string HostsItemGroupName => "SourceLinkGitLabHost"; protected override string ProviderDisplayName => "GitLab"; + private const string VersionMetadataName = "Version"; + + // see https://gitlab.com/gitlab-org/gitlab/-/issues/28848 + private static readonly Version s_versionWithNewUrlFormat = new Version(12, 0); + protected override string? BuildSourceLinkUrl(Uri contentUri, Uri gitUri, string relativeUrl, string revisionId, ITaskItem? hostItem) - => UriUtilities.Combine(UriUtilities.Combine(contentUri.ToString(), relativeUrl), "raw/" + revisionId + "/*"); + { + var path = GetVersion(hostItem) >= s_versionWithNewUrlFormat + ? "-/raw/" + revisionId + "/*" + : "raw/" + revisionId + "/*"; + return UriUtilities.Combine(UriUtilities.Combine(contentUri.ToString(), relativeUrl), path); + } + + private Version GetVersion(ITaskItem? hostItem) + { + var versionAsString = hostItem?.GetMetadata(VersionMetadataName); + if (!NullableString.IsNullOrEmpty(versionAsString)) + { + if (Version.TryParse(versionAsString, out var version)) + { + return version; + } + + Log.LogError(CommonResources.ItemOfItemGroupMustSpecifyMetadata, hostItem!.ItemSpec, HostsItemGroupName, VersionMetadataName); + } + + return s_versionWithNewUrlFormat; + } } } From eb501912e5ffca67618ed2c9b72b8555a919ee1d Mon Sep 17 00:00:00 2001 From: Michelle McDaniel Date: Mon, 16 Aug 2021 17:09:29 -0700 Subject: [PATCH 145/161] Move to v3 publishing (#737) --- azure-pipelines.yml | 1 + eng/Publishing.props | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 eng/Publishing.props diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9030c7d4..56424fcd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -153,6 +153,7 @@ stages: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - template: eng\common\templates\post-build\post-build.yml parameters: + publishingInfraVersion: 3 # Symbol validation isn't being very reliable lately. This should be enabled back # once this issue is resolved: https://github.com/dotnet/arcade/issues/2871 enableSymbolValidation: false diff --git a/eng/Publishing.props b/eng/Publishing.props new file mode 100644 index 00000000..797de4ea --- /dev/null +++ b/eng/Publishing.props @@ -0,0 +1,6 @@ + + + + 3 + + \ No newline at end of file From 6dcc7d005e38829efb6714d2ecfc4c0cb383e7d9 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 23 Aug 2021 12:37:21 +0000 Subject: [PATCH 146/161] [main] Update dependencies from dotnet/arcade (#729) [main] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 +- eng/common/build.sh | 1 + eng/common/cross/arm/sources.list.trusty | 11 -- eng/common/cross/arm/trusty-lttng-2.4.patch | 71 ---------- eng/common/cross/arm/trusty.patch | 97 ------------- eng/common/cross/arm64/sources.list.trusty | 11 -- eng/common/cross/build-rootfs.sh | 36 +++-- eng/common/cross/x86/sources.list.trusty | 11 -- eng/common/init-tools-native.sh | 2 +- eng/common/internal/Tools.csproj | 1 - eng/common/native/common-library.sh | 8 +- .../post-build/sourcelink-validation.ps1 | 17 ++- eng/common/sdk-task.ps1 | 1 + .../templates/job/publish-build-assets.yml | 26 +++- .../templates/job/source-index-stage1.yml | 21 ++- eng/common/templates/phases/base.yml | 130 ------------------ .../templates/phases/publish-build-assets.yml | 52 ------- eng/common/templates/steps/source-build.yml | 35 ++++- eng/common/tools.ps1 | 21 +++ eng/common/tools.sh | 7 + global.json | 4 +- 21 files changed, 133 insertions(+), 434 deletions(-) delete mode 100644 eng/common/cross/arm/sources.list.trusty delete mode 100644 eng/common/cross/arm/trusty-lttng-2.4.patch delete mode 100644 eng/common/cross/arm/trusty.patch delete mode 100644 eng/common/cross/arm64/sources.list.trusty delete mode 100644 eng/common/cross/x86/sources.list.trusty delete mode 100644 eng/common/templates/phases/base.yml delete mode 100644 eng/common/templates/phases/publish-build-assets.yml diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 0b89c2ef..48d4c88e 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 62a8aafffd4c68ef887680f6837abdff906a662c + fe787bd48ed72e51a98eb5e4e5e5af74edb531e5 https://github.com/dotnet/source-build-reference-packages diff --git a/eng/common/build.sh b/eng/common/build.sh index 55b298f1..9d3042a9 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -187,6 +187,7 @@ function InitializeCustomToolset { } function Build { + TryLogClientIpAddress InitializeToolset InitializeCustomToolset diff --git a/eng/common/cross/arm/sources.list.trusty b/eng/common/cross/arm/sources.list.trusty deleted file mode 100644 index 07d8f88d..00000000 --- a/eng/common/cross/arm/sources.list.trusty +++ /dev/null @@ -1,11 +0,0 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ trusty main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ trusty-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ trusty-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-security main restricted universe multiverse \ No newline at end of file diff --git a/eng/common/cross/arm/trusty-lttng-2.4.patch b/eng/common/cross/arm/trusty-lttng-2.4.patch deleted file mode 100644 index 8e4dd7ae..00000000 --- a/eng/common/cross/arm/trusty-lttng-2.4.patch +++ /dev/null @@ -1,71 +0,0 @@ -From e72c9d7ead60e3317bd6d1fade995c07021c947b Mon Sep 17 00:00:00 2001 -From: Mathieu Desnoyers -Date: Thu, 7 May 2015 13:25:04 -0400 -Subject: [PATCH] Fix: building probe providers with C++ compiler - -Robert Daniels wrote: -> > I'm attempting to use lttng userspace tracing with a C++ application -> > on an ARM platform. I'm using GCC 4.8.4 on Linux 3.14 with the 2.6 -> > release of lttng. I've compiled lttng-modules, lttng-ust, and -> > lttng-tools and have been able to get a simple test working with C -> > code. When I attempt to run the hello.cxx test on my target it will -> > segfault. -> -> -> I spent a little time digging into this issue and finally discovered the -> cause of my segfault with ARM C++ tracepoints. -> -> There is a struct called 'lttng_event' in ust-events.h which contains an -> empty union 'u'. This was the cause of my issue. Under C, this empty union -> compiles to a zero byte member while under C++ it compiles to a one byte -> member, and in my case was four-byte aligned which caused my C++ code to -> have the 'cds_list_head node' offset incorrectly by four bytes. This lead -> to an incorrect linked list structure which caused my issue. -> -> Since this union is empty, I simply removed it from the struct and everything -> worked correctly. -> -> I don't know the history or purpose behind this empty union so I'd like to -> know if this is a safe fix. If it is I can submit a patch with the union -> removed. - -That's a very nice catch! - -We do not support building tracepoint probe provider with -g++ yet, as stated in lttng-ust(3): - -"- Note for C++ support: although an application instrumented with - tracepoints can be compiled with g++, tracepoint probes should be - compiled with gcc (only tested with gcc so far)." - -However, if it works fine with this fix, then I'm tempted to take it, -especially because removing the empty union does not appear to affect -the layout of struct lttng_event as seen from liblttng-ust, which must -be compiled with a C compiler, and from probe providers compiled with -a C compiler. So all we are changing is the layout of a probe provider -compiled with a C++ compiler, which is anyway buggy at the moment, -because it is not compatible with the layout expected by liblttng-ust -compiled with a C compiler. - -Reported-by: Robert Daniels -Signed-off-by: Mathieu Desnoyers ---- - include/lttng/ust-events.h | 2 -- - 1 file changed, 2 deletions(-) - -diff --git a/usr/include/lttng/ust-events.h b/usr/include/lttng/ust-events.h -index 328a875..3d7a274 100644 ---- a/usr/include/lttng/ust-events.h -+++ b/usr/include/lttng/ust-events.h -@@ -407,8 +407,6 @@ struct lttng_event { - void *_deprecated1; - struct lttng_ctx *ctx; - enum lttng_ust_instrumentation instrumentation; -- union { -- } u; - struct cds_list_head node; /* Event list in session */ - struct cds_list_head _deprecated2; - void *_deprecated3; --- -2.7.4 - diff --git a/eng/common/cross/arm/trusty.patch b/eng/common/cross/arm/trusty.patch deleted file mode 100644 index 2f2972f8..00000000 --- a/eng/common/cross/arm/trusty.patch +++ /dev/null @@ -1,97 +0,0 @@ -diff -u -r a/usr/include/urcu/uatomic/generic.h b/usr/include/urcu/uatomic/generic.h ---- a/usr/include/urcu/uatomic/generic.h 2014-03-28 06:04:42.000000000 +0900 -+++ b/usr/include/urcu/uatomic/generic.h 2017-02-13 10:35:21.189927116 +0900 -@@ -65,17 +65,17 @@ - switch (len) { - #ifdef UATOMIC_HAS_ATOMIC_BYTE - case 1: -- return __sync_val_compare_and_swap_1(addr, old, _new); -+ return __sync_val_compare_and_swap_1((uint8_t *) addr, old, _new); - #endif - #ifdef UATOMIC_HAS_ATOMIC_SHORT - case 2: -- return __sync_val_compare_and_swap_2(addr, old, _new); -+ return __sync_val_compare_and_swap_2((uint16_t *) addr, old, _new); - #endif - case 4: -- return __sync_val_compare_and_swap_4(addr, old, _new); -+ return __sync_val_compare_and_swap_4((uint32_t *) addr, old, _new); - #if (CAA_BITS_PER_LONG == 64) - case 8: -- return __sync_val_compare_and_swap_8(addr, old, _new); -+ return __sync_val_compare_and_swap_8((uint64_t *) addr, old, _new); - #endif - } - _uatomic_link_error(); -@@ -100,20 +100,20 @@ - switch (len) { - #ifdef UATOMIC_HAS_ATOMIC_BYTE - case 1: -- __sync_and_and_fetch_1(addr, val); -+ __sync_and_and_fetch_1((uint8_t *) addr, val); - return; - #endif - #ifdef UATOMIC_HAS_ATOMIC_SHORT - case 2: -- __sync_and_and_fetch_2(addr, val); -+ __sync_and_and_fetch_2((uint16_t *) addr, val); - return; - #endif - case 4: -- __sync_and_and_fetch_4(addr, val); -+ __sync_and_and_fetch_4((uint32_t *) addr, val); - return; - #if (CAA_BITS_PER_LONG == 64) - case 8: -- __sync_and_and_fetch_8(addr, val); -+ __sync_and_and_fetch_8((uint64_t *) addr, val); - return; - #endif - } -@@ -139,20 +139,20 @@ - switch (len) { - #ifdef UATOMIC_HAS_ATOMIC_BYTE - case 1: -- __sync_or_and_fetch_1(addr, val); -+ __sync_or_and_fetch_1((uint8_t *) addr, val); - return; - #endif - #ifdef UATOMIC_HAS_ATOMIC_SHORT - case 2: -- __sync_or_and_fetch_2(addr, val); -+ __sync_or_and_fetch_2((uint16_t *) addr, val); - return; - #endif - case 4: -- __sync_or_and_fetch_4(addr, val); -+ __sync_or_and_fetch_4((uint32_t *) addr, val); - return; - #if (CAA_BITS_PER_LONG == 64) - case 8: -- __sync_or_and_fetch_8(addr, val); -+ __sync_or_and_fetch_8((uint64_t *) addr, val); - return; - #endif - } -@@ -180,17 +180,17 @@ - switch (len) { - #ifdef UATOMIC_HAS_ATOMIC_BYTE - case 1: -- return __sync_add_and_fetch_1(addr, val); -+ return __sync_add_and_fetch_1((uint8_t *) addr, val); - #endif - #ifdef UATOMIC_HAS_ATOMIC_SHORT - case 2: -- return __sync_add_and_fetch_2(addr, val); -+ return __sync_add_and_fetch_2((uint16_t *) addr, val); - #endif - case 4: -- return __sync_add_and_fetch_4(addr, val); -+ return __sync_add_and_fetch_4((uint32_t *) addr, val); - #if (CAA_BITS_PER_LONG == 64) - case 8: -- return __sync_add_and_fetch_8(addr, val); -+ return __sync_add_and_fetch_8((uint64_t *) addr, val); - #endif - } - _uatomic_link_error(); diff --git a/eng/common/cross/arm64/sources.list.trusty b/eng/common/cross/arm64/sources.list.trusty deleted file mode 100644 index 07d8f88d..00000000 --- a/eng/common/cross/arm64/sources.list.trusty +++ /dev/null @@ -1,11 +0,0 @@ -deb http://ports.ubuntu.com/ubuntu-ports/ trusty main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main restricted universe -deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main restricted universe - -deb http://ports.ubuntu.com/ubuntu-ports/ trusty-backports main restricted -deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-backports main restricted - -deb http://ports.ubuntu.com/ubuntu-ports/ trusty-security main restricted universe multiverse -deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-security main restricted universe multiverse \ No newline at end of file diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 735a4c82..5c05b39f 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -6,10 +6,10 @@ usage() { echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [--skipunmount] --rootfsdir ]" echo "BuildArch can be: arm(default), armel, arm64, x86" - echo "CodeName - optional, Code name for Linux, can be: trusty, xenial(default), zesty, bionic, alpine, alpine3.9 or alpine3.13. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." - echo " for FreeBSD can be: freebsd11 or freebsd12." + echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine, alpine3.9 or alpine3.13. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." + echo " for FreeBSD can be: freebsd11, freebsd12, freebsd13" echo " for illumos can be: illumos." - echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FReeBSD" + echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FreeBSD" echo "--skipunmount - optional, will skip the unmount of rootfs folder." echo "--use-mirror - optional, use mirror URL to fetch resources, when available." exit 1 @@ -60,13 +60,15 @@ __AlpinePackages+=" krb5-dev" __AlpinePackages+=" openssl-dev" __AlpinePackages+=" zlib-dev" -__FreeBSDBase="12.1-RELEASE" +__FreeBSDBase="12.2-RELEASE" __FreeBSDPkg="1.12.0" +__FreeBSDABI="12" __FreeBSDPackages="libunwind" __FreeBSDPackages+=" icu" __FreeBSDPackages+=" libinotify" __FreeBSDPackages+=" lttng-ust" __FreeBSDPackages+=" krb5" +__FreeBSDPackages+=" terminfo-db" __IllumosPackages="icu-64.2nb2" __IllumosPackages+=" mit-krb5-1.16.2nb4" @@ -144,11 +146,6 @@ while :; do no-lldb) unset __LLDB_Package ;; - trusty) # Ubuntu 14.04 - if [ "$__CodeName" != "jessie" ]; then - __CodeName=trusty - fi - ;; xenial) # Ubuntu 16.04 if [ "$__CodeName" != "jessie" ]; then __CodeName=xenial @@ -208,12 +205,20 @@ while :; do ;; freebsd11) __FreeBSDBase="11.3-RELEASE" + __FreeBSDABI="11" ;& freebsd12) __CodeName=freebsd __BuildArch=x64 __SkipUnmount=1 ;; + freebsd13) + __CodeName=freebsd + __FreeBSDBase="13.0-RELEASE" + __FreeBSDABI="13" + __BuildArch=x64 + __SkipUnmount=1 + ;; illumos) __CodeName=illumos __BuildArch=x64 @@ -291,9 +296,9 @@ if [[ "$__CodeName" == "alpine" ]]; then rm -r $__ApkToolsDir elif [[ "$__CodeName" == "freebsd" ]]; then mkdir -p $__RootfsDir/usr/local/etc + JOBS="$(getconf _NPROCESSORS_ONLN)" wget -O - https://download.freebsd.org/ftp/releases/amd64/${__FreeBSDBase}/base.txz | tar -C $__RootfsDir -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version - # For now, ask for 11 ABI even on 12. This can be revisited later. - echo "ABI = \"FreeBSD:11:amd64\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > ${__RootfsDir}/usr/local/etc/pkg.conf + echo "ABI = \"FreeBSD:${__FreeBSDABI}:amd64\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > ${__RootfsDir}/usr/local/etc/pkg.conf echo "FreeBSD: { url: "pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"${__RootfsDir}/usr/share/keys/pkg\", enabled: yes }" > ${__RootfsDir}/etc/pkg/FreeBSD.conf mkdir -p $__RootfsDir/tmp # get and build package manager @@ -301,7 +306,7 @@ elif [[ "$__CodeName" == "freebsd" ]]; then cd $__RootfsDir/tmp/pkg-${__FreeBSDPkg} # needed for install to succeed mkdir -p $__RootfsDir/host/etc - ./autogen.sh && ./configure --prefix=$__RootfsDir/host && make && make install + ./autogen.sh && ./configure --prefix=$__RootfsDir/host && make -j "$JOBS" && make install rm -rf $__RootfsDir/tmp/pkg-${__FreeBSDPkg} # install packages we need. INSTALL_AS_USER=$(whoami) $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf update @@ -364,13 +369,6 @@ elif [[ -n $__CodeName ]]; then umount $__RootfsDir/* || true fi - if [[ "$__BuildArch" == "arm" && "$__CodeName" == "trusty" ]]; then - pushd $__RootfsDir - patch -p1 < $__CrossDir/$__BuildArch/trusty.patch - patch -p1 < $__CrossDir/$__BuildArch/trusty-lttng-2.4.patch - popd - fi - if [[ "$__BuildArch" == "armel" && "$__CodeName" == "jessie" ]]; then pushd $__RootfsDir patch -p1 < $__CrossDir/$__BuildArch/armel.jessie.patch diff --git a/eng/common/cross/x86/sources.list.trusty b/eng/common/cross/x86/sources.list.trusty deleted file mode 100644 index 9b308543..00000000 --- a/eng/common/cross/x86/sources.list.trusty +++ /dev/null @@ -1,11 +0,0 @@ -deb http://archive.ubuntu.com/ubuntu/ trusty main restricted universe -deb-src http://archive.ubuntu.com/ubuntu/ trusty main restricted universe - -deb http://archive.ubuntu.com/ubuntu/ trusty-updates main restricted universe -deb-src http://archive.ubuntu.com/ubuntu/ trusty-updates main restricted universe - -deb http://archive.ubuntu.com/ubuntu/ trusty-backports main restricted -deb-src http://archive.ubuntu.com/ubuntu/ trusty-backports main restricted - -deb http://archive.ubuntu.com/ubuntu/ trusty-security main restricted universe multiverse -deb-src http://archive.ubuntu.com/ubuntu/ trusty-security main restricted universe multiverse diff --git a/eng/common/init-tools-native.sh b/eng/common/init-tools-native.sh index 5bd205b5..3e6a8d6a 100755 --- a/eng/common/init-tools-native.sh +++ b/eng/common/init-tools-native.sh @@ -10,7 +10,7 @@ force=false download_retries=5 retry_wait_time_seconds=30 global_json_file="$(dirname "$(dirname "${scriptroot}")")/global.json" -declare -A native_assets +declare -a native_assets . $scriptroot/pipeline-logging-functions.sh . $scriptroot/native/common-library.sh diff --git a/eng/common/internal/Tools.csproj b/eng/common/internal/Tools.csproj index 2067b8df..beb9c464 100644 --- a/eng/common/internal/Tools.csproj +++ b/eng/common/internal/Tools.csproj @@ -1,5 +1,4 @@ - net472 diff --git a/eng/common/native/common-library.sh b/eng/common/native/common-library.sh index bf272dcf..080c2c28 100755 --- a/eng/common/native/common-library.sh +++ b/eng/common/native/common-library.sh @@ -148,8 +148,12 @@ function NewScriptShim { fi if [[ ! -f $tool_file_path ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Specified tool file path:'$tool_file_path' does not exist" - return 1 + # try to see if the path is lower cased + tool_file_path="$(echo $tool_file_path | tr "[:upper:]" "[:lower:]")" + if [[ ! -f $tool_file_path ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Specified tool file path:'$tool_file_path' does not exist" + return 1 + fi fi local shim_contents=$'#!/usr/bin/env bash\n' diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1 index 85c89861..3b6fc953 100644 --- a/eng/common/post-build/sourcelink-validation.ps1 +++ b/eng/common/post-build/sourcelink-validation.ps1 @@ -17,6 +17,7 @@ $global:RepoFiles = @{} $MaxParallelJobs = 16 $MaxRetries = 5 +$RetryWaitTimeInSeconds = 30 # Wait time between check for system load $SecondsBetweenLoadChecks = 10 @@ -99,9 +100,9 @@ $ValidatePackage = { $Status = 200 $Cache = $using:RepoFiles - $totalRetries = 0 + $attempts = 0 - while ($totalRetries -lt $using:MaxRetries) { + while ($attempts -lt $using:MaxRetries) { if ( !($Cache.ContainsKey($FilePath)) ) { try { $Uri = $Link -as [System.URI] @@ -113,7 +114,7 @@ $ValidatePackage = { else { # If it's not a github link, we want to break out of the loop and not retry. $Status = 0 - $totalRetries = $using:MaxRetries + $attempts = $using:MaxRetries } } catch { @@ -123,9 +124,15 @@ $ValidatePackage = { } if ($Status -ne 200) { - $totalRetries++ + $attempts++ - if ($totalRetries -ge $using:MaxRetries) { + if ($attempts -lt $using:MaxRetries) + { + $attemptsLeft = $using:MaxRetries - $attempts + Write-Warning "Download failed, $attemptsLeft attempts remaining, will retry in $using:RetryWaitTimeInSeconds seconds" + Start-Sleep -Seconds $using:RetryWaitTimeInSeconds + } + else { if ($NumFailedLinks -eq 0) { if ($FailedFiles.Value -eq 0) { Write-Host diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index b1bca63a..7ffa3591 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -83,6 +83,7 @@ try { } if ($restore) { + Try-LogClientIpAddress Build 'Restore' } diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 3b9e2524..fe9dfdf7 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -94,7 +94,31 @@ jobs: PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs.txt' PublishLocation: Container ArtifactName: ReleaseConfigs - + + - task: powershell@2 + displayName: Check if SymbolPublishingExclusionsFile.txt exists + inputs: + targetType: inline + script: | + $symbolExclusionfile = "$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt" + if(Test-Path -Path $symbolExclusionfile) + { + Write-Host "SymbolExclusionFile exists" + Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]true" + } + else{ + Write-Host "Symbols Exclusion file does not exists" + Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]false" + } + + - task: PublishBuildArtifacts@1 + displayName: Publish SymbolPublishingExclusionsFile Artifact + condition: eq(variables['SymbolExclusionFile'], 'true') + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' + PublishLocation: Container + ArtifactName: ReleaseConfigs + - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - template: /eng/common/templates/steps/publish-logs.yml parameters: diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml index b58d4236..1cc0c29e 100644 --- a/eng/common/templates/job/source-index-stage1.yml +++ b/eng/common/templates/job/source-index-stage1.yml @@ -34,29 +34,24 @@ jobs: inputs: packageType: sdk version: 3.1.x - - - task: UseDotNet@2 - displayName: Use .NET Core sdk - inputs: - useGlobalJson: true + installationPath: $(Agent.TempDirectory)/dotnet + workingDirectory: $(Agent.TempDirectory) - script: | - dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path .source-index/tools - dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path .source-index/tools - echo ##vso[task.prependpath]$(Build.SourcesDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools displayName: Download Tools + # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. + workingDirectory: $(Agent.TempDirectory) - script: ${{ parameters.sourceIndexBuildCommand }} displayName: Build Repository - - script: BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output + - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output displayName: Process Binlog into indexable sln - env: - DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX: 2 - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - script: UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) + - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) displayName: Upload stage1 artifacts to source index env: BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) - DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX: 2 diff --git a/eng/common/templates/phases/base.yml b/eng/common/templates/phases/base.yml deleted file mode 100644 index 0123cf43..00000000 --- a/eng/common/templates/phases/base.yml +++ /dev/null @@ -1,130 +0,0 @@ -parameters: - # Optional: Clean sources before building - clean: true - - # Optional: Git fetch depth - fetchDepth: '' - - # Optional: name of the phase (not specifying phase name may cause name collisions) - name: '' - # Optional: display name of the phase - displayName: '' - - # Optional: condition for the job to run - condition: '' - - # Optional: dependencies of the phase - dependsOn: '' - - # Required: A defined YAML queue - queue: {} - - # Required: build steps - steps: [] - - # Optional: variables - variables: {} - - # Optional: should run as a public build even in the internal project - # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. - runAsPublic: false - - ## Telemetry variables - - # Optional: enable sending telemetry - # if 'true', these "variables" must be specified in the variables object or as part of the queue matrix - # _HelixBuildConfig - differentiate between Debug, Release, other - # _HelixSource - Example: build/product - # _HelixType - Example: official/dotnet/arcade/$(Build.SourceBranch) - enableTelemetry: false - - # Optional: Enable installing Microbuild plugin - # if 'true', these "variables" must be specified in the variables object or as part of the queue matrix - # _TeamName - the name of your team - # _SignType - 'test' or 'real' - enableMicrobuild: false - -# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, -# and some (Microbuild) should only be applied to non-PR cases for internal builds. - -phases: -- phase: ${{ parameters.name }} - - ${{ if ne(parameters.displayName, '') }}: - displayName: ${{ parameters.displayName }} - - ${{ if ne(parameters.condition, '') }}: - condition: ${{ parameters.condition }} - - ${{ if ne(parameters.dependsOn, '') }}: - dependsOn: ${{ parameters.dependsOn }} - - queue: ${{ parameters.queue }} - - ${{ if ne(parameters.variables, '') }}: - variables: - ${{ insert }}: ${{ parameters.variables }} - - steps: - - checkout: self - clean: ${{ parameters.clean }} - ${{ if ne(parameters.fetchDepth, '') }}: - fetchDepth: ${{ parameters.fetchDepth }} - - - ${{ if eq(parameters.enableTelemetry, 'true') }}: - - template: /eng/common/templates/steps/telemetry-start.yml - parameters: - buildConfig: $(_HelixBuildConfig) - helixSource: $(_HelixSource) - helixType: $(_HelixType) - runAsPublic: ${{ parameters.runAsPublic }} - - - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - # Internal only resource, and Microbuild signing shouldn't be applied to PRs. - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: MicroBuildSigningPlugin@2 - displayName: Install MicroBuild plugin - inputs: - signType: $(_SignType) - zipSources: false - feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json - - env: - TeamName: $(_TeamName) - continueOnError: false - condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) - - # Run provided build steps - - ${{ parameters.steps }} - - - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - # Internal only resources - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: MicroBuildCleanup@1 - displayName: Execute Microbuild cleanup tasks - condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) - env: - TeamName: $(_TeamName) - - - ${{ if eq(parameters.enableTelemetry, 'true') }}: - - template: /eng/common/templates/steps/telemetry-end.yml - parameters: - helixSource: $(_HelixSource) - helixType: $(_HelixType) - - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: CopyFiles@2 - displayName: Gather Asset Manifests - inputs: - SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest' - TargetFolder: '$(Build.StagingDirectory)/AssetManifests' - continueOnError: false - condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) - - task: PublishBuildArtifacts@1 - displayName: Push Asset Manifests - inputs: - PathtoPublish: '$(Build.StagingDirectory)/AssetManifests' - PublishLocation: Container - ArtifactName: AssetManifests - continueOnError: false - condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) diff --git a/eng/common/templates/phases/publish-build-assets.yml b/eng/common/templates/phases/publish-build-assets.yml deleted file mode 100644 index 4e51e472..00000000 --- a/eng/common/templates/phases/publish-build-assets.yml +++ /dev/null @@ -1,52 +0,0 @@ -parameters: - dependsOn: '' - queue: {} - configuration: 'Debug' - condition: succeeded() - continueOnError: false - runAsPublic: false - publishUsingPipelines: false -phases: - - phase: Asset_Registry_Publish - displayName: Publish to Build Asset Registry - dependsOn: ${{ parameters.dependsOn }} - queue: ${{ parameters.queue }} - variables: - _BuildConfig: ${{ parameters.configuration }} - steps: - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: DownloadBuildArtifacts@0 - displayName: Download artifact - inputs: - artifactName: AssetManifests - downloadPath: '$(Build.StagingDirectory)/Download' - checkDownloadedFiles: true - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} - - task: AzureKeyVault@1 - inputs: - azureSubscription: 'DotNet-Engineering-Services_KeyVault' - KeyVaultName: EngKeyVault - SecretsFilter: 'MaestroAccessToken' - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} - - task: PowerShell@2 - displayName: Publish Build Assets - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet - /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' - /p:BuildAssetRegistryToken=$(MaestroAccessToken) - /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com - /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} - /p:Configuration=$(_BuildConfig) - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} - - task: PublishBuildArtifacts@1 - displayName: Publish Logs to VSTS - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' - PublishLocation: Container - ArtifactName: $(Agent.Os)_Asset_Registry_Publish - continueOnError: true - condition: always() diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml index 705b7a1c..ba40dc82 100644 --- a/eng/common/templates/steps/source-build.yml +++ b/eng/common/templates/steps/source-build.yml @@ -18,6 +18,35 @@ steps: set -x df -h + # If building on the internal project, the artifact feeds variable may be available (usually only if needed) + # In that case, call the feed setup script to add internal feeds corresponding to public ones. + # In addition, add an msbuild argument to copy the WIP from the repo to the target build location. + # This is because SetupNuGetSources.sh will alter the current NuGet.config file, and we need to preserve those + # changes. + $internalRestoreArgs= + if [ '$(dn-bot-dnceng-artifact-feeds-rw)' != '$''(dn-bot-dnceng-artifact-feeds-rw)' ]; then + # Temporarily work around https://github.com/dotnet/arcade/issues/7709 + chmod +x $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh + $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $(dn-bot-dnceng-artifact-feeds-rw) + internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true' + + # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo. + # This only works if there is a username/email configured, which won't be the case in most CI runs. + git config --get user.email + if [ $? -ne 0 ]; then + git config user.email dn-bot@microsoft.com + git config user.name dn-bot + fi + fi + + # If building on the internal project, the internal storage variable may be available (usually only if needed) + # In that case, add variables to allow the download of internal runtimes if the specified versions are not found + # in the default public locations. + internalRuntimeDownloadArgs= + if [ '$(dotnetclimsrc-read-sas-token-base64)' != '$''(dotnetclimsrc-read-sas-token-base64)' ]; then + internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetclimsrc.blob.core.windows.net/dotnet /p:DotNetRuntimeSourceFeedKey=$(dotnetclimsrc-read-sas-token-base64) --runtimesourcefeed https://dotnetclimsrc.blob.core.windows.net/dotnet --runtimesourcefeedkey $(dotnetclimsrc-read-sas-token-base64)' + fi + buildConfig=Release # Check if AzDO substitutes in a build config from a variable, and use it if so. if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then @@ -29,11 +58,6 @@ steps: officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)' fi - internalRuntimeDownloadArgs= - if [ '$(dotnetclimsrc-read-sas-token-base64)' != '$''(dotnetclimsrc-read-sas-token-base64)' ]; then - internalRuntimeDownloadArgs='--runtimesourcefeed https://dotnetclimsrc.blob.core.windows.net/dotnet --runtimesourcefeedkey $(dotnetclimsrc-read-sas-token-base64)' - fi - targetRidArgs= if [ '${{ parameters.platform.targetRID }}' != '' ]; then targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' @@ -49,6 +73,7 @@ steps: --restore --build --pack $publishArgs -bl \ $officialBuildArgs \ $internalRuntimeDownloadArgs \ + $internalRestoreArgs \ $targetRidArgs \ /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ /p:ArcadeBuildFromSource=true diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 5d526c74..e607aa43 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -154,6 +154,9 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { return $global:_DotNetInstallDir } + # In case of network error, try to log the current IP for reference + Try-LogClientIpAddress + # Don't resolve runtime, shared framework, or SDK from other locations to ensure build determinism $env:DOTNET_MULTILEVEL_LOOKUP=0 @@ -872,3 +875,21 @@ if (!$disableConfigureToolsetImport) { } } } + +function Try-LogClientIpAddress() +{ + Write-Host "Attempting to log this client's IP for Azure Package feed telemetry purposes" + try + { + $result = Invoke-WebRequest -Uri "http://co1.msedge.net/fdv2/diagnostics.aspx" -UseBasicParsing + $lines = $result.Content.Split([Environment]::NewLine) + $socketIp = $lines | Select-String -Pattern "^Socket IP:.*" + Write-Host $socketIp + $clientIp = $lines | Select-String -Pattern "^Client IP:.*" + Write-Host $clientIp + } + catch + { + Write-Host "Unable to get this machine's effective IP address for logging: $_" + } +} diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 828119be..3c5f3a12 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -399,6 +399,13 @@ function StopProcesses { return 0 } +function TryLogClientIpAddress () { + echo 'Attempting to log this client''s IP for Azure Package feed telemetry purposes' + if command -v curl > /dev/null; then + curl -s 'http://co1.msedge.net/fdv2/diagnostics.aspx' | grep ' IP: ' + fi +} + function MSBuild { local args=$@ if [[ "$pipelines_log" == true ]]; then diff --git a/global.json b/global.json index dfc9bf3c..eae10d81 100644 --- a/global.json +++ b/global.json @@ -1,8 +1,8 @@ { "tools": { - "dotnet": "6.0.100-preview.6.21355.2" + "dotnet": "6.0.100-rc.1.21379.2" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21379.2" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21420.4" } } From b47f05da7091ea0f69a43d4589c0fda3fdbb1ec7 Mon Sep 17 00:00:00 2001 From: Dan Seefeldt Date: Thu, 2 Sep 2021 19:54:59 -0500 Subject: [PATCH 147/161] Conditionally build for only 6.0 when building from source (#742) --- src/Microsoft.Build.Tasks.Git/Microsoft.Build.Tasks.Git.csproj | 1 + .../Microsoft.SourceLink.AzureDevOpsServer.Git.csproj | 1 + .../Microsoft.SourceLink.AzureRepos.Git.csproj | 1 + .../Microsoft.SourceLink.Bitbucket.Git.csproj | 1 + src/SourceLink.Common/Microsoft.SourceLink.Common.csproj | 1 + src/SourceLink.GitHub/Microsoft.SourceLink.GitHub.csproj | 1 + src/SourceLink.GitLab/Microsoft.SourceLink.GitLab.csproj | 1 + src/SourceLink.GitWeb/Microsoft.SourceLink.GitWeb.csproj | 1 + src/SourceLink.Gitea/Microsoft.SourceLink.Gitea.csproj | 1 + 9 files changed, 9 insertions(+) diff --git a/src/Microsoft.Build.Tasks.Git/Microsoft.Build.Tasks.Git.csproj b/src/Microsoft.Build.Tasks.Git/Microsoft.Build.Tasks.Git.csproj index 3fa8c71d..da0277bf 100644 --- a/src/Microsoft.Build.Tasks.Git/Microsoft.Build.Tasks.Git.csproj +++ b/src/Microsoft.Build.Tasks.Git/Microsoft.Build.Tasks.Git.csproj @@ -3,6 +3,7 @@ net472;net6.0 + net6.0 true diff --git a/src/SourceLink.AzureDevOpsServer.Git/Microsoft.SourceLink.AzureDevOpsServer.Git.csproj b/src/SourceLink.AzureDevOpsServer.Git/Microsoft.SourceLink.AzureDevOpsServer.Git.csproj index ceec252b..84e89bdf 100644 --- a/src/SourceLink.AzureDevOpsServer.Git/Microsoft.SourceLink.AzureDevOpsServer.Git.csproj +++ b/src/SourceLink.AzureDevOpsServer.Git/Microsoft.SourceLink.AzureDevOpsServer.Git.csproj @@ -3,6 +3,7 @@ net472;net6.0 + net6.0 true diff --git a/src/SourceLink.AzureRepos.Git/Microsoft.SourceLink.AzureRepos.Git.csproj b/src/SourceLink.AzureRepos.Git/Microsoft.SourceLink.AzureRepos.Git.csproj index bdb048e8..b01b4291 100644 --- a/src/SourceLink.AzureRepos.Git/Microsoft.SourceLink.AzureRepos.Git.csproj +++ b/src/SourceLink.AzureRepos.Git/Microsoft.SourceLink.AzureRepos.Git.csproj @@ -3,6 +3,7 @@ net472;net6.0 + net6.0 true diff --git a/src/SourceLink.Bitbucket.Git/Microsoft.SourceLink.Bitbucket.Git.csproj b/src/SourceLink.Bitbucket.Git/Microsoft.SourceLink.Bitbucket.Git.csproj index bf27b3fc..11ac33df 100644 --- a/src/SourceLink.Bitbucket.Git/Microsoft.SourceLink.Bitbucket.Git.csproj +++ b/src/SourceLink.Bitbucket.Git/Microsoft.SourceLink.Bitbucket.Git.csproj @@ -3,6 +3,7 @@ net472;net6.0 + net6.0 true diff --git a/src/SourceLink.Common/Microsoft.SourceLink.Common.csproj b/src/SourceLink.Common/Microsoft.SourceLink.Common.csproj index 09ac8ec5..534b3163 100644 --- a/src/SourceLink.Common/Microsoft.SourceLink.Common.csproj +++ b/src/SourceLink.Common/Microsoft.SourceLink.Common.csproj @@ -3,6 +3,7 @@ net472;net6.0 + net6.0 true diff --git a/src/SourceLink.GitHub/Microsoft.SourceLink.GitHub.csproj b/src/SourceLink.GitHub/Microsoft.SourceLink.GitHub.csproj index 626b9cc5..0c025ca9 100644 --- a/src/SourceLink.GitHub/Microsoft.SourceLink.GitHub.csproj +++ b/src/SourceLink.GitHub/Microsoft.SourceLink.GitHub.csproj @@ -3,6 +3,7 @@ net472;net6.0 + net6.0 true diff --git a/src/SourceLink.GitLab/Microsoft.SourceLink.GitLab.csproj b/src/SourceLink.GitLab/Microsoft.SourceLink.GitLab.csproj index 4a1d0af7..ada89861 100644 --- a/src/SourceLink.GitLab/Microsoft.SourceLink.GitLab.csproj +++ b/src/SourceLink.GitLab/Microsoft.SourceLink.GitLab.csproj @@ -3,6 +3,7 @@ net472;net6.0 + net6.0 true diff --git a/src/SourceLink.GitWeb/Microsoft.SourceLink.GitWeb.csproj b/src/SourceLink.GitWeb/Microsoft.SourceLink.GitWeb.csproj index 19f92a8d..b9ec7b4a 100644 --- a/src/SourceLink.GitWeb/Microsoft.SourceLink.GitWeb.csproj +++ b/src/SourceLink.GitWeb/Microsoft.SourceLink.GitWeb.csproj @@ -3,6 +3,7 @@ net472;net6.0 + net6.0 true diff --git a/src/SourceLink.Gitea/Microsoft.SourceLink.Gitea.csproj b/src/SourceLink.Gitea/Microsoft.SourceLink.Gitea.csproj index fa9e33e4..acca3af5 100644 --- a/src/SourceLink.Gitea/Microsoft.SourceLink.Gitea.csproj +++ b/src/SourceLink.Gitea/Microsoft.SourceLink.Gitea.csproj @@ -3,6 +3,7 @@ net472;net6.0 + net6.0 true From 389b28318be327cc8c48259f8134b784e52799f8 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 6 Sep 2021 12:36:49 +0000 Subject: [PATCH 148/161] [main] Update dependencies from dotnet/arcade (#743) [main] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- eng/Versions.props | 2 +- eng/common/build.sh | 5 ++++- eng/common/post-build/sourcelink-validation.ps1 | 8 ++++++-- eng/common/sdk-task.ps1 | 4 +++- eng/common/templates/job/job.yml | 2 +- eng/common/tools.ps1 | 17 ++++++++++++++--- eng/common/tools.sh | 11 ++++++++++- global.json | 4 ++-- 9 files changed, 43 insertions(+), 14 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 48d4c88e..2d3cac33 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - fe787bd48ed72e51a98eb5e4e5e5af74edb531e5 + 0ec8dbacaefa7ccdb4bc96e120e07dc60c6eda98 https://github.com/dotnet/source-build-reference-packages diff --git a/eng/Versions.props b/eng/Versions.props index 4afc3c3e..ad116f1c 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,4 +1,4 @@ - + diff --git a/eng/common/build.sh b/eng/common/build.sh index 9d3042a9..bc07a1c6 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -187,7 +187,10 @@ function InitializeCustomToolset { } function Build { - TryLogClientIpAddress + + if [[ "$ci" == true ]]; then + TryLogClientIpAddress + fi InitializeToolset InitializeCustomToolset diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1 index 3b6fc953..e8ab29af 100644 --- a/eng/common/post-build/sourcelink-validation.ps1 +++ b/eng/common/post-build/sourcelink-validation.ps1 @@ -107,8 +107,12 @@ $ValidatePackage = { try { $Uri = $Link -as [System.URI] - # Only GitHub links are valid - if ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match 'github' -or $Uri.Host -match 'githubusercontent')) { + if ($Link -match "submodules") { + # Skip submodule links until sourcelink properly handles submodules + $Status = 200 + } + elseif ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match 'github' -or $Uri.Host -match 'githubusercontent')) { + # Only GitHub links are valid $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode } else { diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index 7ffa3591..7ab9baac 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -83,7 +83,9 @@ try { } if ($restore) { - Try-LogClientIpAddress + if ($ci) { + Try-LogClientIpAddress + } Build 'Restore' } diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 86696793..37dceb1b 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -103,7 +103,7 @@ jobs: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - - task: MicroBuildSigningPlugin@2 + - task: MicroBuildSigningPlugin@3 displayName: Install MicroBuild plugin inputs: signType: $(_SignType) diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index e607aa43..02347914 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -154,9 +154,6 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { return $global:_DotNetInstallDir } - # In case of network error, try to log the current IP for reference - Try-LogClientIpAddress - # Don't resolve runtime, shared framework, or SDK from other locations to ensure build determinism $env:DOTNET_MULTILEVEL_LOOKUP=0 @@ -166,6 +163,9 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { # Disable telemetry on CI. if ($ci) { $env:DOTNET_CLI_TELEMETRY_OPTOUT=1 + + # In case of network error, try to log the current IP for reference + Try-LogClientIpAddress } # Source Build uses DotNetCoreSdkDir variable @@ -709,6 +709,15 @@ function MSBuild() { Write-PipelineSetVariable -Name 'NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS' -Value '20' } + if ($ci) { + $env:NUGET_ENABLE_EXPERIMENTAL_HTTP_RETRY = 'true' + $env:NUGET_EXPERIMENTAL_MAX_NETWORK_TRY_COUNT = 6 + $env:NUGET_EXPERIMENTAL_NETWORK_RETRY_DELAY_MILLISECONDS = 1000 + Write-PipelineSetVariable -Name 'NUGET_ENABLE_EXPERIMENTAL_HTTP_RETRY' -Value 'true' + Write-PipelineSetVariable -Name 'NUGET_EXPERIMENTAL_MAX_NETWORK_TRY_COUNT' -Value '6' + Write-PipelineSetVariable -Name 'NUGET_EXPERIMENTAL_NETWORK_RETRY_DELAY_MILLISECONDS' -Value '1000' + } + $toolsetBuildProject = InitializeToolset $basePath = Split-Path -parent $toolsetBuildProject $possiblePaths = @( @@ -717,6 +726,8 @@ function MSBuild() { (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll')), (Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.ArcadeLogging.dll')), (Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.Arcade.Sdk.dll')) + (Join-Path $basePath (Join-Path netcoreapp3.1 'Microsoft.DotNet.ArcadeLogging.dll')), + (Join-Path $basePath (Join-Path netcoreapp3.1 'Microsoft.DotNet.Arcade.Sdk.dll')) ) $selectedPath = $null foreach ($path in $possiblePaths) { diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 3c5f3a12..6a4871ef 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -402,7 +402,7 @@ function StopProcesses { function TryLogClientIpAddress () { echo 'Attempting to log this client''s IP for Azure Package feed telemetry purposes' if command -v curl > /dev/null; then - curl -s 'http://co1.msedge.net/fdv2/diagnostics.aspx' | grep ' IP: ' + curl -s 'http://co1.msedge.net/fdv2/diagnostics.aspx' | grep ' IP: ' || true fi } @@ -417,6 +417,13 @@ function MSBuild { export NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS=20 Write-PipelineSetVariable -name "NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS" -value "20" Write-PipelineSetVariable -name "NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS" -value "20" + + export NUGET_ENABLE_EXPERIMENTAL_HTTP_RETRY=true + export NUGET_EXPERIMENTAL_MAX_NETWORK_TRY_COUNT=6 + export NUGET_EXPERIMENTAL_NETWORK_RETRY_DELAY_MILLISECONDS=1000 + Write-PipelineSetVariable -name "NUGET_ENABLE_EXPERIMENTAL_HTTP_RETRY" -value "true" + Write-PipelineSetVariable -name "NUGET_EXPERIMENTAL_MAX_NETWORK_TRY_COUNT" -value "6" + Write-PipelineSetVariable -name "NUGET_EXPERIMENTAL_NETWORK_RETRY_DELAY_MILLISECONDS" -value "1000" fi local toolset_dir="${_InitializeToolset%/*}" @@ -427,6 +434,8 @@ function MSBuild { possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" ) possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.ArcadeLogging.dll" ) possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.Arcade.Sdk.dll" ) + possiblePaths+=( "$toolset_dir/netcoreapp3.1/Microsoft.DotNet.ArcadeLogging.dll" ) + possiblePaths+=( "$toolset_dir/netcoreapp3.1/Microsoft.DotNet.Arcade.Sdk.dll" ) for path in "${possiblePaths[@]}"; do if [[ -f $path ]]; then selectedPath=$path diff --git a/global.json b/global.json index eae10d81..a8e844ca 100644 --- a/global.json +++ b/global.json @@ -1,8 +1,8 @@ { "tools": { - "dotnet": "6.0.100-rc.1.21379.2" + "dotnet": "6.0.100-rc.1.21430.12" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21420.4" + "Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.21453.2" } } From 0dbfbe2a408cce8438298ded794166353c860bcb Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 13 Sep 2021 12:38:06 +0000 Subject: [PATCH 149/161] Update dependencies from https://github.com/dotnet/arcade build 20210909.5 (#746) [main] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 2d3cac33..e8956bf5 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 0ec8dbacaefa7ccdb4bc96e120e07dc60c6eda98 + fa4a48044d33222537e6dbd000f8a2adaa7a15c7 https://github.com/dotnet/source-build-reference-packages diff --git a/global.json b/global.json index a8e844ca..9f37a8fd 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "6.0.100-rc.1.21430.12" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.21453.2" + "Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.21459.5" } } From fd48bde3c9769842b41fe696f15a16b1008bae18 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 20 Sep 2021 12:39:08 +0000 Subject: [PATCH 150/161] Update dependencies from https://github.com/dotnet/arcade build 20210916.4 (#748) [main] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 +- eng/common/cross/build-rootfs.sh | 40 ++------ eng/common/cross/toolchain.cmake | 9 +- ...nd-native-compiler.sh => init-compiler.sh} | 98 ++++++++++--------- global.json | 2 +- 5 files changed, 69 insertions(+), 84 deletions(-) rename eng/common/native/{find-native-compiler.sh => init-compiler.sh} (52%) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index e8956bf5..e49aea12 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - fa4a48044d33222537e6dbd000f8a2adaa7a15c7 + 734d875ba2abfb5c8f5f52368e7d4711ee366e31 https://github.com/dotnet/source-build-reference-packages diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 5c05b39f..6fa2c8aa 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -6,7 +6,7 @@ usage() { echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [--skipunmount] --rootfsdir ]" echo "BuildArch can be: arm(default), armel, arm64, x86" - echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine, alpine3.9 or alpine3.13. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." + echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine, alpine3.13 or alpine3.14. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." echo " for FreeBSD can be: freebsd11, freebsd12, freebsd13" echo " for illumos can be: illumos." echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FreeBSD" @@ -32,9 +32,9 @@ __UbuntuPackages="build-essential" __AlpinePackages="alpine-base" __AlpinePackages+=" build-base" __AlpinePackages+=" linux-headers" -__AlpinePackagesEdgeCommunity=" lldb-dev" -__AlpinePackagesEdgeMain+=" python3" -__AlpinePackagesEdgeMain+=" libedit" +__AlpinePackages+=" lldb-dev" +__AlpinePackages+=" python3" +__AlpinePackages+=" libedit" # symlinks fixer __UbuntuPackages+=" symlinks" @@ -185,23 +185,17 @@ while :; do __UbuntuRepo= __Tizen=tizen ;; - alpine|alpine3.9) + alpine|alpine3.13) __CodeName=alpine __UbuntuRepo= - __AlpineVersion=3.9 - __AlpinePackagesEdgeMain+=" llvm11-libs" - __AlpinePackagesEdgeMain+=" clang-libs" + __AlpineVersion=3.13 + __AlpinePackages+=" llvm10-libs" ;; - alpine3.13) + alpine3.14) __CodeName=alpine __UbuntuRepo= - __AlpineVersion=3.13 - # Alpine 3.13 has all the packages we need in the 3.13 repository - __AlpinePackages+=$__AlpinePackagesEdgeCommunity - __AlpinePackagesEdgeCommunity= - __AlpinePackages+=$__AlpinePackagesEdgeMain - __AlpinePackagesEdgeMain= - __AlpinePackages+=" llvm10-libs" + __AlpineVersion=3.14 + __AlpinePackages+=" llvm11-libs" ;; freebsd11) __FreeBSDBase="11.3-RELEASE" @@ -279,20 +273,6 @@ if [[ "$__CodeName" == "alpine" ]]; then -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ add $__AlpinePackages - if [[ -n "$__AlpinePackagesEdgeMain" ]]; then - $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \ - -X http://dl-cdn.alpinelinux.org/alpine/edge/main \ - -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ - add $__AlpinePackagesEdgeMain - fi - - if [[ -n "$__AlpinePackagesEdgeCommunity" ]]; then - $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \ - -X http://dl-cdn.alpinelinux.org/alpine/edge/community \ - -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ - add $__AlpinePackagesEdgeCommunity - fi - rm -r $__ApkToolsDir elif [[ "$__CodeName" == "freebsd" ]]; then mkdir -p $__RootfsDir/usr/local/etc diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index fc11001a..6501c3a9 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -44,7 +44,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "x86") set(TOOLCHAIN "i686-linux-gnu") elseif (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") set(CMAKE_SYSTEM_PROCESSOR "x86_64") - set(triple "x86_64-unknown-freebsd11") + set(triple "x86_64-unknown-freebsd12") elseif (ILLUMOS) set(CMAKE_SYSTEM_PROCESSOR "x86_64") set(TOOLCHAIN "x86_64-illumos") @@ -91,6 +91,9 @@ elseif(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") set(CMAKE_CXX_COMPILER_TARGET ${triple}) set(CMAKE_ASM_COMPILER_TARGET ${triple}) set(CMAKE_SYSROOT "${CROSS_ROOTFS}") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=lld") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=lld") + set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fuse-ld=lld") elseif(ILLUMOS) set(CMAKE_SYSROOT "${CROSS_ROOTFS}") @@ -138,8 +141,8 @@ function(add_toolchain_linker_flag Flag) if (NOT Config STREQUAL "") set(CONFIG_SUFFIX "_${Config}") endif() - set("CMAKE_EXE_LINKER_FLAGS${CONFIG_SUFFIX}" "${CMAKE_EXE_LINKER_FLAGS${CONFIG_SUFFIX}} ${Flag}" PARENT_SCOPE) - set("CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}" "${CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}} ${Flag}" PARENT_SCOPE) + set("CMAKE_EXE_LINKER_FLAGS${CONFIG_SUFFIX}_INIT" "${CMAKE_EXE_LINKER_FLAGS${CONFIG_SUFFIX}_INIT} ${Flag}" PARENT_SCOPE) + set("CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}_INIT" "${CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}_INIT} ${Flag}" PARENT_SCOPE) endfunction() if(CMAKE_SYSTEM_NAME STREQUAL "Linux") diff --git a/eng/common/native/find-native-compiler.sh b/eng/common/native/init-compiler.sh similarity index 52% rename from eng/common/native/find-native-compiler.sh rename to eng/common/native/init-compiler.sh index aed19d07..1daadf32 100644 --- a/eng/common/native/find-native-compiler.sh +++ b/eng/common/native/init-compiler.sh @@ -1,39 +1,32 @@ #!/usr/bin/env bash # -# This file locates the native compiler with the given name and version and sets the environment variables to locate it. +# This file detects the C/C++ compiler and exports it to the CC/CXX environment variables # -source="${BASH_SOURCE[0]}" - -# resolve $SOURCE until the file is no longer a symlink -while [[ -h $source ]]; do - scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - source="$(readlink "$source")" - - # if $source was a relative symlink, we need to resolve it relative to the path where the - # symlink file was located - [[ $source != /* ]] && source="$scriptroot/$source" -done -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - -if [ $# -lt 0 ] -then +if [[ "$#" -lt 2 ]]; then echo "Usage..." - echo "find-native-compiler.sh " + echo "init-compiler.sh " + echo "Specify the target architecture." echo "Specify the name of compiler (clang or gcc)." echo "Specify the major version of compiler." echo "Specify the minor version of compiler." exit 1 fi -. $scriptroot/../pipeline-logging-functions.sh +. "$( cd -P "$( dirname "$0" )" && pwd )"/../pipeline-logging-functions.sh -compiler="$1" +build_arch="$1" +compiler="$2" cxxCompiler="$compiler++" -majorVersion="$2" -minorVersion="$3" +majorVersion="$3" +minorVersion="$4" -if [ "$compiler" = "gcc" ]; then cxxCompiler="g++"; fi +# clear the existing CC and CXX from environment +CC= +CXX= +LDFLAGS= + +if [[ "$compiler" == "gcc" ]]; then cxxCompiler="g++"; fi check_version_exists() { desired_version=-1 @@ -50,38 +43,38 @@ check_version_exists() { echo "$desired_version" } -if [ -z "$CLR_CC" ]; then +if [[ -z "$CLR_CC" ]]; then # Set default versions - if [ -z "$majorVersion" ]; then + if [[ -z "$majorVersion" ]]; then # note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero. - if [ "$compiler" = "clang" ]; then versions=( 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5 ) - elif [ "$compiler" = "gcc" ]; then versions=( 9 8 7 6 5 4.9 ); fi + if [[ "$compiler" == "clang" ]]; then versions=( 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5 ) + elif [[ "$compiler" == "gcc" ]]; then versions=( 11 10 9 8 7 6 5 4.9 ); fi for version in "${versions[@]}"; do parts=(${version//./ }) desired_version="$(check_version_exists "${parts[0]}" "${parts[1]}")" - if [ "$desired_version" != "-1" ]; then majorVersion="${parts[0]}"; break; fi + if [[ "$desired_version" != "-1" ]]; then majorVersion="${parts[0]}"; break; fi done - if [ -z "$majorVersion" ]; then + if [[ -z "$majorVersion" ]]; then if command -v "$compiler" > /dev/null; then - if [ "$(uname)" != "Darwin" ]; then + if [[ "$(uname)" != "Darwin" ]]; then Write-PipelineTelemetryError -category "Build" -type "warning" "Specific version of $compiler not found, falling back to use the one in PATH." fi - export CC="$(command -v "$compiler")" - export CXX="$(command -v "$cxxCompiler")" + CC="$(command -v "$compiler")" + CXX="$(command -v "$cxxCompiler")" else Write-PipelineTelemetryError -category "Build" "No usable version of $compiler found." exit 1 fi else - if [ "$compiler" = "clang" ] && [ "$majorVersion" -lt 5 ]; then - if [ "$build_arch" = "arm" ] || [ "$build_arch" = "armel" ]; then + if [[ "$compiler" == "clang" && "$majorVersion" -lt 5 ]]; then + if [[ "$build_arch" == "arm" || "$build_arch" == "armel" ]]; then if command -v "$compiler" > /dev/null; then Write-PipelineTelemetryError -category "Build" -type "warning" "Found clang version $majorVersion which is not supported on arm/armel architectures, falling back to use clang from PATH." - export CC="$(command -v "$compiler")" - export CXX="$(command -v "$cxxCompiler")" + CC="$(command -v "$compiler")" + CXX="$(command -v "$cxxCompiler")" else Write-PipelineTelemetryError -category "Build" "Found clang version $majorVersion which is not supported on arm/armel architectures, and there is no clang in PATH." exit 1 @@ -91,31 +84,40 @@ if [ -z "$CLR_CC" ]; then fi else desired_version="$(check_version_exists "$majorVersion" "$minorVersion")" - if [ "$desired_version" = "-1" ]; then + if [[ "$desired_version" == "-1" ]]; then Write-PipelineTelemetryError -category "Build" "Could not find specific version of $compiler: $majorVersion $minorVersion." exit 1 fi fi - if [ -z "$CC" ]; then - export CC="$(command -v "$compiler$desired_version")" - export CXX="$(command -v "$cxxCompiler$desired_version")" - if [ -z "$CXX" ]; then export CXX="$(command -v "$cxxCompiler")"; fi + if [[ -z "$CC" ]]; then + CC="$(command -v "$compiler$desired_version")" + CXX="$(command -v "$cxxCompiler$desired_version")" + if [[ -z "$CXX" ]]; then CXX="$(command -v "$cxxCompiler")"; fi fi else - if [ ! -f "$CLR_CC" ]; then + if [[ ! -f "$CLR_CC" ]]; then Write-PipelineTelemetryError -category "Build" "CLR_CC is set but path '$CLR_CC' does not exist" exit 1 fi - export CC="$CLR_CC" - export CXX="$CLR_CXX" + CC="$CLR_CC" + CXX="$CLR_CXX" fi -if [ -z "$CC" ]; then - Write-PipelineTelemetryError -category "Build" "Unable to find $compiler." +if [[ -z "$CC" ]]; then + Write-PipelineTelemetryError -category "Build" "Unable to find $compiler." exit 1 fi -export CCC_CC="$CC" -export CCC_CXX="$CXX" -export SCAN_BUILD_COMMAND="$(command -v "scan-build$desired_version")" +if [[ "$compiler" == "clang" ]]; then + if command -v "lld$desired_version" > /dev/null; then + # Only lld version >= 9 can be considered stable + if [[ "$majorVersion" -ge 9 ]]; then + LDFLAGS="-fuse-ld=lld" + fi + fi +fi + +SCAN_BUILD_COMMAND="$(command -v "scan-build$desired_version")" + +export CC CXX LDFLAGS SCAN_BUILD_COMMAND diff --git a/global.json b/global.json index 9f37a8fd..b5dceecd 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "6.0.100-rc.1.21430.12" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.21459.5" + "Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.21466.4" } } From 496d52c4498aee4e69173a1cb10669f8a268be73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Matou=C5=A1ek?= Date: Thu, 23 Sep 2021 16:57:41 -0700 Subject: [PATCH 151/161] Update Readme (#736) --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index 13d0163e..931e2fa9 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,19 @@ For projects hosted by [GitLab](https://gitlab.com) reference [Microsoft.SourceL ``` +If your project is hosted by GitLab older than version 12.0 you must specify `SourceLinkGitLabHost` item group in addition to the package reference: + +```xml + + + +``` + +The item group `SourceLinkGitLabHost` specifies the domain of the GitLab host and the version of GitLab. +The version is important since URL format for accessing files changes with version 12.0. By default Source Link assumes new format (version 12.0+). + +You might also consider using environment variable [`CI_SERVER_VERSION`](https://docs.gitlab.com/ee/ci/variables/predefined_variables.html) (`Version="$(CI_SERVER_VERSION)"`) if available in your build environment. + ### Bitbucket For projects in git repositories hosted on [Bitbucket.org](https://bitbucket.org) or hosted on an on-prem Bitbucket server reference [Microsoft.SourceLink.Bitbucket.Git](https://www.nuget.org/packages/Microsoft.SourceLink.Bitbucket.Git) package: From 76dead04f4f2774e23e70b093a4fd618fc01b7ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Matou=C5=A1ek?= Date: Thu, 23 Sep 2021 17:14:40 -0700 Subject: [PATCH 152/161] Update README.md (#752) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 931e2fa9..1867c86e 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,7 @@ For projects hosted by [GitLab](https://gitlab.com) reference [Microsoft.SourceL ``` +Starting with version 1.1.0-beta-21470-02, Microsoft.SourceLink.GitLab assumes GitLab version 12.0+ by default. If your project is hosted by GitLab older than version 12.0 you must specify `SourceLinkGitLabHost` item group in addition to the package reference: ```xml From 91411fc2c299817f7fdaccebae49cb516ea93bb6 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 27 Sep 2021 12:51:56 +0000 Subject: [PATCH 153/161] Update dependencies from https://github.com/dotnet/arcade build 20210924.2 (#753) [main] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- eng/common/tools.ps1 | 26 ++++++++++++++++++-------- global.json | 2 +- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index e49aea12..e6aa67d0 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 734d875ba2abfb5c8f5f52368e7d4711ee366e31 + 3ea0d860c6973f2cbadc9e895c7ec2cbdaec4ad5 https://github.com/dotnet/source-build-reference-packages diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 02347914..44484289 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -709,14 +709,7 @@ function MSBuild() { Write-PipelineSetVariable -Name 'NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS' -Value '20' } - if ($ci) { - $env:NUGET_ENABLE_EXPERIMENTAL_HTTP_RETRY = 'true' - $env:NUGET_EXPERIMENTAL_MAX_NETWORK_TRY_COUNT = 6 - $env:NUGET_EXPERIMENTAL_NETWORK_RETRY_DELAY_MILLISECONDS = 1000 - Write-PipelineSetVariable -Name 'NUGET_ENABLE_EXPERIMENTAL_HTTP_RETRY' -Value 'true' - Write-PipelineSetVariable -Name 'NUGET_EXPERIMENTAL_MAX_NETWORK_TRY_COUNT' -Value '6' - Write-PipelineSetVariable -Name 'NUGET_EXPERIMENTAL_NETWORK_RETRY_DELAY_MILLISECONDS' -Value '1000' - } + Enable-Nuget-EnhancedRetry $toolsetBuildProject = InitializeToolset $basePath = Split-Path -parent $toolsetBuildProject @@ -764,6 +757,8 @@ function MSBuild-Core() { } } + Enable-Nuget-EnhancedRetry + $buildTool = InitializeBuildTool $cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse /p:ContinuousIntegrationBuild=$ci" @@ -904,3 +899,18 @@ function Try-LogClientIpAddress() Write-Host "Unable to get this machine's effective IP address for logging: $_" } } + +# +# If $ci flag is set, turn on (and log that we did) special environment variables for improved Nuget client retry logic. +# +function Enable-Nuget-EnhancedRetry() { + if ($ci) { + Write-Host "Setting NUGET enhanced retry environment variables" + $env:NUGET_ENABLE_EXPERIMENTAL_HTTP_RETRY = 'true' + $env:NUGET_EXPERIMENTAL_MAX_NETWORK_TRY_COUNT = 6 + $env:NUGET_EXPERIMENTAL_NETWORK_RETRY_DELAY_MILLISECONDS = 1000 + Write-PipelineSetVariable -Name 'NUGET_ENABLE_EXPERIMENTAL_HTTP_RETRY' -Value 'true' + Write-PipelineSetVariable -Name 'NUGET_EXPERIMENTAL_MAX_NETWORK_TRY_COUNT' -Value '6' + Write-PipelineSetVariable -Name 'NUGET_EXPERIMENTAL_NETWORK_RETRY_DELAY_MILLISECONDS' -Value '1000' + } +} diff --git a/global.json b/global.json index b5dceecd..6e1d2bf0 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "6.0.100-rc.1.21430.12" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.21466.4" + "Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.21474.2" } } From 8031e5220baf2acad991e661d8308b783d2acf3e Mon Sep 17 00:00:00 2001 From: Jon Fortescue Date: Thu, 30 Sep 2021 10:12:54 -0700 Subject: [PATCH 154/161] Switch to 1ES R&D pools on main (#754) --- azure-pipelines.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 56424fcd..98844bf5 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -49,11 +49,11 @@ stages: - job: Windows pool: ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: - name: NetCorePublic-Pool - queue: BuildPool.Windows.10.Amd64.VS2017.Open + name: NetCore1ESPool-Public + demands: ImageOverride -equals Build.Windows.10.Amd64.VS2017.Open ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - name: NetCoreInternal-Pool - queue: BuildPool.Windows.10.Amd64.VS2017 + name: NetCore1ESPool-Internal + demands: ImageOverride -equals Build.Windows.10.Amd64.VS2017 variables: # Only enable publishing in official builds - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: @@ -129,11 +129,11 @@ stages: displayName: 'Linux' pool: ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: - name: NetCorePublic-Pool - queue: BuildPool.Ubuntu.1604.Amd64.Open + name: NetCore1ESPool-Public + demands: ImageOverride -equals Build.Ubuntu.1604.Amd64.Open ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - name: NetCoreInternal-Pool - queue: BuildPool.Ubuntu.1604.Amd64 + name: NetCore1ESPool-Internal + demands: ImageOverride -equals Build.Ubuntu.1604.Amd64 strategy: matrix: Debug: From a3b0464ac52e6f435a4323ceabdd9107a13d8283 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 11 Oct 2021 12:33:49 +0000 Subject: [PATCH 155/161] Update dependencies from https://github.com/dotnet/arcade build 20211008.1 (#755) [main] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- eng/common/native/init-compiler.sh | 22 +++++++++++--------- eng/common/templates/steps/send-to-helix.yml | 3 --- global.json | 2 +- 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index e6aa67d0..04028b45 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 3ea0d860c6973f2cbadc9e895c7ec2cbdaec4ad5 + 020ac68b5848016459652ebf4ed965b9800fe426 https://github.com/dotnet/source-build-reference-packages diff --git a/eng/common/native/init-compiler.sh b/eng/common/native/init-compiler.sh index 1daadf32..8c944f30 100644 --- a/eng/common/native/init-compiler.sh +++ b/eng/common/native/init-compiler.sh @@ -3,9 +3,10 @@ # This file detects the C/C++ compiler and exports it to the CC/CXX environment variables # -if [[ "$#" -lt 2 ]]; then +if [[ "$#" -lt 3 ]]; then echo "Usage..." - echo "init-compiler.sh " + echo "init-compiler.sh