From baeece44ba075009c6bfe95891a8c1b3d4571cb3 Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Fri, 16 Jun 2023 23:51:14 +0800 Subject: [PATCH] patch for 1.15.1 (#16385) ### Description Sign Maven Package. ### Motivation and Context --------- Co-authored-by: Guenther Schmuelling --- js/web/package.json | 12 +- .../c-api-noopenmp-packaging-pipelines.yml | 25 ++- .../nuget/templates/test_linux.yml | 16 +- .../azure-pipelines/templates/c-api-cpu.yml | 20 ++- .../flex-downloadPipelineArtifact.yml | 32 ++++ .../templates/mac-cpu-packaging-pipeline.yml | 149 +++++++++++------- .../templates/mac-esrp-dylib.yml | 57 +++++++ 7 files changed, 226 insertions(+), 85 deletions(-) create mode 100644 tools/ci_build/github/azure-pipelines/templates/flex-downloadPipelineArtifact.yml create mode 100644 tools/ci_build/github/azure-pipelines/templates/mac-esrp-dylib.yml diff --git a/js/web/package.json b/js/web/package.json index f1890fe70fc8..0de6a2ef1f66 100644 --- a/js/web/package.json +++ b/js/web/package.json @@ -69,17 +69,17 @@ "exports": { ".": { "node": { - "default": "./dist/ort-web.node.js", - "types": "./types.d.ts" + "types": "./types.d.ts", + "default": "./dist/ort-web.node.js" }, "default": { - "default": "./dist/ort.min.js", - "types": "./types.d.ts" + "types": "./types.d.ts", + "default": "./dist/ort.min.js" } }, "./webgpu": { - "default": "./dist/ort.webgpu.min.js", - "types": "./types.d.ts" + "types": "./types.d.ts", + "default": "./dist/ort.webgpu.min.js" } }, "types": "./types.d.ts", diff --git a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml index 62ab1b251f9c..8d786526aa55 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml @@ -73,8 +73,8 @@ resources: ref: 5eda9aded5462201e6310105728d33016e637ea7 variables: -- name: ReleaseVersionSuffix - value: '' + - name: ReleaseVersionSuffix + value: '' stages: - stage: Setup @@ -235,24 +235,21 @@ stages: submodules: false - template: templates/set-version-number-variables-step.yml - - task: DownloadPipelineArtifact@2 - displayName: 'Download Pipeline Artifact - Win x64' - inputs: - buildType: 'current' + - template: templates/flex-downloadPipelineArtifact.yml + parameters: + stepName: 'Download Pipeline Artifact - Win x64' artifactName: 'drop-onnxruntime-java-win-x64-tensorrt' targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64' - - task: DownloadPipelineArtifact@2 - displayName: 'Download Pipeline Artifact - Linux x64' - inputs: - buildType: 'current' + - template: templates/flex-downloadPipelineArtifact.yml + parameters: + stepName: 'Download Pipeline Artifact - Linux x64' artifactName: 'drop-onnxruntime-java-linux-x64-cuda' targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-linux-x64' - - task: DownloadPipelineArtifact@2 - displayName: 'Download Pipeline Artifact - Linux x64' - inputs: - buildType: 'current' + - template: templates/flex-downloadPipelineArtifact.yml + parameters: + stepName: 'Download Pipeline Artifact - Linux x64' artifactName: 'drop-onnxruntime-java-linux-x64-tensorrt' targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-linux-x64-tensorrt' diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml index ff3ac6ed0a83..1f2a7b446fbe 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml @@ -20,18 +20,18 @@ stages: variables: - name: OnnxRuntimeBuildDirectory value: '$(Build.BinariesDirectory)' - steps: - - task: DownloadPipelineArtifact@0 - displayName: 'Download Signed NuGet' - inputs: + steps: + - template: ../../templates/flex-downloadPipelineArtifact.yml + parameters: + stepName: 'Download Signed NuGet' artifactName: drop-signed-nuget-${{ parameters.ArtifactSuffix }} targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - - task: DownloadPipelineArtifact@0 - displayName: 'Download Linux CustomOp TestData' - inputs: - artifactName: '${{ parameters.NativePackagePrefix }}-linux-x64' + - template: ../../templates/flex-downloadPipelineArtifact.yml + parameters: + stepName: 'Download Linux CustomOp TestData' + artifactName: 'onnxruntime-linux-x64' targetPath: '$(Build.BinariesDirectory)/testdata' - template: get-nuget-package-version-as-variable.yml diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml index a6af75b57ceb..f25a2c935043 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml @@ -38,6 +38,16 @@ parameters: type: string default: 'default' +- name: SpecificArtifact + displayName: Use Specific Artifact + type: boolean + default: false + +- name: BuildId + displayName: Specific Artifact's BuildId + type: string + default: '0' + stages: - template: linux-cpu-packaging-pipeline.yml parameters: @@ -47,6 +57,9 @@ stages: parameters: AllowReleasedOpsetOnly: 1 BuildForAllArchs: true + SpecificArtifact: ${{ parameters.SpecificArtifact }} + BuildId: ${{ parameters.BuildId }} + DoEsrp: ${{ parameters.DoEsrp }} - stage: Android_Java_API_AAR_Packaging_Mobile dependsOn: [] @@ -192,7 +205,7 @@ stages: - stage: Jar_Packaging dependsOn: - Linux_C_API_Packaging_CPU - - MacOS_C_API_Packaging_CPU + - MacOS_C_API_Package_Publish - Windows_Packaging_CPU_x86_${{ parameters.BuildVariant }} - Windows_Packaging_CPU_x64_${{ parameters.BuildVariant }} - Windows_Packaging_CPU_arm_${{ parameters.BuildVariant }} @@ -276,7 +289,7 @@ stages: dependsOn: - Setup - Linux_C_API_Packaging_CPU - - MacOS_C_API_Packaging_CPU + - MacOS_C_API_Package_Publish - Windows_Packaging_CPU_x86_${{ parameters.BuildVariant }} - Windows_Packaging_CPU_x64_${{ parameters.BuildVariant }} - Windows_Packaging_CPU_arm_${{ parameters.BuildVariant }} @@ -512,10 +525,11 @@ stages: - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 displayName: 'Clean Agent Directories' condition: always() + - stage: Nodejs_Packaging_CPU dependsOn: - Linux_C_API_Packaging_CPU - - MacOS_C_API_Packaging_CPU + - MacOS_C_API_Package_Publish - Windows_Packaging_CPU_x64_${{ parameters.BuildVariant }} - Windows_Packaging_CPU_arm64_${{ parameters.BuildVariant }} condition: succeeded() diff --git a/tools/ci_build/github/azure-pipelines/templates/flex-downloadPipelineArtifact.yml b/tools/ci_build/github/azure-pipelines/templates/flex-downloadPipelineArtifact.yml new file mode 100644 index 000000000000..0f4e0553d05b --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/templates/flex-downloadPipelineArtifact.yml @@ -0,0 +1,32 @@ +parameters: + - name: StepName + type: string + + - name: ArtifactName + type: string + + - name: TargetPath + type: string + + - name: SpecificArtifact + type: boolean + default: false + + - name: BuildId + type: string + default: '0' + +steps: + - task: DownloadPipelineArtifact@2 + displayName: ${{ parameters.StepName }}} + inputs: + artifactName: ${{ parameters.ArtifactName}} + targetPath: '${{ parameters.TargetPath }}' + ${{ if eq(parameters.SpecificArtifact, false)}}: + buildType: 'current' + ${{ else }}: + source: 'specific' + project: $(System.TeamProject) + pipeline: $(Build.DefinitionName) + runVersion: 'specific' + buildId: ${{ parameters.BuildId }} diff --git a/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packaging-pipeline.yml index 803cf2ef30bc..a705ae1452e1 100644 --- a/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packaging-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packaging-pipeline.yml @@ -22,78 +22,119 @@ parameters: type: boolean default: false +- name: DoESRP + displayName: Do ESRP + type: boolean + default: false + +# these 2 parameters are used for debugging. +- name: SpecificArtifact + displayName: Use Specific Artifact (Debugging only) + type: boolean + default: false + +- name: BuildId + displayName: Pipeline BuildId, you could find it in the URL + type: string + default: '0' + stages: - stage: MacOS_C_API_Packaging_CPU dependsOn: [] jobs: - - ${{ if eq(parameters.BuildForAllArchs, true) }}: - template: mac-cpu-packing-jobs.yml parameters: MacosArch: 'x86_64' AllowReleasedOpsetOnly: ${{ parameters.AllowReleasedOpsetOnly }} AdditionalBuildFlags: ${{ parameters.AdditionalBuildFlags }} WithCache: ${{ parameters.WithCache }} - - template: mac-cpu-packing-jobs.yml - parameters: - MacosArch: 'arm64' - AllowReleasedOpsetOnly: ${{ parameters.AllowReleasedOpsetOnly }} - AdditionalBuildFlags: ${{ parameters.AdditionalBuildFlags }} - WithCache: ${{ parameters.WithCache }} - - template: mac-cpu-packing-jobs.yml - parameters: - MacosArch: 'universal2' - AllowReleasedOpsetOnly: ${{ parameters.AllowReleasedOpsetOnly }} - AdditionalBuildFlags: ${{ parameters.AdditionalBuildFlags }} - WithCache: ${{ parameters.WithCache }} - - job: MacOS_C_API_Package_Publish_All - dependsOn: - - MacOS_C_API_Packaging_CPU_x86_64 - - MacOS_C_API_Packaging_CPU_arm64 - - MacOS_C_API_Packaging_CPU_universal2 - pool: - vmImage: 'macOS-12' - steps: - - task: DownloadPipelineArtifact@2 - inputs: - artifact: 'onnxruntime-osx-x86_64' - targetPath: '$(Build.ArtifactStagingDirectory)' - - task: DownloadPipelineArtifact@2 - inputs: - artifact: 'onnxruntime-osx-arm64' - targetPath: '$(Build.ArtifactStagingDirectory)' - - task: DownloadPipelineArtifact@2 - inputs: - artifact: 'onnxruntime-osx-universal2' - targetPath: '$(Build.ArtifactStagingDirectory)' - - task: PublishPipelineArtifact@1 - inputs: - targetPath: '$(Build.ArtifactStagingDirectory)' - artifact: 'onnxruntime-osx' - condition: 'succeededOrFailed()' - - template: component-governance-component-detection-steps.yml + + - ${{ if eq(parameters.BuildForAllArchs, true) }}: + - template: mac-cpu-packing-jobs.yml parameters: - condition: 'succeeded' - - ${{ if ne(parameters.BuildForAllArchs, true) }}: - - template: mac-cpu-packing-jobs.yml - parameters: - MacosArch: 'x86_64' - AllowReleasedOpsetOnly: ${{ parameters.AllowReleasedOpsetOnly }} - AdditionalBuildFlags: ${{ parameters.AdditionalBuildFlags }} - WithCache: ${{ parameters.WithCache }} - - job: MacOS_C_API_Package_Publish_x86_64 - dependsOn: MacOS_C_API_Packaging_CPU_x86_64 + MacosArch: 'arm64' + AllowReleasedOpsetOnly: ${{ parameters.AllowReleasedOpsetOnly }} + AdditionalBuildFlags: ${{ parameters.AdditionalBuildFlags }} + WithCache: ${{ parameters.WithCache }} + - template: mac-cpu-packing-jobs.yml + parameters: + MacosArch: 'universal2' + AllowReleasedOpsetOnly: ${{ parameters.AllowReleasedOpsetOnly }} + AdditionalBuildFlags: ${{ parameters.AdditionalBuildFlags }} + WithCache: ${{ parameters.WithCache }} + +- stage: MacOS_C_API_Package_Publish + dependsOn: MacOS_C_API_Packaging_CPU + jobs: + - job: MacOS_C_API_Package_Publish pool: vmImage: 'macOS-12' steps: - - task: DownloadPipelineArtifact@2 - inputs: - artifact: 'onnxruntime-osx-x86_64' - targetPath: '$(Build.ArtifactStagingDirectory)' + - checkout: none + - template: flex-downloadPipelineArtifact.yml + parameters: + StepName: 'Download Pipeline onnxruntime-osx-x86_64' + ArtifactName: 'onnxruntime-osx-x86_64' + TargetPath: '$(Build.ArtifactStagingDirectory)' + SpecificArtifact: ${{ parameters.SpecificArtifact }} + BuildId: ${{ parameters.BuildId }} + + - ${{ if eq(parameters.BuildForAllArchs, true) }}: + - template: flex-downloadPipelineArtifact.yml + parameters: + StepName: 'Download Pipeline onnxruntime-osx-arm64' + ArtifactName: 'onnxruntime-osx-arm64' + TargetPath: '$(Build.ArtifactStagingDirectory)' + SpecificArtifact: ${{ parameters.SpecificArtifact }} + BuildId: ${{ parameters.BuildId }} + - template: flex-downloadPipelineArtifact.yml + parameters: + StepName: 'Download Pipeline onnxruntime-osx-universal2' + ArtifactName: 'onnxruntime-osx-universal2' + TargetPath: '$(Build.ArtifactStagingDirectory)' + SpecificArtifact: ${{ parameters.SpecificArtifact }} + BuildId: ${{ parameters.BuildId }} + + - ${{ if eq(parameters.DoESRP, true)}}: + - script: | + pushd '$(Build.ArtifactStagingDirectory)' + find . '*.tgz' -exec tar -zxvf {} \; + rm -f *.tgz; + find . -type d -name 'onnxruntime-osx-*' -exec zip -FSr --symlinks {}.zip {} \; + find . -type d -name 'onnxruntime-osx-*' -exec rm -rf {} \; + ls -l + popd + displayName: tgz to zip + - template: mac-esrp-dylib.yml + parameters: + FolderPath: '$(Build.ArtifactStagingDirectory)' + DisplayName: 'ESRP - Sign Mac' + DoEsrp: true + Pattern: '*.zip' + - script: | + pushd '$(Build.ArtifactStagingDirectory)' + find . '*.zip' -exec unzip {} \; + rm -f *.zip; + find . -type d -name 'onnxruntime-osx-*' -exec tar -czf {}.tgz {} \; + find . -type d -name 'onnxruntime-osx-*' -exec rm -rf {} \; + ls -l + popd + displayName: zip to tgz + - bash: | + set -ex + mkdir -p $(Agent.TempDirectory)/macpackage + find $(Build.ArtifactStagingDirectory) -name "*.tgz" -exec tar -zxvf {} -C $(Agent.TempDirectory)/macpackage \; + find $(Agent.TempDirectory)/macpackage -name "*.dylib" -exec codesign -dvvv {} \; + find $(Agent.TempDirectory)/macpackage -name "*.dylib" -exec ls -l {} \; + rm -rf $(Agent.TempDirectory)/macpackage + displayName: 'Verify code signing' + - task: PublishPipelineArtifact@1 inputs: targetPath: '$(Build.ArtifactStagingDirectory)' artifact: 'onnxruntime-osx' condition: 'succeededOrFailed()' + - template: component-governance-component-detection-steps.yml parameters: - condition: 'succeeded' \ No newline at end of file + condition: 'succeeded' diff --git a/tools/ci_build/github/azure-pipelines/templates/mac-esrp-dylib.yml b/tools/ci_build/github/azure-pipelines/templates/mac-esrp-dylib.yml new file mode 100644 index 000000000000..a9a0d87a30e3 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/templates/mac-esrp-dylib.yml @@ -0,0 +1,57 @@ +parameters: +- name: DoEsrp + type: boolean + default: true + +- name: FolderPath + type: string + default: '' + +- name: DisplayName + type: string + default: '' + +- name: Pattern + type: string + default: '*.zip' + +steps: +- task: EsrpCodeSigning@3 + displayName: ${{ parameters.DisplayName }} + condition: and(succeeded(), eq('${{ parameters.DoEsrp }}', true)) + inputs: + ConnectedServiceName: 'OnnxRuntime CodeSign 20190817' + FolderPath: ${{ parameters.FolderPath }} + Pattern: ${{ parameters.Pattern }} + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "keyCode": "CP-401337-Apple", + "operationSetCode": "MacAppDeveloperSign", + "parameters": [ + { + "parameterName": "OpusName", + "parameterValue": "Microsoft" + }, + { + "parameterName": "OpusInfo", + "parameterValue": "http://www.microsoft.com" + }, + { + "parameterName": "PageHash", + "parameterValue": "/NPH" + }, + { + "parameterName": "FileDigest", + "parameterValue": "/fd sha256" + }, + { + "parameterName": "TimeStamp", + "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" + } + ], + "toolName": "sign", + "toolVersion": "1.0" + } + ]