From 52d06bf332b38a51b337f2f59f8cfb405116a542 Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Thu, 21 Sep 2023 17:35:13 +0800 Subject: [PATCH 01/25] specify the onnxruntime explicitly --- .../job_templates/olive-build-template.yaml | 14 +++++++++++--- .azure_pipelines/olive-ci.yaml | 3 +++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.azure_pipelines/job_templates/olive-build-template.yaml b/.azure_pipelines/job_templates/olive-build-template.yaml index 2b6c440c2..4b03c3784 100644 --- a/.azure_pipelines/job_templates/olive-build-template.yaml +++ b/.azure_pipelines/job_templates/olive-build-template.yaml @@ -4,6 +4,7 @@ parameters: name: '' pool: '' device: 'cpu' + onnxruntime: 'onnxruntime' jobs: - template: olive-test-template.yaml @@ -40,14 +41,21 @@ jobs: versionSpec: 3.8 displayName: Use Python 3.8 - - script: make install-olive INSTALL_EXTRAS=[$(device)] + - script: pip install . displayName: Install Olive + - scritp: | + pip install $(onnxruntime) + displayName: Install onnxruntime + - task: AzureCLI@1 inputs: azureSubscription: $(OLIVE_RG_SERVICE_CONNECTION) scriptLocation: 'inlineScript' - inlineScript: make test-examples + inlineScript: | + python -m pip install pytest \ + python -m pip install -r $(Build.SourcesDirectory)/examples/$(exampleFolder)R/requirements.txt \ + python -m pytest -v -s --log-cli-level=WARNING --junitxml=$(Build.SourcesDirectory)/logs/test_examples-TestOlive.xml $(Build.SourcesDirectory)/examples/test/test_$(exampleName).py displayName: Test Examples env: OLIVEWHEELS_STORAGE_CONNECTION_STRING: $(olive-wheels-storage-connection-string) @@ -74,7 +82,7 @@ jobs: testRunTitle: '$(Build.BuildNumber)[$(Agent.JobName)]' displayName: Upload pipeline run test results - - script: make clean WINDOWS=$(WINDOWS) + - script: git clean -dfX condition: always() displayName: Clean remaining artifacts diff --git a/.azure_pipelines/olive-ci.yaml b/.azure_pipelines/olive-ci.yaml index a897bb2f3..d95b2a998 100644 --- a/.azure_pipelines/olive-ci.yaml +++ b/.azure_pipelines/olive-ci.yaml @@ -32,6 +32,7 @@ jobs: name: Linux_CI pool: $(OLIVE_POOL_UBUNTU2004) windows: False + onnxruntime: onnxruntime examples: bert_ptq_cpu: exampleFolder: bert @@ -63,6 +64,7 @@ jobs: name: Windows_CI pool: $(OLIVE_POOL_WIN2019) windows: True + onnxruntime: onnxruntime examples: bert_ptq_cpu: exampleFolder: bert @@ -89,6 +91,7 @@ jobs: pool: $(OLIVE_POOL_UBUNTU2004) windows: False device: gpu + onnxruntime: onnxruntime-gpu examples: bert_cuda_gpu: exampleFolder: bert From 21712db81a2cbb7ad371c8d19280241ceba0b024 Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Thu, 21 Sep 2023 17:40:05 +0800 Subject: [PATCH 02/25] fix typo --- .azure_pipelines/job_templates/olive-build-template.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure_pipelines/job_templates/olive-build-template.yaml b/.azure_pipelines/job_templates/olive-build-template.yaml index 4b03c3784..a657caa45 100644 --- a/.azure_pipelines/job_templates/olive-build-template.yaml +++ b/.azure_pipelines/job_templates/olive-build-template.yaml @@ -44,9 +44,9 @@ jobs: - script: pip install . displayName: Install Olive - - scritp: | + - script: | pip install $(onnxruntime) - displayName: Install onnxruntime + displayName: Install $(onnxruntime) - task: AzureCLI@1 inputs: From 129f48ac7efec1f3b7228197c857e672d6cac889 Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Thu, 21 Sep 2023 17:49:43 +0800 Subject: [PATCH 03/25] fix ci pipeline syntax error --- .azure_pipelines/job_templates/olive-build-template.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure_pipelines/job_templates/olive-build-template.yaml b/.azure_pipelines/job_templates/olive-build-template.yaml index a657caa45..be952c866 100644 --- a/.azure_pipelines/job_templates/olive-build-template.yaml +++ b/.azure_pipelines/job_templates/olive-build-template.yaml @@ -45,8 +45,8 @@ jobs: displayName: Install Olive - script: | - pip install $(onnxruntime) - displayName: Install $(onnxruntime) + pip install ${{ parameters.onnxruntime }} + displayName: Install ${{ parameters.onnxruntime }} - task: AzureCLI@1 inputs: From cf422f80059b9f8554105f6ee3e0195288f66be2 Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Thu, 21 Sep 2023 18:42:20 +0800 Subject: [PATCH 04/25] fix azurecli syntax --- .azure_pipelines/job_templates/olive-build-template.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure_pipelines/job_templates/olive-build-template.yaml b/.azure_pipelines/job_templates/olive-build-template.yaml index be952c866..8eef59e7e 100644 --- a/.azure_pipelines/job_templates/olive-build-template.yaml +++ b/.azure_pipelines/job_templates/olive-build-template.yaml @@ -53,8 +53,8 @@ jobs: azureSubscription: $(OLIVE_RG_SERVICE_CONNECTION) scriptLocation: 'inlineScript' inlineScript: | - python -m pip install pytest \ - python -m pip install -r $(Build.SourcesDirectory)/examples/$(exampleFolder)R/requirements.txt \ + python -m pip install pytest + python -m pip install -r $(Build.SourcesDirectory)/examples/$(exampleFolder)R/requirements.txt python -m pytest -v -s --log-cli-level=WARNING --junitxml=$(Build.SourcesDirectory)/logs/test_examples-TestOlive.xml $(Build.SourcesDirectory)/examples/test/test_$(exampleName).py displayName: Test Examples env: From ecfc002c747593c828880f1989c76c8f5359698a Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Thu, 21 Sep 2023 18:53:01 +0800 Subject: [PATCH 05/25] fix typo --- .azure_pipelines/job_templates/olive-build-template.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure_pipelines/job_templates/olive-build-template.yaml b/.azure_pipelines/job_templates/olive-build-template.yaml index 8eef59e7e..cfed10377 100644 --- a/.azure_pipelines/job_templates/olive-build-template.yaml +++ b/.azure_pipelines/job_templates/olive-build-template.yaml @@ -54,7 +54,7 @@ jobs: scriptLocation: 'inlineScript' inlineScript: | python -m pip install pytest - python -m pip install -r $(Build.SourcesDirectory)/examples/$(exampleFolder)R/requirements.txt + python -m pip install -r $(Build.SourcesDirectory)/examples/$(exampleFolder)/requirements.txt python -m pytest -v -s --log-cli-level=WARNING --junitxml=$(Build.SourcesDirectory)/logs/test_examples-TestOlive.xml $(Build.SourcesDirectory)/examples/test/test_$(exampleName).py displayName: Test Examples env: From 455da93fcb56a32d65cb5a605c33aa715a3675a9 Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Thu, 21 Sep 2023 19:29:20 +0800 Subject: [PATCH 06/25] install ORT for unit test --- .../job_templates/olive-build-doc-template.yaml | 2 +- .../job_templates/olive-build-template.yaml | 4 ++-- .../job_templates/olive-test-template.yaml | 15 ++++++++++++--- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/.azure_pipelines/job_templates/olive-build-doc-template.yaml b/.azure_pipelines/job_templates/olive-build-doc-template.yaml index df6249201..adb17a22d 100644 --- a/.azure_pipelines/job_templates/olive-build-doc-template.yaml +++ b/.azure_pipelines/job_templates/olive-build-doc-template.yaml @@ -27,7 +27,7 @@ jobs: displayName: Checkout release branch condition: ne('${{parameters.doc_version}}', 'latest') - - script: make install-olive INSTALL_EXTRAS=[cpu] + - script: pip install .[cpu] displayName: Install Olive - script: | diff --git a/.azure_pipelines/job_templates/olive-build-template.yaml b/.azure_pipelines/job_templates/olive-build-template.yaml index cfed10377..59182aa10 100644 --- a/.azure_pipelines/job_templates/olive-build-template.yaml +++ b/.azure_pipelines/job_templates/olive-build-template.yaml @@ -14,6 +14,7 @@ jobs: device: ${{parameters.device}} WINDOWS: ${{parameters.windows}} test_type: 'unit_test' + onnxruntime: ${{parameters.onnxruntime}} - template: olive-test-template.yaml parameters: @@ -22,6 +23,7 @@ jobs: device: ${{parameters.device}} WINDOWS: ${{parameters.windows}} test_type: 'integ_test' + onnxruntime: ${{parameters.onnxruntime}} - job: ${{parameters.name}}_Test_Examples timeoutInMinutes: 300 @@ -65,8 +67,6 @@ jobs: AZURE_TENANT_ID: $(azure-tenant-id) AZURE_CLIENT_ID: $(olive-rg-sp-id) AZURE_CLIENT_SECRET: $(olive-rg-sp-secret) - EXAMPLE_FOLDER: $(exampleFolder) - EXAMPLE_NAME: $(exampleName) - task: ComponentGovernanceComponentDetection@0 inputs: diff --git a/.azure_pipelines/job_templates/olive-test-template.yaml b/.azure_pipelines/job_templates/olive-test-template.yaml index 40d17e137..2e110b46c 100644 --- a/.azure_pipelines/job_templates/olive-test-template.yaml +++ b/.azure_pipelines/job_templates/olive-test-template.yaml @@ -4,6 +4,7 @@ parameters: test_type: '' device: 'cpu' python_version: '3.8' + onnxruntime: 'onnxruntime' jobs: - ${{ if eq(parameters.device, 'cpu') }}: @@ -29,14 +30,22 @@ jobs: condition: and(eq(variables.WINDOWS, 'False'), eq(variables.testType, 'integ_test')) continueOnError: true - - script: make install-olive INSTALL_EXTRAS=[$(device)] + - script: | + pip install . + pip install ${{ parameters.onnxruntime }} displayName: Install Olive - task: AzureCLI@1 inputs: azureSubscription: $(OLIVE_RG_SERVICE_CONNECTION) scriptLocation: 'inlineScript' - inlineScript: make $(testType) + inlineScript: | + python -m pip install pytest + python -m pip install -r $(Build.SourcesDirectory)/test/requirements-test.txt + + coverage run --source=$(Build.SourcesDirectory)/olive -m pytest -v -s --log-cli-level=WARNING --junitxml=$(Build.SourcesDirectory)/logs/test-TestOlive.xml $(Build.SourcesDirectory)/test/$(testType) + coverage xml + displayName: Test Olive env: OLIVEWHEELS_STORAGE_CONNECTION_STRING: $(olive-wheels-storage-connection-string) @@ -72,6 +81,6 @@ jobs: codeCoverageTool: 'Cobertura' summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml' - - script: make clean WINDOWS=$(WINDOWS) + - script: git clean -dfX condition: always() displayName: Clean remaining artifacts From cc88abc4cc05703eeb505f85575ae7b2a10e1406 Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Thu, 21 Sep 2023 21:37:38 +0800 Subject: [PATCH 07/25] fix windows multiple EP --- .../job_templates/olive-test-template.yaml | 60 +++++++++++++------ 1 file changed, 41 insertions(+), 19 deletions(-) diff --git a/.azure_pipelines/job_templates/olive-test-template.yaml b/.azure_pipelines/job_templates/olive-test-template.yaml index 2e110b46c..b255c072c 100644 --- a/.azure_pipelines/job_templates/olive-test-template.yaml +++ b/.azure_pipelines/job_templates/olive-test-template.yaml @@ -34,27 +34,49 @@ jobs: pip install . pip install ${{ parameters.onnxruntime }} displayName: Install Olive + - ${{ if and(eq(variables.WINDOWS, 'True'), eq(variables.testType, 'multiple_ep')) }}: + - task: AzureCLI@1 + inputs: + azureSubscription: $(OLIVE_RG_SERVICE_CONNECTION) + scriptLocation: 'inlineScript' + inlineScript: | + python -m pip install pytest + curl --output openvino_toolkit.zip https://storage.openvinotoolkit.org/repositories/openvino/packages/2023.0.1/windows/w_openvino_toolkit_windows_2023.0.1.11005.fa1c41994f3_x86_64.zip + 7z x openvino_toolkit.zip + w_openvino_toolkit_windows_2023.0.1.11005.fa1c41994f3_x86_64\\setupvars.bat + python -m pip install numpy psutil coverage protobuf==3.20.3 + coverage run --source=$(Build.SourcesDirectory)/olive -m pytest -v -s --log-cli-level=WARNING --junitxml=$(Build.SourcesDirectory)/logs/test-TestOlive.xml $(Build.SourcesDirectory)/test/$(testType) + coverage xml + displayName: Test Olive + env: + OLIVEWHEELS_STORAGE_CONNECTION_STRING: $(olive-wheels-storage-connection-string) + WORKSPACE_SUBSCRIPTION_ID: $(workspace-subscription-id) + WORKSPACE_RESOURCE_GROUP: $(workspace-resource-group) + WORKSPACE_NAME: $(workspace-name) + AZURE_TENANT_ID: $(azure-tenant-id) + AZURE_CLIENT_ID: $(olive-rg-sp-id) + AZURE_CLIENT_SECRET: $(olive-rg-sp-secret) + - ${{ else }}: + - task: AzureCLI@1 + inputs: + azureSubscription: $(OLIVE_RG_SERVICE_CONNECTION) + scriptLocation: 'inlineScript' + inlineScript: | + python -m pip install pytest + python -m pip install -r $(Build.SourcesDirectory)/test/requirements-test.txt - - task: AzureCLI@1 - inputs: - azureSubscription: $(OLIVE_RG_SERVICE_CONNECTION) - scriptLocation: 'inlineScript' - inlineScript: | - python -m pip install pytest - python -m pip install -r $(Build.SourcesDirectory)/test/requirements-test.txt - - coverage run --source=$(Build.SourcesDirectory)/olive -m pytest -v -s --log-cli-level=WARNING --junitxml=$(Build.SourcesDirectory)/logs/test-TestOlive.xml $(Build.SourcesDirectory)/test/$(testType) - coverage xml + coverage run --source=$(Build.SourcesDirectory)/olive -m pytest -v -s --log-cli-level=WARNING --junitxml=$(Build.SourcesDirectory)/logs/test-TestOlive.xml $(Build.SourcesDirectory)/test/$(testType) + coverage xml - displayName: Test Olive - env: - OLIVEWHEELS_STORAGE_CONNECTION_STRING: $(olive-wheels-storage-connection-string) - WORKSPACE_SUBSCRIPTION_ID: $(workspace-subscription-id) - WORKSPACE_RESOURCE_GROUP: $(workspace-resource-group) - WORKSPACE_NAME: $(workspace-name) - AZURE_TENANT_ID: $(azure-tenant-id) - AZURE_CLIENT_ID: $(olive-rg-sp-id) - AZURE_CLIENT_SECRET: $(olive-rg-sp-secret) + displayName: Test Olive + env: + OLIVEWHEELS_STORAGE_CONNECTION_STRING: $(olive-wheels-storage-connection-string) + WORKSPACE_SUBSCRIPTION_ID: $(workspace-subscription-id) + WORKSPACE_RESOURCE_GROUP: $(workspace-resource-group) + WORKSPACE_NAME: $(workspace-name) + AZURE_TENANT_ID: $(azure-tenant-id) + AZURE_CLIENT_ID: $(olive-rg-sp-id) + AZURE_CLIENT_SECRET: $(olive-rg-sp-secret) - task: CredScan@3 displayName: 'Run CredScan' From 0993db588dafbfdb9ee0ea7128c90002fc728ade Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Thu, 21 Sep 2023 21:52:26 +0800 Subject: [PATCH 08/25] add call for batch --- .../job_templates/olive-test-template.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.azure_pipelines/job_templates/olive-test-template.yaml b/.azure_pipelines/job_templates/olive-test-template.yaml index b255c072c..ee65bc154 100644 --- a/.azure_pipelines/job_templates/olive-test-template.yaml +++ b/.azure_pipelines/job_templates/olive-test-template.yaml @@ -40,13 +40,13 @@ jobs: azureSubscription: $(OLIVE_RG_SERVICE_CONNECTION) scriptLocation: 'inlineScript' inlineScript: | - python -m pip install pytest - curl --output openvino_toolkit.zip https://storage.openvinotoolkit.org/repositories/openvino/packages/2023.0.1/windows/w_openvino_toolkit_windows_2023.0.1.11005.fa1c41994f3_x86_64.zip - 7z x openvino_toolkit.zip - w_openvino_toolkit_windows_2023.0.1.11005.fa1c41994f3_x86_64\\setupvars.bat - python -m pip install numpy psutil coverage protobuf==3.20.3 - coverage run --source=$(Build.SourcesDirectory)/olive -m pytest -v -s --log-cli-level=WARNING --junitxml=$(Build.SourcesDirectory)/logs/test-TestOlive.xml $(Build.SourcesDirectory)/test/$(testType) - coverage xml + call python -m pip install pytest + call curl --output openvino_toolkit.zip https://storage.openvinotoolkit.org/repositories/openvino/packages/2023.0.1/windows/w_openvino_toolkit_windows_2023.0.1.11005.fa1c41994f3_x86_64.zip + call 7z x openvino_toolkit.zip + call w_openvino_toolkit_windows_2023.0.1.11005.fa1c41994f3_x86_64\\setupvars.bat + call python -m pip install numpy psutil coverage protobuf==3.20.3 + call coverage run --source=$(Build.SourcesDirectory)/olive -m pytest -v -s --log-cli-level=WARNING --junitxml=$(Build.SourcesDirectory)/logs/test-TestOlive.xml $(Build.SourcesDirectory)/test/$(testType) + call coverage xml displayName: Test Olive env: OLIVEWHEELS_STORAGE_CONNECTION_STRING: $(olive-wheels-storage-connection-string) From 38f67a8851a6dfaf5239b2beef76c039fa8035e2 Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Fri, 22 Sep 2023 08:02:43 +0800 Subject: [PATCH 09/25] add ort 1.15.1 test suites --- .../job_templates/olive-build-template.yaml | 8 +-- .azure_pipelines/olive-ci.yaml | 70 +++++++++++++++++++ 2 files changed, 74 insertions(+), 4 deletions(-) diff --git a/.azure_pipelines/job_templates/olive-build-template.yaml b/.azure_pipelines/job_templates/olive-build-template.yaml index 59182aa10..3ddf21a27 100644 --- a/.azure_pipelines/job_templates/olive-build-template.yaml +++ b/.azure_pipelines/job_templates/olive-build-template.yaml @@ -9,7 +9,7 @@ parameters: jobs: - template: olive-test-template.yaml parameters: - name: ${{parameters.name}}_Unit_Test_Olive + name: ${{parameters.name}}_${{parameters.onnxruntime}}_Unit_Test_Olive pool: ${{parameters.pool}} device: ${{parameters.device}} WINDOWS: ${{parameters.windows}} @@ -18,14 +18,14 @@ jobs: - template: olive-test-template.yaml parameters: - name: ${{parameters.name}}_Integration_Test_Olive + name: ${{parameters.name}}_${{parameters.onnxruntime}}_Integration_Test_Olive pool: ${{parameters.pool}} device: ${{parameters.device}} WINDOWS: ${{parameters.windows}} test_type: 'integ_test' onnxruntime: ${{parameters.onnxruntime}} -- job: ${{parameters.name}}_Test_Examples +- job: ${{parameters.name}}_${{parameters.onnxruntime}}_Test_Examples timeoutInMinutes: 300 pool: name: ${{ parameters.pool}} @@ -88,7 +88,7 @@ jobs: - template: olive-test-template.yaml parameters: - name: ${{parameters.name}}_Multiple_EP_Test_Olive + name: ${{parameters.name}}_${{parameters.onnxruntime}}_Multiple_EP_Test_Olive pool: ${{parameters.pool}} device: ${{parameters.device}} WINDOWS: ${{parameters.windows}} diff --git a/.azure_pipelines/olive-ci.yaml b/.azure_pipelines/olive-ci.yaml index d95b2a998..c1a7a9d4e 100644 --- a/.azure_pipelines/olive-ci.yaml +++ b/.azure_pipelines/olive-ci.yaml @@ -97,6 +97,76 @@ jobs: exampleFolder: bert exampleName: bert_cuda_gpu +- template: job_templates/olive-build-template.yaml + parameters: + name: Linux_CI + pool: $(OLIVE_POOL_UBUNTU2004) + windows: False + onnxruntime: onnxruntime==1.15.1 + examples: + bert_ptq_cpu: + exampleFolder: bert + exampleName: bert_ptq_cpu + bert_ptq_cpu_aml: + exampleFolder: bert + exampleName: bert_ptq_cpu_aml + bert_ptq_cpu_docker: + exampleFolder: bert + exampleName: bert_ptq_cpu_docker + cifar10_openvino_intel_hw: + exampleFolder: cifar10_openvino_intel_hw + exampleName: cifar10_openvino_intel_hw + resnet_ptq_cpu: + exampleFolder: resnet + exampleName: resnet_ptq_cpu + resnet_vitis_ai_ptq_cpu: + exampleFolder: resnet + exampleName: resnet_vitis_ai_ptq_cpu + resnet_qat: + exampleFolder: resnet + exampleName: resnet_qat + whisper: + exampleFolder: whisper + exampleName: whisper + +- template: job_templates/olive-build-template.yaml + parameters: + name: Windows_CI + pool: $(OLIVE_POOL_WIN2019) + windows: True + onnxruntime: onnxruntime==1.15.1 + examples: + bert_ptq_cpu: + exampleFolder: bert + exampleName: bert_ptq_cpu + bert_ptq_cpu_aml: + exampleFolder: bert + exampleName: bert_ptq_cpu_aml + cifar10_openvino_intel_hw: + exampleFolder: cifar10_openvino_intel_hw + exampleName: cifar10_openvino_intel_hw + resnet_ptq_cpu: + exampleFolder: resnet + exampleName: resnet_ptq_cpu + resnet_qat: + exampleFolder: resnet + exampleName: resnet_qat + whisper: + exampleFolder: whisper + exampleName: whisper + +- template: job_templates/olive-build-template.yaml + parameters: + name: Linux_GPU_CI + pool: $(OLIVE_POOL_UBUNTU2004) + windows: False + device: gpu + onnxruntime: onnxruntime-gpu==1.15.1 + examples: + bert_cuda_gpu: + exampleFolder: bert + exampleName: bert_cuda_gpu + - template: job_templates/olive-build-doc-template.yaml parameters: job_name: Test_BuildDocs From feabf3067821e25ee93e09b482246133aa143aca Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Fri, 22 Sep 2023 08:20:22 +0800 Subject: [PATCH 10/25] try to fix the name issue --- .../job_templates/olive-build-template.yaml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.azure_pipelines/job_templates/olive-build-template.yaml b/.azure_pipelines/job_templates/olive-build-template.yaml index 3ddf21a27..629db5307 100644 --- a/.azure_pipelines/job_templates/olive-build-template.yaml +++ b/.azure_pipelines/job_templates/olive-build-template.yaml @@ -6,10 +6,13 @@ parameters: device: 'cpu' onnxruntime: 'onnxruntime' +variables: + onnxruntime: ${{replace(replace(parameters.onnxruntime, '=', '_'), '.', '_'}} + jobs: - template: olive-test-template.yaml parameters: - name: ${{parameters.name}}_${{parameters.onnxruntime}}_Unit_Test_Olive + name: ${{parameters.name}}_$(onnxruntime)_Unit_Test_Olive pool: ${{parameters.pool}} device: ${{parameters.device}} WINDOWS: ${{parameters.windows}} @@ -18,14 +21,14 @@ jobs: - template: olive-test-template.yaml parameters: - name: ${{parameters.name}}_${{parameters.onnxruntime}}_Integration_Test_Olive + name: ${{parameters.name}}_$(onnxruntime)_Integration_Test_Olive pool: ${{parameters.pool}} device: ${{parameters.device}} WINDOWS: ${{parameters.windows}} test_type: 'integ_test' onnxruntime: ${{parameters.onnxruntime}} -- job: ${{parameters.name}}_${{parameters.onnxruntime}}_Test_Examples +- job: ${{parameters.name}}_$(onnxruntime)_Test_Examples timeoutInMinutes: 300 pool: name: ${{ parameters.pool}} @@ -88,7 +91,7 @@ jobs: - template: olive-test-template.yaml parameters: - name: ${{parameters.name}}_${{parameters.onnxruntime}}_Multiple_EP_Test_Olive + name: ${{parameters.name}}_$(onnxruntime)_Multiple_EP_Test_Olive pool: ${{parameters.pool}} device: ${{parameters.device}} WINDOWS: ${{parameters.windows}} From ddd4bbf5e626e05b231fb4027fb00bbd2e073e4e Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Fri, 22 Sep 2023 08:31:05 +0800 Subject: [PATCH 11/25] fix naming issue --- .../job_templates/olive-build-template.yaml | 11 ++++------- .azure_pipelines/olive-ci.yaml | 12 ++++++------ 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/.azure_pipelines/job_templates/olive-build-template.yaml b/.azure_pipelines/job_templates/olive-build-template.yaml index 629db5307..59182aa10 100644 --- a/.azure_pipelines/job_templates/olive-build-template.yaml +++ b/.azure_pipelines/job_templates/olive-build-template.yaml @@ -6,13 +6,10 @@ parameters: device: 'cpu' onnxruntime: 'onnxruntime' -variables: - onnxruntime: ${{replace(replace(parameters.onnxruntime, '=', '_'), '.', '_'}} - jobs: - template: olive-test-template.yaml parameters: - name: ${{parameters.name}}_$(onnxruntime)_Unit_Test_Olive + name: ${{parameters.name}}_Unit_Test_Olive pool: ${{parameters.pool}} device: ${{parameters.device}} WINDOWS: ${{parameters.windows}} @@ -21,14 +18,14 @@ jobs: - template: olive-test-template.yaml parameters: - name: ${{parameters.name}}_$(onnxruntime)_Integration_Test_Olive + name: ${{parameters.name}}_Integration_Test_Olive pool: ${{parameters.pool}} device: ${{parameters.device}} WINDOWS: ${{parameters.windows}} test_type: 'integ_test' onnxruntime: ${{parameters.onnxruntime}} -- job: ${{parameters.name}}_$(onnxruntime)_Test_Examples +- job: ${{parameters.name}}_Test_Examples timeoutInMinutes: 300 pool: name: ${{ parameters.pool}} @@ -91,7 +88,7 @@ jobs: - template: olive-test-template.yaml parameters: - name: ${{parameters.name}}_$(onnxruntime)_Multiple_EP_Test_Olive + name: ${{parameters.name}}_Multiple_EP_Test_Olive pool: ${{parameters.pool}} device: ${{parameters.device}} WINDOWS: ${{parameters.windows}} diff --git a/.azure_pipelines/olive-ci.yaml b/.azure_pipelines/olive-ci.yaml index c1a7a9d4e..b682323ff 100644 --- a/.azure_pipelines/olive-ci.yaml +++ b/.azure_pipelines/olive-ci.yaml @@ -29,7 +29,7 @@ variables: jobs: - template: job_templates/olive-build-template.yaml parameters: - name: Linux_CI + name: Linux_CI_latest_ORT pool: $(OLIVE_POOL_UBUNTU2004) windows: False onnxruntime: onnxruntime @@ -61,7 +61,7 @@ jobs: - template: job_templates/olive-build-template.yaml parameters: - name: Windows_CI + name: Windows_CI_latest_ORT pool: $(OLIVE_POOL_WIN2019) windows: True onnxruntime: onnxruntime @@ -87,7 +87,7 @@ jobs: - template: job_templates/olive-build-template.yaml parameters: - name: Linux_GPU_CI + name: Linux_GPU_CI_latest_ORT pool: $(OLIVE_POOL_UBUNTU2004) windows: False device: gpu @@ -99,7 +99,7 @@ jobs: - template: job_templates/olive-build-template.yaml parameters: - name: Linux_CI + name: Linux_CI_ORT_1_15_1 pool: $(OLIVE_POOL_UBUNTU2004) windows: False onnxruntime: onnxruntime==1.15.1 @@ -131,7 +131,7 @@ jobs: - template: job_templates/olive-build-template.yaml parameters: - name: Windows_CI + name: Windows_CI_ORT_1_15_1 pool: $(OLIVE_POOL_WIN2019) windows: True onnxruntime: onnxruntime==1.15.1 @@ -157,7 +157,7 @@ jobs: - template: job_templates/olive-build-template.yaml parameters: - name: Linux_GPU_CI + name: Linux_GPU_CI_ORT_1_15_1 pool: $(OLIVE_POOL_UBUNTU2004) windows: False device: gpu From 4052ee72ccf3f25320e6101ad35b4561dcb4af84 Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Fri, 22 Sep 2023 11:34:40 +0800 Subject: [PATCH 12/25] add nightly ort support --- .../job_templates/olive-build-template.yaml | 11 ++- .../job_templates/olive-test-template.yaml | 11 ++- .azure_pipelines/olive-ci.yaml | 76 +---------------- .azure_pipelines/olive-ort-nightly.yaml | 82 +++++++++++++++++++ examples/resnet/requirements.txt | 1 - examples/test/test_resnet_ptq_cpu.py | 6 ++ examples/test/test_resnet_vitis_ai_ptq_cpu.py | 4 +- examples/whisper/requirements.txt | 1 - .../vitis_ai/test_vitis_ai_quantization.py | 2 +- 9 files changed, 112 insertions(+), 82 deletions(-) create mode 100644 .azure_pipelines/olive-ort-nightly.yaml diff --git a/.azure_pipelines/job_templates/olive-build-template.yaml b/.azure_pipelines/job_templates/olive-build-template.yaml index 59182aa10..25d3187dd 100644 --- a/.azure_pipelines/job_templates/olive-build-template.yaml +++ b/.azure_pipelines/job_templates/olive-build-template.yaml @@ -46,9 +46,14 @@ jobs: - script: pip install . displayName: Install Olive - - script: | - pip install ${{ parameters.onnxruntime }} - displayName: Install ${{ parameters.onnxruntime }} + - ${{ if startsWith(parameters.onnxruntime, 'ort-nightly') }}: + - script: | + pip install ${{ parameters.onnxruntime }} --index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/ + displayName: Install ${{ parameters.onnxruntime }} + - ${{ else }}: + - script: | + pip install ${{ parameters.onnxruntime }} + displayName: Install ${{ parameters.onnxruntime }} - task: AzureCLI@1 inputs: diff --git a/.azure_pipelines/job_templates/olive-test-template.yaml b/.azure_pipelines/job_templates/olive-test-template.yaml index ee65bc154..a9114438b 100644 --- a/.azure_pipelines/job_templates/olive-test-template.yaml +++ b/.azure_pipelines/job_templates/olive-test-template.yaml @@ -32,8 +32,17 @@ jobs: - script: | pip install . - pip install ${{ parameters.onnxruntime }} displayName: Install Olive + + - ${{ if startsWith(parameters.onnxruntime, 'ort-nightly') }}: + - script: | + pip install ${{ parameters.onnxruntime }} --index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/ + displayName: Install ${{ parameters.onnxruntime }} + - ${{ else }}: + - script: | + pip install ${{ parameters.onnxruntime }} + displayName: Install ${{ parameters.onnxruntime }} + - ${{ if and(eq(variables.WINDOWS, 'True'), eq(variables.testType, 'multiple_ep')) }}: - task: AzureCLI@1 inputs: diff --git a/.azure_pipelines/olive-ci.yaml b/.azure_pipelines/olive-ci.yaml index b682323ff..d95b2a998 100644 --- a/.azure_pipelines/olive-ci.yaml +++ b/.azure_pipelines/olive-ci.yaml @@ -29,7 +29,7 @@ variables: jobs: - template: job_templates/olive-build-template.yaml parameters: - name: Linux_CI_latest_ORT + name: Linux_CI pool: $(OLIVE_POOL_UBUNTU2004) windows: False onnxruntime: onnxruntime @@ -61,7 +61,7 @@ jobs: - template: job_templates/olive-build-template.yaml parameters: - name: Windows_CI_latest_ORT + name: Windows_CI pool: $(OLIVE_POOL_WIN2019) windows: True onnxruntime: onnxruntime @@ -87,7 +87,7 @@ jobs: - template: job_templates/olive-build-template.yaml parameters: - name: Linux_GPU_CI_latest_ORT + name: Linux_GPU_CI pool: $(OLIVE_POOL_UBUNTU2004) windows: False device: gpu @@ -97,76 +97,6 @@ jobs: exampleFolder: bert exampleName: bert_cuda_gpu -- template: job_templates/olive-build-template.yaml - parameters: - name: Linux_CI_ORT_1_15_1 - pool: $(OLIVE_POOL_UBUNTU2004) - windows: False - onnxruntime: onnxruntime==1.15.1 - examples: - bert_ptq_cpu: - exampleFolder: bert - exampleName: bert_ptq_cpu - bert_ptq_cpu_aml: - exampleFolder: bert - exampleName: bert_ptq_cpu_aml - bert_ptq_cpu_docker: - exampleFolder: bert - exampleName: bert_ptq_cpu_docker - cifar10_openvino_intel_hw: - exampleFolder: cifar10_openvino_intel_hw - exampleName: cifar10_openvino_intel_hw - resnet_ptq_cpu: - exampleFolder: resnet - exampleName: resnet_ptq_cpu - resnet_vitis_ai_ptq_cpu: - exampleFolder: resnet - exampleName: resnet_vitis_ai_ptq_cpu - resnet_qat: - exampleFolder: resnet - exampleName: resnet_qat - whisper: - exampleFolder: whisper - exampleName: whisper - -- template: job_templates/olive-build-template.yaml - parameters: - name: Windows_CI_ORT_1_15_1 - pool: $(OLIVE_POOL_WIN2019) - windows: True - onnxruntime: onnxruntime==1.15.1 - examples: - bert_ptq_cpu: - exampleFolder: bert - exampleName: bert_ptq_cpu - bert_ptq_cpu_aml: - exampleFolder: bert - exampleName: bert_ptq_cpu_aml - cifar10_openvino_intel_hw: - exampleFolder: cifar10_openvino_intel_hw - exampleName: cifar10_openvino_intel_hw - resnet_ptq_cpu: - exampleFolder: resnet - exampleName: resnet_ptq_cpu - resnet_qat: - exampleFolder: resnet - exampleName: resnet_qat - whisper: - exampleFolder: whisper - exampleName: whisper - -- template: job_templates/olive-build-template.yaml - parameters: - name: Linux_GPU_CI_ORT_1_15_1 - pool: $(OLIVE_POOL_UBUNTU2004) - windows: False - device: gpu - onnxruntime: onnxruntime-gpu==1.15.1 - examples: - bert_cuda_gpu: - exampleFolder: bert - exampleName: bert_cuda_gpu - - template: job_templates/olive-build-doc-template.yaml parameters: job_name: Test_BuildDocs diff --git a/.azure_pipelines/olive-ort-nightly.yaml b/.azure_pipelines/olive-ort-nightly.yaml new file mode 100644 index 000000000..f92312466 --- /dev/null +++ b/.azure_pipelines/olive-ort-nightly.yaml @@ -0,0 +1,82 @@ +trigger: none + +schedules: +- cron: '0 0 * * *' + displayName: Daily build + branches: + include: + - main + +variables: + ComponentDetection.Timeout: 2400 + +jobs: +- template: job_templates/olive-build-template.yaml + parameters: + name: Linux_CI + pool: $(OLIVE_POOL_UBUNTU2004) + windows: False + onnxruntime: ort-nightly + examples: + bert_ptq_cpu: + exampleFolder: bert + exampleName: bert_ptq_cpu + bert_ptq_cpu_aml: + exampleFolder: bert + exampleName: bert_ptq_cpu_aml + bert_ptq_cpu_docker: + exampleFolder: bert + exampleName: bert_ptq_cpu_docker + cifar10_openvino_intel_hw: + exampleFolder: cifar10_openvino_intel_hw + exampleName: cifar10_openvino_intel_hw + resnet_ptq_cpu: + exampleFolder: resnet + exampleName: resnet_ptq_cpu + resnet_vitis_ai_ptq_cpu: + exampleFolder: resnet + exampleName: resnet_vitis_ai_ptq_cpu + resnet_qat: + exampleFolder: resnet + exampleName: resnet_qat + whisper: + exampleFolder: whisper + exampleName: whisper + +- template: job_templates/olive-build-template.yaml + parameters: + name: Windows_CI + pool: $(OLIVE_POOL_WIN2019) + windows: True + onnxruntime: ort-nightly + examples: + bert_ptq_cpu: + exampleFolder: bert + exampleName: bert_ptq_cpu + bert_ptq_cpu_aml: + exampleFolder: bert + exampleName: bert_ptq_cpu_aml + cifar10_openvino_intel_hw: + exampleFolder: cifar10_openvino_intel_hw + exampleName: cifar10_openvino_intel_hw + resnet_ptq_cpu: + exampleFolder: resnet + exampleName: resnet_ptq_cpu + resnet_qat: + exampleFolder: resnet + exampleName: resnet_qat + whisper: + exampleFolder: whisper + exampleName: whisper + +- template: job_templates/olive-build-template.yaml + parameters: + name: Linux_GPU_CI + pool: $(OLIVE_POOL_UBUNTU2004) + windows: False + device: gpu + onnxruntime: ort-nightly-gpu + examples: + bert_cuda_gpu: + exampleFolder: bert + exampleName: bert_cuda_gpu diff --git a/examples/resnet/requirements.txt b/examples/resnet/requirements.txt index 71e30b9b1..8b6195a92 100644 --- a/examples/resnet/requirements.txt +++ b/examples/resnet/requirements.txt @@ -1,7 +1,6 @@ azure-ai-ml azure-identity azureml-fsspec -onnxruntime<=1.15.1 pytorch-lightning scipy tabulate diff --git a/examples/test/test_resnet_ptq_cpu.py b/examples/test/test_resnet_ptq_cpu.py index 6b05d39ee..bb0437d4b 100644 --- a/examples/test/test_resnet_ptq_cpu.py +++ b/examples/test/test_resnet_ptq_cpu.py @@ -6,6 +6,8 @@ from pathlib import Path import pytest +from onnxruntime import __version__ as OrtVersion +from packaging import version from utils import check_output, patch_config from olive.common.utils import retry_func, run_subprocess @@ -30,6 +32,10 @@ def setup(): @pytest.mark.parametrize("execution_order", ["pass-by-pass"]) @pytest.mark.parametrize("system", ["local_system", "aml_system"]) @pytest.mark.parametrize("olive_json", ["resnet_ptq_cpu.json", "resnet_ptq_cpu_aml_dataset.json"]) +@pytest.mark.skipif( + version.parse(OrtVersion) == version.parse("1.16.0"), + reason="resnet is not supported in ORT 1.16.0 caused by https://github.com/microsoft/onnxruntime/issues/17627", +) def test_resnet(search_algorithm, execution_order, system, olive_json): # TODO: add gpu e2e test from olive.workflows import run as olive_run diff --git a/examples/test/test_resnet_vitis_ai_ptq_cpu.py b/examples/test/test_resnet_vitis_ai_ptq_cpu.py index 5d16c4293..4e29eb819 100644 --- a/examples/test/test_resnet_vitis_ai_ptq_cpu.py +++ b/examples/test/test_resnet_vitis_ai_ptq_cpu.py @@ -33,8 +33,8 @@ def setup(): @pytest.mark.parametrize("system", ["local_system", "aml_system"]) @pytest.mark.parametrize("olive_json", ["resnet_vitis_ai_ptq_cpu.json"]) @pytest.mark.skipif( - version.parse(OrtVersion) >= version.parse("1.16.0"), - reason="VitisAIQuantization is not supported in ORT 1.16.0 with TensorsData", + version.parse(OrtVersion) == version.parse("1.16.0"), + reason="resnet is not supported in ORT 1.16.0 caused by https://github.com/microsoft/onnxruntime/issues/17627", ) def test_resnet(search_algorithm, execution_order, system, olive_json): # TODO: add gpu e2e test diff --git a/examples/whisper/requirements.txt b/examples/whisper/requirements.txt index de2e74988..ff62cde85 100644 --- a/examples/whisper/requirements.txt +++ b/examples/whisper/requirements.txt @@ -1,6 +1,5 @@ neural-compressor onnx==1.14.0 -onnxruntime>=1.15.0 onnxruntime-extensions==0.8.0 tabulate torch>=1.13.1 diff --git a/test/unit_test/passes/vitis_ai/test_vitis_ai_quantization.py b/test/unit_test/passes/vitis_ai/test_vitis_ai_quantization.py index f99db13dc..7441d3525 100644 --- a/test/unit_test/passes/vitis_ai/test_vitis_ai_quantization.py +++ b/test/unit_test/passes/vitis_ai/test_vitis_ai_quantization.py @@ -38,7 +38,7 @@ def dummy_calibration_reader(data_dir=None, batch_size=1, *args, **kwargs): @pytest.mark.skipif( - version.parse(OrtVersion) >= version.parse("1.16.0"), + version.parse(OrtVersion) == version.parse("1.16.0"), reason="VitisAIQuantization is not supported in ORT 1.16.0 with TensorsData", ) def test_vitis_ai_quantization_pass(tmp_path): From 3c2327c5c388596434b33c83a1c96152932c196f Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Fri, 22 Sep 2023 12:09:34 +0800 Subject: [PATCH 13/25] add pr none --- .azure_pipelines/olive-ort-nightly.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.azure_pipelines/olive-ort-nightly.yaml b/.azure_pipelines/olive-ort-nightly.yaml index f92312466..1e84820cd 100644 --- a/.azure_pipelines/olive-ort-nightly.yaml +++ b/.azure_pipelines/olive-ort-nightly.yaml @@ -1,5 +1,7 @@ trigger: none +pr: none + schedules: - cron: '0 0 * * *' displayName: Daily build From 79347c9009a945b147f87080353c4f4ee8091505 Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Fri, 22 Sep 2023 12:13:05 +0800 Subject: [PATCH 14/25] try to use alternative connection --- .azure_pipelines/job_templates/olive-build-template.yaml | 2 +- .azure_pipelines/job_templates/olive-test-template.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.azure_pipelines/job_templates/olive-build-template.yaml b/.azure_pipelines/job_templates/olive-build-template.yaml index 25d3187dd..7e86b2803 100644 --- a/.azure_pipelines/job_templates/olive-build-template.yaml +++ b/.azure_pipelines/job_templates/olive-build-template.yaml @@ -57,7 +57,7 @@ jobs: - task: AzureCLI@1 inputs: - azureSubscription: $(OLIVE_RG_SERVICE_CONNECTION) + azureSubscription: $(OLIVE_RG_SERVICE_CONNECTION2) scriptLocation: 'inlineScript' inlineScript: | python -m pip install pytest diff --git a/.azure_pipelines/job_templates/olive-test-template.yaml b/.azure_pipelines/job_templates/olive-test-template.yaml index a9114438b..04e21f7d8 100644 --- a/.azure_pipelines/job_templates/olive-test-template.yaml +++ b/.azure_pipelines/job_templates/olive-test-template.yaml @@ -46,7 +46,7 @@ jobs: - ${{ if and(eq(variables.WINDOWS, 'True'), eq(variables.testType, 'multiple_ep')) }}: - task: AzureCLI@1 inputs: - azureSubscription: $(OLIVE_RG_SERVICE_CONNECTION) + azureSubscription: $(OLIVE_RG_SERVICE_CONNECTION2) scriptLocation: 'inlineScript' inlineScript: | call python -m pip install pytest @@ -68,7 +68,7 @@ jobs: - ${{ else }}: - task: AzureCLI@1 inputs: - azureSubscription: $(OLIVE_RG_SERVICE_CONNECTION) + azureSubscription: $(OLIVE_RG_SERVICE_CONNECTION2) scriptLocation: 'inlineScript' inlineScript: | python -m pip install pytest From 825cded8dc5c0e7272b8946c31928a990a654341 Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Fri, 22 Sep 2023 13:06:40 +0800 Subject: [PATCH 15/25] install the ort-nightly dependencies --- .azure_pipelines/job_templates/olive-build-template.yaml | 4 +++- .azure_pipelines/job_templates/olive-test-template.yaml | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.azure_pipelines/job_templates/olive-build-template.yaml b/.azure_pipelines/job_templates/olive-build-template.yaml index 7e86b2803..0f3b3b08d 100644 --- a/.azure_pipelines/job_templates/olive-build-template.yaml +++ b/.azure_pipelines/job_templates/olive-build-template.yaml @@ -48,6 +48,8 @@ jobs: - ${{ if startsWith(parameters.onnxruntime, 'ort-nightly') }}: - script: | + pip install onnxruntime + pip uninstall onnxruntime pip install ${{ parameters.onnxruntime }} --index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/ displayName: Install ${{ parameters.onnxruntime }} - ${{ else }}: @@ -57,7 +59,7 @@ jobs: - task: AzureCLI@1 inputs: - azureSubscription: $(OLIVE_RG_SERVICE_CONNECTION2) + azureSubscription: $(OLIVE_RG_SERVICE_CONNECTION) scriptLocation: 'inlineScript' inlineScript: | python -m pip install pytest diff --git a/.azure_pipelines/job_templates/olive-test-template.yaml b/.azure_pipelines/job_templates/olive-test-template.yaml index 04e21f7d8..98b1ee400 100644 --- a/.azure_pipelines/job_templates/olive-test-template.yaml +++ b/.azure_pipelines/job_templates/olive-test-template.yaml @@ -36,6 +36,8 @@ jobs: - ${{ if startsWith(parameters.onnxruntime, 'ort-nightly') }}: - script: | + pip install onnxruntime + pip uninstall onnxruntime pip install ${{ parameters.onnxruntime }} --index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/ displayName: Install ${{ parameters.onnxruntime }} - ${{ else }}: @@ -46,7 +48,7 @@ jobs: - ${{ if and(eq(variables.WINDOWS, 'True'), eq(variables.testType, 'multiple_ep')) }}: - task: AzureCLI@1 inputs: - azureSubscription: $(OLIVE_RG_SERVICE_CONNECTION2) + azureSubscription: $(OLIVE_RG_SERVICE_CONNECTION) scriptLocation: 'inlineScript' inlineScript: | call python -m pip install pytest @@ -68,7 +70,7 @@ jobs: - ${{ else }}: - task: AzureCLI@1 inputs: - azureSubscription: $(OLIVE_RG_SERVICE_CONNECTION2) + azureSubscription: $(OLIVE_RG_SERVICE_CONNECTION) scriptLocation: 'inlineScript' inlineScript: | python -m pip install pytest From 3743ef0275e88d36ef53e5c058f5d17ee5039428 Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Fri, 22 Sep 2023 13:18:41 +0800 Subject: [PATCH 16/25] update link --- .azure_pipelines/job_templates/olive-build-template.yaml | 4 ++-- .azure_pipelines/job_templates/olive-test-template.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.azure_pipelines/job_templates/olive-build-template.yaml b/.azure_pipelines/job_templates/olive-build-template.yaml index 0f3b3b08d..45768934c 100644 --- a/.azure_pipelines/job_templates/olive-build-template.yaml +++ b/.azure_pipelines/job_templates/olive-build-template.yaml @@ -49,8 +49,8 @@ jobs: - ${{ if startsWith(parameters.onnxruntime, 'ort-nightly') }}: - script: | pip install onnxruntime - pip uninstall onnxruntime - pip install ${{ parameters.onnxruntime }} --index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/ + pip uninstall -y onnxruntime + pip install ${{ parameters.onnxruntime }} --index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/pypi/simple/ displayName: Install ${{ parameters.onnxruntime }} - ${{ else }}: - script: | diff --git a/.azure_pipelines/job_templates/olive-test-template.yaml b/.azure_pipelines/job_templates/olive-test-template.yaml index 98b1ee400..f57160894 100644 --- a/.azure_pipelines/job_templates/olive-test-template.yaml +++ b/.azure_pipelines/job_templates/olive-test-template.yaml @@ -37,8 +37,8 @@ jobs: - ${{ if startsWith(parameters.onnxruntime, 'ort-nightly') }}: - script: | pip install onnxruntime - pip uninstall onnxruntime - pip install ${{ parameters.onnxruntime }} --index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/ + pip uninstall -y onnxruntime + pip install ${{ parameters.onnxruntime }} --index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/pypi/simple/ displayName: Install ${{ parameters.onnxruntime }} - ${{ else }}: - script: | From 2df5ed8ced0e383f7284ebf58854003f7288ba4f Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Fri, 22 Sep 2023 13:53:00 +0800 Subject: [PATCH 17/25] add ort for multiple EP --- .azure_pipelines/job_templates/olive-build-template.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.azure_pipelines/job_templates/olive-build-template.yaml b/.azure_pipelines/job_templates/olive-build-template.yaml index 45768934c..f555020f1 100644 --- a/.azure_pipelines/job_templates/olive-build-template.yaml +++ b/.azure_pipelines/job_templates/olive-build-template.yaml @@ -100,6 +100,7 @@ jobs: device: ${{parameters.device}} WINDOWS: ${{parameters.windows}} test_type: 'multiple_ep' + onnxruntime: ${{parameters.onnxruntime}} ${{ if eq(parameters.windows, 'True') }}: python_version: '3.10' ${{ else }}: From 0561e3ca0375b06847050197147e45114237959e Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Fri, 22 Sep 2023 14:50:10 +0800 Subject: [PATCH 18/25] remove needless examples in night pipeline --- .../job_templates/olive-build-template.yaml | 28 ++++++++++--------- .azure_pipelines/olive-ort-nightly.yaml | 11 ++------ 2 files changed, 17 insertions(+), 22 deletions(-) diff --git a/.azure_pipelines/job_templates/olive-build-template.yaml b/.azure_pipelines/job_templates/olive-build-template.yaml index f555020f1..f27f6686a 100644 --- a/.azure_pipelines/job_templates/olive-build-template.yaml +++ b/.azure_pipelines/job_templates/olive-build-template.yaml @@ -5,6 +5,7 @@ parameters: pool: '' device: 'cpu' onnxruntime: 'onnxruntime' + enable_multiple_ep: True jobs: - template: olive-test-template.yaml @@ -25,6 +26,20 @@ jobs: test_type: 'integ_test' onnxruntime: ${{parameters.onnxruntime}} +- ${{ if enable_multiple_ep }}: + - template: olive-test-template.yaml + parameters: + name: ${{parameters.name}}_Multiple_EP_Test_Olive + pool: ${{parameters.pool}} + device: ${{parameters.device}} + WINDOWS: ${{parameters.windows}} + test_type: 'multiple_ep' + onnxruntime: ${{parameters.onnxruntime}} + ${{ if eq(parameters.windows, 'True') }}: + python_version: '3.10' + ${{ else }}: + python_version: '3.8' + - job: ${{parameters.name}}_Test_Examples timeoutInMinutes: 300 pool: @@ -92,16 +107,3 @@ jobs: - script: git clean -dfX condition: always() displayName: Clean remaining artifacts - -- template: olive-test-template.yaml - parameters: - name: ${{parameters.name}}_Multiple_EP_Test_Olive - pool: ${{parameters.pool}} - device: ${{parameters.device}} - WINDOWS: ${{parameters.windows}} - test_type: 'multiple_ep' - onnxruntime: ${{parameters.onnxruntime}} - ${{ if eq(parameters.windows, 'True') }}: - python_version: '3.10' - ${{ else }}: - python_version: '3.8' diff --git a/.azure_pipelines/olive-ort-nightly.yaml b/.azure_pipelines/olive-ort-nightly.yaml index 1e84820cd..f390eaa53 100644 --- a/.azure_pipelines/olive-ort-nightly.yaml +++ b/.azure_pipelines/olive-ort-nightly.yaml @@ -19,16 +19,11 @@ jobs: pool: $(OLIVE_POOL_UBUNTU2004) windows: False onnxruntime: ort-nightly + enable_multiple_ep: False examples: bert_ptq_cpu: exampleFolder: bert exampleName: bert_ptq_cpu - bert_ptq_cpu_aml: - exampleFolder: bert - exampleName: bert_ptq_cpu_aml - bert_ptq_cpu_docker: - exampleFolder: bert - exampleName: bert_ptq_cpu_docker cifar10_openvino_intel_hw: exampleFolder: cifar10_openvino_intel_hw exampleName: cifar10_openvino_intel_hw @@ -51,13 +46,11 @@ jobs: pool: $(OLIVE_POOL_WIN2019) windows: True onnxruntime: ort-nightly + enable_multiple_ep: False examples: bert_ptq_cpu: exampleFolder: bert exampleName: bert_ptq_cpu - bert_ptq_cpu_aml: - exampleFolder: bert - exampleName: bert_ptq_cpu_aml cifar10_openvino_intel_hw: exampleFolder: cifar10_openvino_intel_hw exampleName: cifar10_openvino_intel_hw From a7dd3575ebbbbb3ebef0adfc054a00fd608cf3c7 Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Fri, 22 Sep 2023 14:51:57 +0800 Subject: [PATCH 19/25] fix syntax error --- .azure_pipelines/job_templates/olive-build-template.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure_pipelines/job_templates/olive-build-template.yaml b/.azure_pipelines/job_templates/olive-build-template.yaml index f27f6686a..ff90bace2 100644 --- a/.azure_pipelines/job_templates/olive-build-template.yaml +++ b/.azure_pipelines/job_templates/olive-build-template.yaml @@ -26,7 +26,7 @@ jobs: test_type: 'integ_test' onnxruntime: ${{parameters.onnxruntime}} -- ${{ if enable_multiple_ep }}: +- ${{ if parameters.enable_multiple_ep }}: - template: olive-test-template.yaml parameters: name: ${{parameters.name}}_Multiple_EP_Test_Olive From bbb6787a54f8a473a681619179b31fa0cb7a21cc Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Fri, 22 Sep 2023 15:35:34 +0800 Subject: [PATCH 20/25] change multiple ep nightly check --- .azure_pipelines/job_templates/olive-build-doc-template.yaml | 2 +- .azure_pipelines/job_templates/olive-build-template.yaml | 4 ++-- .azure_pipelines/job_templates/olive-test-template.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.azure_pipelines/job_templates/olive-build-doc-template.yaml b/.azure_pipelines/job_templates/olive-build-doc-template.yaml index adb17a22d..5adb2715b 100644 --- a/.azure_pipelines/job_templates/olive-build-doc-template.yaml +++ b/.azure_pipelines/job_templates/olive-build-doc-template.yaml @@ -27,7 +27,7 @@ jobs: displayName: Checkout release branch condition: ne('${{parameters.doc_version}}', 'latest') - - script: pip install .[cpu] + - script: python -m pip install .[cpu] displayName: Install Olive - script: | diff --git a/.azure_pipelines/job_templates/olive-build-template.yaml b/.azure_pipelines/job_templates/olive-build-template.yaml index ff90bace2..6ab53241a 100644 --- a/.azure_pipelines/job_templates/olive-build-template.yaml +++ b/.azure_pipelines/job_templates/olive-build-template.yaml @@ -26,7 +26,7 @@ jobs: test_type: 'integ_test' onnxruntime: ${{parameters.onnxruntime}} -- ${{ if parameters.enable_multiple_ep }}: +- ${{ if eq(parameters.enable_multiple_ep, 'True') }}: - template: olive-test-template.yaml parameters: name: ${{parameters.name}}_Multiple_EP_Test_Olive @@ -58,7 +58,7 @@ jobs: versionSpec: 3.8 displayName: Use Python 3.8 - - script: pip install . + - script: python -m pip install . displayName: Install Olive - ${{ if startsWith(parameters.onnxruntime, 'ort-nightly') }}: diff --git a/.azure_pipelines/job_templates/olive-test-template.yaml b/.azure_pipelines/job_templates/olive-test-template.yaml index f57160894..504a06422 100644 --- a/.azure_pipelines/job_templates/olive-test-template.yaml +++ b/.azure_pipelines/job_templates/olive-test-template.yaml @@ -31,7 +31,7 @@ jobs: continueOnError: true - script: | - pip install . + python -m pip install . displayName: Install Olive - ${{ if startsWith(parameters.onnxruntime, 'ort-nightly') }}: From fa37d8bbade8aa1cf83d3a4bb0e325e1350ebf1d Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Fri, 22 Sep 2023 16:36:18 +0800 Subject: [PATCH 21/25] refactor ci pipeline --- .../job_templates/olive-build-template.yaml | 87 +------------------ .../job_templates/olive-test-template.yaml | 1 - .../job_templates/olve-example-template.yaml | 74 ++++++++++++++++ .azure_pipelines/olive-ci.yaml | 48 ++++++++-- .azure_pipelines/olive-ort-nightly.yaml | 36 ++++++-- 5 files changed, 148 insertions(+), 98 deletions(-) create mode 100644 .azure_pipelines/job_templates/olve-example-template.yaml diff --git a/.azure_pipelines/job_templates/olive-build-template.yaml b/.azure_pipelines/job_templates/olive-build-template.yaml index 6ab53241a..349a58a99 100644 --- a/.azure_pipelines/job_templates/olive-build-template.yaml +++ b/.azure_pipelines/job_templates/olive-build-template.yaml @@ -5,7 +5,6 @@ parameters: pool: '' device: 'cpu' onnxruntime: 'onnxruntime' - enable_multiple_ep: True jobs: - template: olive-test-template.yaml @@ -13,7 +12,7 @@ jobs: name: ${{parameters.name}}_Unit_Test_Olive pool: ${{parameters.pool}} device: ${{parameters.device}} - WINDOWS: ${{parameters.windows}} + windows: ${{parameters.windows}} test_type: 'unit_test' onnxruntime: ${{parameters.onnxruntime}} @@ -22,88 +21,6 @@ jobs: name: ${{parameters.name}}_Integration_Test_Olive pool: ${{parameters.pool}} device: ${{parameters.device}} - WINDOWS: ${{parameters.windows}} + windows: ${{parameters.windows}} test_type: 'integ_test' onnxruntime: ${{parameters.onnxruntime}} - -- ${{ if eq(parameters.enable_multiple_ep, 'True') }}: - - template: olive-test-template.yaml - parameters: - name: ${{parameters.name}}_Multiple_EP_Test_Olive - pool: ${{parameters.pool}} - device: ${{parameters.device}} - WINDOWS: ${{parameters.windows}} - test_type: 'multiple_ep' - onnxruntime: ${{parameters.onnxruntime}} - ${{ if eq(parameters.windows, 'True') }}: - python_version: '3.10' - ${{ else }}: - python_version: '3.8' - -- job: ${{parameters.name}}_Test_Examples - timeoutInMinutes: 300 - pool: - name: ${{ parameters.pool}} - strategy: - matrix: - ${{ insert }}: ${{ parameters.examples }} - variables: - WINDOWS: ${{ parameters.windows }} - runCodesignValidationInjection: false - device: ${{ parameters.device }} - - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: 3.8 - displayName: Use Python 3.8 - - - script: python -m pip install . - displayName: Install Olive - - - ${{ if startsWith(parameters.onnxruntime, 'ort-nightly') }}: - - script: | - pip install onnxruntime - pip uninstall -y onnxruntime - pip install ${{ parameters.onnxruntime }} --index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/pypi/simple/ - displayName: Install ${{ parameters.onnxruntime }} - - ${{ else }}: - - script: | - pip install ${{ parameters.onnxruntime }} - displayName: Install ${{ parameters.onnxruntime }} - - - task: AzureCLI@1 - inputs: - azureSubscription: $(OLIVE_RG_SERVICE_CONNECTION) - scriptLocation: 'inlineScript' - inlineScript: | - python -m pip install pytest - python -m pip install -r $(Build.SourcesDirectory)/examples/$(exampleFolder)/requirements.txt - python -m pytest -v -s --log-cli-level=WARNING --junitxml=$(Build.SourcesDirectory)/logs/test_examples-TestOlive.xml $(Build.SourcesDirectory)/examples/test/test_$(exampleName).py - displayName: Test Examples - env: - OLIVEWHEELS_STORAGE_CONNECTION_STRING: $(olive-wheels-storage-connection-string) - WORKSPACE_SUBSCRIPTION_ID: $(workspace-subscription-id) - WORKSPACE_RESOURCE_GROUP: $(workspace-resource-group) - WORKSPACE_NAME: $(workspace-name) - AZURE_TENANT_ID: $(azure-tenant-id) - AZURE_CLIENT_ID: $(olive-rg-sp-id) - AZURE_CLIENT_SECRET: $(olive-rg-sp-secret) - - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - displayName: Component Detection - - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/*TestOlive*.xml' - testRunTitle: '$(Build.BuildNumber)[$(Agent.JobName)]' - displayName: Upload pipeline run test results - - - script: git clean -dfX - condition: always() - displayName: Clean remaining artifacts diff --git a/.azure_pipelines/job_templates/olive-test-template.yaml b/.azure_pipelines/job_templates/olive-test-template.yaml index 504a06422..c450e309e 100644 --- a/.azure_pipelines/job_templates/olive-test-template.yaml +++ b/.azure_pipelines/job_templates/olive-test-template.yaml @@ -15,7 +15,6 @@ jobs: variables: WINDOWS: ${{ parameters.windows}} runCodesignValidationInjection: false - device: ${{ parameters.device }} testType: ${{ parameters.test_type }} python_version: ${{ parameters.python_version }} diff --git a/.azure_pipelines/job_templates/olve-example-template.yaml b/.azure_pipelines/job_templates/olve-example-template.yaml new file mode 100644 index 000000000..52711d6fe --- /dev/null +++ b/.azure_pipelines/job_templates/olve-example-template.yaml @@ -0,0 +1,74 @@ +# Olive Build and Test Pipeline template for examples on Azure DevOps + +parameters: + name: '' + pool: '' + python_version: '3.8' + onnxruntime: 'onnxruntime' + +jobs: + - job: ${{parameters.name}}_Test_Examples + timeoutInMinutes: 300 + pool: + name: ${{ parameters.pool}} + strategy: + matrix: + ${{ insert }}: ${{ parameters.examples }} + variables: + runCodesignValidationInjection: false + + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: ${{ parameters.python_version }} + displayName: Use Python ${{ parameters.python_version }} + + - script: python -m pip install . + displayName: Install Olive + + - ${{ if startsWith(parameters.onnxruntime, 'ort-nightly') }}: + - script: | + pip install onnxruntime + pip uninstall -y onnxruntime + pip install ${{ parameters.onnxruntime }} --index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/pypi/simple/ + displayName: Install ${{ parameters.onnxruntime }} + - ${{ else }}: + - script: | + pip install ${{ parameters.onnxruntime }} + displayName: Install ${{ parameters.onnxruntime }} + + - task: AzureCLI@1 + inputs: + azureSubscription: $(OLIVE_RG_SERVICE_CONNECTION) + scriptLocation: 'inlineScript' + inlineScript: | + python -m pip install pytest + python -m pip install -r $(Build.SourcesDirectory)/examples/$(exampleFolder)/requirements.txt + python -m pytest -v -s --log-cli-level=WARNING --junitxml=$(Build.SourcesDirectory)/logs/test_examples-TestOlive.xml $(Build.SourcesDirectory)/examples/test/test_$(exampleName).py + displayName: Test Examples + env: + OLIVEWHEELS_STORAGE_CONNECTION_STRING: $(olive-wheels-storage-connection-string) + WORKSPACE_SUBSCRIPTION_ID: $(workspace-subscription-id) + WORKSPACE_RESOURCE_GROUP: $(workspace-resource-group) + WORKSPACE_NAME: $(workspace-name) + AZURE_TENANT_ID: $(azure-tenant-id) + AZURE_CLIENT_ID: $(olive-rg-sp-id) + AZURE_CLIENT_SECRET: $(olive-rg-sp-secret) + + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + displayName: Component Detection + + - task: PublishTestResults@2 + condition: succeededOrFailed() + inputs: + testResultsFiles: '**/*TestOlive*.xml' + testRunTitle: '$(Build.BuildNumber)[$(Agent.JobName)]' + displayName: Upload pipeline run test results + + - script: git clean -dfX + condition: always() + displayName: Clean remaining artifacts diff --git a/.azure_pipelines/olive-ci.yaml b/.azure_pipelines/olive-ci.yaml index d95b2a998..70e32a7f8 100644 --- a/.azure_pipelines/olive-ci.yaml +++ b/.azure_pipelines/olive-ci.yaml @@ -27,12 +27,28 @@ variables: ComponentDetection.Timeout: 2400 jobs: +# Linux unit test and integration test - template: job_templates/olive-build-template.yaml parameters: name: Linux_CI pool: $(OLIVE_POOL_UBUNTU2004) windows: False onnxruntime: onnxruntime + +# Windows unit test and integration test +- template: job_templates/olive-build-template.yaml + parameters: + name: Windows_CI + pool: $(OLIVE_POOL_WIN2019) + windows: True + onnxruntime: onnxruntime + +# Linux examples test +- template: job_templates/olve-example-template.yaml + parameters: + name: Linux_CI + pool: $(OLIVE_POOL_UBUNTU2004) + onnxruntime: onnxruntime examples: bert_ptq_cpu: exampleFolder: bert @@ -59,11 +75,11 @@ jobs: exampleFolder: whisper exampleName: whisper -- template: job_templates/olive-build-template.yaml +# # Windows examples test +- template: job_templates/olve-example-template.yaml parameters: name: Windows_CI pool: $(OLIVE_POOL_WIN2019) - windows: True onnxruntime: onnxruntime examples: bert_ptq_cpu: @@ -85,18 +101,40 @@ jobs: exampleFolder: whisper exampleName: whisper -- template: job_templates/olive-build-template.yaml +# Linux GPU examples testing. +- template: job_templates/olve-example-template.yaml parameters: name: Linux_GPU_CI pool: $(OLIVE_POOL_UBUNTU2004) - windows: False - device: gpu onnxruntime: onnxruntime-gpu examples: bert_cuda_gpu: exampleFolder: bert exampleName: bert_cuda_gpu +# Multiple EP Linux testing +- template: job_templates/olive-test-template.yaml + parameters: + name: Linux_CI_Multiple_EP_Test_Olive + pool: $(OLIVE_POOL_UBUNTU2004) + device: 'cpu' + windows: 'False' + test_type: 'multiple_ep' + onnxruntime: onnxruntime + python_version: '3.8' + +# Multiple EP Windows testing +- template: job_templates/olive-test-template.yaml + parameters: + name: Windows_CI_Multiple_EP_Test_Olive + pool: $(OLIVE_POOL_WIN2019) + device: 'cpu' + windows: 'True' + test_type: 'multiple_ep' + onnxruntime: onnxruntime + python_version: '3.10' + +# build documents - template: job_templates/olive-build-doc-template.yaml parameters: job_name: Test_BuildDocs diff --git a/.azure_pipelines/olive-ort-nightly.yaml b/.azure_pipelines/olive-ort-nightly.yaml index f390eaa53..14359ea0c 100644 --- a/.azure_pipelines/olive-ort-nightly.yaml +++ b/.azure_pipelines/olive-ort-nightly.yaml @@ -13,17 +13,38 @@ variables: ComponentDetection.Timeout: 2400 jobs: +# Linux unit test and integration test - template: job_templates/olive-build-template.yaml parameters: name: Linux_CI pool: $(OLIVE_POOL_UBUNTU2004) windows: False onnxruntime: ort-nightly - enable_multiple_ep: False + +# Windows unit test and integration test +- template: job_templates/olive-build-template.yaml + parameters: + name: Windows_CI + pool: $(OLIVE_POOL_WIN2019) + windows: True + onnxruntime: ort-nightly + +# Linux examples test +- template: job_templates/olve-example-template.yaml + parameters: + name: Linux_CI + pool: $(OLIVE_POOL_UBUNTU2004) + onnxruntime: ort-nightly examples: bert_ptq_cpu: exampleFolder: bert exampleName: bert_ptq_cpu + bert_ptq_cpu_aml: + exampleFolder: bert + exampleName: bert_ptq_cpu_aml + bert_ptq_cpu_docker: + exampleFolder: bert + exampleName: bert_ptq_cpu_docker cifar10_openvino_intel_hw: exampleFolder: cifar10_openvino_intel_hw exampleName: cifar10_openvino_intel_hw @@ -40,17 +61,19 @@ jobs: exampleFolder: whisper exampleName: whisper -- template: job_templates/olive-build-template.yaml +# # Windows examples test +- template: job_templates/olve-example-template.yaml parameters: name: Windows_CI pool: $(OLIVE_POOL_WIN2019) - windows: True onnxruntime: ort-nightly - enable_multiple_ep: False examples: bert_ptq_cpu: exampleFolder: bert exampleName: bert_ptq_cpu + bert_ptq_cpu_aml: + exampleFolder: bert + exampleName: bert_ptq_cpu_aml cifar10_openvino_intel_hw: exampleFolder: cifar10_openvino_intel_hw exampleName: cifar10_openvino_intel_hw @@ -64,12 +87,11 @@ jobs: exampleFolder: whisper exampleName: whisper -- template: job_templates/olive-build-template.yaml +# Linux GPU examples testing. +- template: job_templates/olve-example-template.yaml parameters: name: Linux_GPU_CI pool: $(OLIVE_POOL_UBUNTU2004) - windows: False - device: gpu onnxruntime: ort-nightly-gpu examples: bert_cuda_gpu: From db7b1b77b1da7a33bf0a57bc6190e310b5859d32 Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Fri, 22 Sep 2023 16:49:26 +0800 Subject: [PATCH 22/25] remove nightly examples for aml and docker --- ...plate.yaml => olive-example-template.yaml} | 0 .azure_pipelines/olive-ci.yaml | 6 +- .azure_pipelines/olive-ort-last.yaml | 90 +++++++++++++++++++ .azure_pipelines/olive-ort-nightly.yaml | 15 +--- 4 files changed, 96 insertions(+), 15 deletions(-) rename .azure_pipelines/job_templates/{olve-example-template.yaml => olive-example-template.yaml} (100%) create mode 100644 .azure_pipelines/olive-ort-last.yaml diff --git a/.azure_pipelines/job_templates/olve-example-template.yaml b/.azure_pipelines/job_templates/olive-example-template.yaml similarity index 100% rename from .azure_pipelines/job_templates/olve-example-template.yaml rename to .azure_pipelines/job_templates/olive-example-template.yaml diff --git a/.azure_pipelines/olive-ci.yaml b/.azure_pipelines/olive-ci.yaml index 70e32a7f8..eb6739cea 100644 --- a/.azure_pipelines/olive-ci.yaml +++ b/.azure_pipelines/olive-ci.yaml @@ -44,7 +44,7 @@ jobs: onnxruntime: onnxruntime # Linux examples test -- template: job_templates/olve-example-template.yaml +- template: job_templates/olive-example-template parameters: name: Linux_CI pool: $(OLIVE_POOL_UBUNTU2004) @@ -76,7 +76,7 @@ jobs: exampleName: whisper # # Windows examples test -- template: job_templates/olve-example-template.yaml +- template: job_templates/olive-example-template parameters: name: Windows_CI pool: $(OLIVE_POOL_WIN2019) @@ -102,7 +102,7 @@ jobs: exampleName: whisper # Linux GPU examples testing. -- template: job_templates/olve-example-template.yaml +- template: job_templates/olive-example-template parameters: name: Linux_GPU_CI pool: $(OLIVE_POOL_UBUNTU2004) diff --git a/.azure_pipelines/olive-ort-last.yaml b/.azure_pipelines/olive-ort-last.yaml new file mode 100644 index 000000000..4d9fba92e --- /dev/null +++ b/.azure_pipelines/olive-ort-last.yaml @@ -0,0 +1,90 @@ +trigger: none + +pr: none + +schedules: +- cron: '0 0 * * *' + displayName: Daily build + branches: + include: + - main + +variables: + ComponentDetection.Timeout: 2400 + +jobs: +# Linux unit test and integration test +- template: job_templates/olive-build-template.yaml + parameters: + name: Linux_CI + pool: $(OLIVE_POOL_UBUNTU2004) + windows: False + onnxruntime: onnxruntime==1.15.1 + +# Windows unit test and integration test +- template: job_templates/olive-build-template.yaml + parameters: + name: Windows_CI + pool: $(OLIVE_POOL_WIN2019) + windows: True + onnxruntime: onnxruntime==1.15.1 + +# Linux examples test +- template: job_templates/olive-example-template + parameters: + name: Linux_CI + pool: $(OLIVE_POOL_UBUNTU2004) + onnxruntime: onnxruntime==1.15.1 + examples: + bert_ptq_cpu: + exampleFolder: bert + exampleName: bert_ptq_cpu + cifar10_openvino_intel_hw: + exampleFolder: cifar10_openvino_intel_hw + exampleName: cifar10_openvino_intel_hw + resnet_ptq_cpu: + exampleFolder: resnet + exampleName: resnet_ptq_cpu + resnet_vitis_ai_ptq_cpu: + exampleFolder: resnet + exampleName: resnet_vitis_ai_ptq_cpu + resnet_qat: + exampleFolder: resnet + exampleName: resnet_qat + whisper: + exampleFolder: whisper + exampleName: whisper + +# # Windows examples test +- template: job_templates/olive-example-template + parameters: + name: Windows_CI + pool: $(OLIVE_POOL_WIN2019) + onnxruntime: onnxruntime==1.15.1 + examples: + bert_ptq_cpu: + exampleFolder: bert + exampleName: bert_ptq_cpu + cifar10_openvino_intel_hw: + exampleFolder: cifar10_openvino_intel_hw + exampleName: cifar10_openvino_intel_hw + resnet_ptq_cpu: + exampleFolder: resnet + exampleName: resnet_ptq_cpu + resnet_qat: + exampleFolder: resnet + exampleName: resnet_qat + whisper: + exampleFolder: whisper + exampleName: whisper + +# Linux GPU examples testing. +- template: job_templates/olive-example-template + parameters: + name: Linux_GPU_CI + pool: $(OLIVE_POOL_UBUNTU2004) + onnxruntime: onnxruntime-gpu==1.15.1 + examples: + bert_cuda_gpu: + exampleFolder: bert + exampleName: bert_cuda_gpu diff --git a/.azure_pipelines/olive-ort-nightly.yaml b/.azure_pipelines/olive-ort-nightly.yaml index 14359ea0c..82a37b905 100644 --- a/.azure_pipelines/olive-ort-nightly.yaml +++ b/.azure_pipelines/olive-ort-nightly.yaml @@ -30,7 +30,7 @@ jobs: onnxruntime: ort-nightly # Linux examples test -- template: job_templates/olve-example-template.yaml +- template: job_templates/olive-example-template parameters: name: Linux_CI pool: $(OLIVE_POOL_UBUNTU2004) @@ -39,12 +39,6 @@ jobs: bert_ptq_cpu: exampleFolder: bert exampleName: bert_ptq_cpu - bert_ptq_cpu_aml: - exampleFolder: bert - exampleName: bert_ptq_cpu_aml - bert_ptq_cpu_docker: - exampleFolder: bert - exampleName: bert_ptq_cpu_docker cifar10_openvino_intel_hw: exampleFolder: cifar10_openvino_intel_hw exampleName: cifar10_openvino_intel_hw @@ -62,7 +56,7 @@ jobs: exampleName: whisper # # Windows examples test -- template: job_templates/olve-example-template.yaml +- template: job_templates/olive-example-template parameters: name: Windows_CI pool: $(OLIVE_POOL_WIN2019) @@ -71,9 +65,6 @@ jobs: bert_ptq_cpu: exampleFolder: bert exampleName: bert_ptq_cpu - bert_ptq_cpu_aml: - exampleFolder: bert - exampleName: bert_ptq_cpu_aml cifar10_openvino_intel_hw: exampleFolder: cifar10_openvino_intel_hw exampleName: cifar10_openvino_intel_hw @@ -88,7 +79,7 @@ jobs: exampleName: whisper # Linux GPU examples testing. -- template: job_templates/olve-example-template.yaml +- template: job_templates/olive-example-template parameters: name: Linux_GPU_CI pool: $(OLIVE_POOL_UBUNTU2004) From fd66b39b1fb4e79089413f84fdfa785c9e750062 Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Fri, 22 Sep 2023 16:52:34 +0800 Subject: [PATCH 23/25] fix template path --- .azure_pipelines/olive-ci.yaml | 6 +++--- .azure_pipelines/olive-ort-last.yaml | 6 +++--- .azure_pipelines/olive-ort-nightly.yaml | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.azure_pipelines/olive-ci.yaml b/.azure_pipelines/olive-ci.yaml index eb6739cea..e71bdbfad 100644 --- a/.azure_pipelines/olive-ci.yaml +++ b/.azure_pipelines/olive-ci.yaml @@ -44,7 +44,7 @@ jobs: onnxruntime: onnxruntime # Linux examples test -- template: job_templates/olive-example-template +- template: job_templates/olive-example-template.yaml parameters: name: Linux_CI pool: $(OLIVE_POOL_UBUNTU2004) @@ -76,7 +76,7 @@ jobs: exampleName: whisper # # Windows examples test -- template: job_templates/olive-example-template +- template: job_templates/olive-example-template.yaml parameters: name: Windows_CI pool: $(OLIVE_POOL_WIN2019) @@ -102,7 +102,7 @@ jobs: exampleName: whisper # Linux GPU examples testing. -- template: job_templates/olive-example-template +- template: job_templates/olive-example-template.yaml parameters: name: Linux_GPU_CI pool: $(OLIVE_POOL_UBUNTU2004) diff --git a/.azure_pipelines/olive-ort-last.yaml b/.azure_pipelines/olive-ort-last.yaml index 4d9fba92e..fbddfa8a4 100644 --- a/.azure_pipelines/olive-ort-last.yaml +++ b/.azure_pipelines/olive-ort-last.yaml @@ -30,7 +30,7 @@ jobs: onnxruntime: onnxruntime==1.15.1 # Linux examples test -- template: job_templates/olive-example-template +- template: job_templates/olive-example-template.yaml parameters: name: Linux_CI pool: $(OLIVE_POOL_UBUNTU2004) @@ -56,7 +56,7 @@ jobs: exampleName: whisper # # Windows examples test -- template: job_templates/olive-example-template +- template: job_templates/olive-example-template.yaml parameters: name: Windows_CI pool: $(OLIVE_POOL_WIN2019) @@ -79,7 +79,7 @@ jobs: exampleName: whisper # Linux GPU examples testing. -- template: job_templates/olive-example-template +- template: job_templates/olive-example-template.yaml parameters: name: Linux_GPU_CI pool: $(OLIVE_POOL_UBUNTU2004) diff --git a/.azure_pipelines/olive-ort-nightly.yaml b/.azure_pipelines/olive-ort-nightly.yaml index 82a37b905..1e8c83583 100644 --- a/.azure_pipelines/olive-ort-nightly.yaml +++ b/.azure_pipelines/olive-ort-nightly.yaml @@ -30,7 +30,7 @@ jobs: onnxruntime: ort-nightly # Linux examples test -- template: job_templates/olive-example-template +- template: job_templates/olive-example-template.yaml parameters: name: Linux_CI pool: $(OLIVE_POOL_UBUNTU2004) @@ -56,7 +56,7 @@ jobs: exampleName: whisper # # Windows examples test -- template: job_templates/olive-example-template +- template: job_templates/olive-example-template.yaml parameters: name: Windows_CI pool: $(OLIVE_POOL_WIN2019) @@ -79,7 +79,7 @@ jobs: exampleName: whisper # Linux GPU examples testing. -- template: job_templates/olive-example-template +- template: job_templates/olive-example-template.yaml parameters: name: Linux_GPU_CI pool: $(OLIVE_POOL_UBUNTU2004) From b98e93b30870b413455ebbb204748b4342d60315 Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Fri, 22 Sep 2023 17:35:07 +0800 Subject: [PATCH 24/25] remove integration test in nightly since most of them are aml --- .azure_pipelines/olive-ort-last.yaml | 16 ++++++++++------ .azure_pipelines/olive-ort-nightly.yaml | 12 ++++++++---- examples/test/test_resnet_vitis_ai_ptq_cpu.py | 2 +- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/.azure_pipelines/olive-ort-last.yaml b/.azure_pipelines/olive-ort-last.yaml index fbddfa8a4..623bc5f51 100644 --- a/.azure_pipelines/olive-ort-last.yaml +++ b/.azure_pipelines/olive-ort-last.yaml @@ -13,21 +13,25 @@ variables: ComponentDetection.Timeout: 2400 jobs: -# Linux unit test and integration test -- template: job_templates/olive-build-template.yaml +# Linux unit test +- template: job_templates/olive-test-template.yaml parameters: name: Linux_CI pool: $(OLIVE_POOL_UBUNTU2004) + device: 'cpu' windows: False - onnxruntime: onnxruntime==1.15.1 + test_type: 'unit_test' + onnxruntime: 'onnxruntime==1.15.1' -# Windows unit test and integration test -- template: job_templates/olive-build-template.yaml +# Windows unit test +- template: job_templates/olive-test-template.yaml parameters: name: Windows_CI pool: $(OLIVE_POOL_WIN2019) + device: 'cpu' windows: True - onnxruntime: onnxruntime==1.15.1 + test_type: 'unit_test' + onnxruntime: 'onnxruntime==1.15.1' # Linux examples test - template: job_templates/olive-example-template.yaml diff --git a/.azure_pipelines/olive-ort-nightly.yaml b/.azure_pipelines/olive-ort-nightly.yaml index 1e8c83583..6df94e8ab 100644 --- a/.azure_pipelines/olive-ort-nightly.yaml +++ b/.azure_pipelines/olive-ort-nightly.yaml @@ -13,20 +13,24 @@ variables: ComponentDetection.Timeout: 2400 jobs: -# Linux unit test and integration test -- template: job_templates/olive-build-template.yaml +# Linux unit test +- template: job_templates/olive-test-template.yaml parameters: name: Linux_CI pool: $(OLIVE_POOL_UBUNTU2004) + device: 'cpu' windows: False + test_type: 'unit_test' onnxruntime: ort-nightly -# Windows unit test and integration test -- template: job_templates/olive-build-template.yaml +# Windows unit test +- template: job_templates/olive-test-template.yaml parameters: name: Windows_CI pool: $(OLIVE_POOL_WIN2019) + device: 'cpu' windows: True + test_type: 'unit_test' onnxruntime: ort-nightly # Linux examples test diff --git a/examples/test/test_resnet_vitis_ai_ptq_cpu.py b/examples/test/test_resnet_vitis_ai_ptq_cpu.py index 4e29eb819..c0fa13ec4 100644 --- a/examples/test/test_resnet_vitis_ai_ptq_cpu.py +++ b/examples/test/test_resnet_vitis_ai_ptq_cpu.py @@ -34,7 +34,7 @@ def setup(): @pytest.mark.parametrize("olive_json", ["resnet_vitis_ai_ptq_cpu.json"]) @pytest.mark.skipif( version.parse(OrtVersion) == version.parse("1.16.0"), - reason="resnet is not supported in ORT 1.16.0 caused by https://github.com/microsoft/onnxruntime/issues/17627", + reason="VitisAIQuantization is not supported in ORT 1.16.0 with TensorsData", ) def test_resnet(search_algorithm, execution_order, system, olive_json): # TODO: add gpu e2e test From 4742dbd37fa1483eb70800a0258ebc4cf48f12fc Mon Sep 17 00:00:00 2001 From: Mike Guo Date: Fri, 22 Sep 2023 18:34:09 +0800 Subject: [PATCH 25/25] cleanup examples for nightly --- .../job_templates/olive-test-template.yaml | 2 +- .azure_pipelines/olive-ort-last.yaml | 10 ++-------- .azure_pipelines/olive-ort-nightly.yaml | 10 ++-------- 3 files changed, 5 insertions(+), 17 deletions(-) diff --git a/.azure_pipelines/job_templates/olive-test-template.yaml b/.azure_pipelines/job_templates/olive-test-template.yaml index c450e309e..77c35574f 100644 --- a/.azure_pipelines/job_templates/olive-test-template.yaml +++ b/.azure_pipelines/job_templates/olive-test-template.yaml @@ -8,7 +8,7 @@ parameters: jobs: - ${{ if eq(parameters.device, 'cpu') }}: - - job: ${{parameters.name}}_Test_Olive + - job: ${{parameters.name}} timeoutInMinutes: 300 pool: name: ${{ parameters.pool}} diff --git a/.azure_pipelines/olive-ort-last.yaml b/.azure_pipelines/olive-ort-last.yaml index 623bc5f51..9782871bf 100644 --- a/.azure_pipelines/olive-ort-last.yaml +++ b/.azure_pipelines/olive-ort-last.yaml @@ -16,7 +16,7 @@ jobs: # Linux unit test - template: job_templates/olive-test-template.yaml parameters: - name: Linux_CI + name: Linux_CI_Unit_Test_Olive pool: $(OLIVE_POOL_UBUNTU2004) device: 'cpu' windows: False @@ -26,7 +26,7 @@ jobs: # Windows unit test - template: job_templates/olive-test-template.yaml parameters: - name: Windows_CI + name: Windows_CI_Unit_Test_Olive pool: $(OLIVE_POOL_WIN2019) device: 'cpu' windows: True @@ -43,9 +43,6 @@ jobs: bert_ptq_cpu: exampleFolder: bert exampleName: bert_ptq_cpu - cifar10_openvino_intel_hw: - exampleFolder: cifar10_openvino_intel_hw - exampleName: cifar10_openvino_intel_hw resnet_ptq_cpu: exampleFolder: resnet exampleName: resnet_ptq_cpu @@ -69,9 +66,6 @@ jobs: bert_ptq_cpu: exampleFolder: bert exampleName: bert_ptq_cpu - cifar10_openvino_intel_hw: - exampleFolder: cifar10_openvino_intel_hw - exampleName: cifar10_openvino_intel_hw resnet_ptq_cpu: exampleFolder: resnet exampleName: resnet_ptq_cpu diff --git a/.azure_pipelines/olive-ort-nightly.yaml b/.azure_pipelines/olive-ort-nightly.yaml index 6df94e8ab..55f05dcc5 100644 --- a/.azure_pipelines/olive-ort-nightly.yaml +++ b/.azure_pipelines/olive-ort-nightly.yaml @@ -16,7 +16,7 @@ jobs: # Linux unit test - template: job_templates/olive-test-template.yaml parameters: - name: Linux_CI + name: Linux_CI_Unit_Test_Olive pool: $(OLIVE_POOL_UBUNTU2004) device: 'cpu' windows: False @@ -26,7 +26,7 @@ jobs: # Windows unit test - template: job_templates/olive-test-template.yaml parameters: - name: Windows_CI + name: Windows_CI_Unit_Test_Olive pool: $(OLIVE_POOL_WIN2019) device: 'cpu' windows: True @@ -43,9 +43,6 @@ jobs: bert_ptq_cpu: exampleFolder: bert exampleName: bert_ptq_cpu - cifar10_openvino_intel_hw: - exampleFolder: cifar10_openvino_intel_hw - exampleName: cifar10_openvino_intel_hw resnet_ptq_cpu: exampleFolder: resnet exampleName: resnet_ptq_cpu @@ -69,9 +66,6 @@ jobs: bert_ptq_cpu: exampleFolder: bert exampleName: bert_ptq_cpu - cifar10_openvino_intel_hw: - exampleFolder: cifar10_openvino_intel_hw - exampleName: cifar10_openvino_intel_hw resnet_ptq_cpu: exampleFolder: resnet exampleName: resnet_ptq_cpu