install the onnxruntime explicitly to support ORT 1.15 and nightly build

.azure_pipelines/job_templates/olive-build-doc-template.yaml

@@ -27,7 +27,7 @@ jobs:
       displayName: Checkout release branch
       condition: ne('${{parameters.doc_version}}', 'latest')
 
-    - script: make install-olive INSTALL_EXTRAS=[cpu]
+    - script: pip install .[cpu]
       displayName: Install Olive
 
     - script: |

.azure_pipelines/job_templates/olive-build-template.yaml

@@ -4,6 +4,7 @@ parameters:
   name: ''
   pool: ''
   device: 'cpu'
+  onnxruntime: 'onnxruntime'
 
 jobs:
 - template: olive-test-template.yaml
@@ -13,6 +14,7 @@ jobs:
     device: ${{parameters.device}}
     WINDOWS: ${{parameters.windows}}
     test_type: 'unit_test'
+    onnxruntime: ${{parameters.onnxruntime}}
 
 - template: olive-test-template.yaml
   parameters:
@@ -21,6 +23,7 @@ jobs:
     device: ${{parameters.device}}
     WINDOWS: ${{parameters.windows}}
     test_type: 'integ_test'
+    onnxruntime: ${{parameters.onnxruntime}}
 
 - job: ${{parameters.name}}_Test_Examples
   timeoutInMinutes: 300
@@ -40,14 +43,28 @@ jobs:
       versionSpec: 3.8
     displayName: Use Python 3.8
 
-  - script: make install-olive INSTALL_EXTRAS=[$(device)]
+  - script: pip install .
     displayName: Install Olive
 
+  - ${{ if startsWith(parameters.onnxruntime, 'ort-nightly') }}:
+    - script: |
+        pip install onnxruntime
+        pip uninstall -y onnxruntime
+        pip install ${{ parameters.onnxruntime }} --index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/pypi/simple/
+      displayName: Install ${{ parameters.onnxruntime }}
+  - ${{ else }}:
+    - script: |
+        pip install ${{ parameters.onnxruntime }}
+      displayName: Install ${{ parameters.onnxruntime }}
+
   - task: AzureCLI@1
     inputs:
       azureSubscription: $(OLIVE_RG_SERVICE_CONNECTION)
       scriptLocation: 'inlineScript'
-      inlineScript: make test-examples
+      inlineScript: |
+        python -m pip install pytest
+        python -m pip install -r $(Build.SourcesDirectory)/examples/$(exampleFolder)/requirements.txt
+        python -m pytest -v -s --log-cli-level=WARNING --junitxml=$(Build.SourcesDirectory)/logs/test_examples-TestOlive.xml $(Build.SourcesDirectory)/examples/test/test_$(exampleName).py
     displayName: Test Examples
     env:
       OLIVEWHEELS_STORAGE_CONNECTION_STRING: $(olive-wheels-storage-connection-string)
@@ -57,8 +74,6 @@ jobs:
       AZURE_TENANT_ID: $(azure-tenant-id)
       AZURE_CLIENT_ID: $(olive-rg-sp-id)
       AZURE_CLIENT_SECRET: $(olive-rg-sp-secret)
-      EXAMPLE_FOLDER: $(exampleFolder)
-      EXAMPLE_NAME: $(exampleName)
 
   - task: ComponentGovernanceComponentDetection@0
     inputs:
@@ -74,7 +89,7 @@ jobs:
       testRunTitle: '$(Build.BuildNumber)[$(Agent.JobName)]'
     displayName: Upload pipeline run test results
 
-  - script: make clean WINDOWS=$(WINDOWS)
+  - script: git clean -dfX
     condition: always()
     displayName: Clean remaining artifacts
 

.azure_pipelines/job_templates/olive-test-template.yaml

@@ -4,6 +4,7 @@ parameters:
   test_type: ''
   device: 'cpu'
   python_version: '3.8'
+  onnxruntime: 'onnxruntime'
 
 jobs:
 - ${{ if eq(parameters.device, 'cpu') }}:
@@ -29,23 +30,64 @@ jobs:
       condition: and(eq(variables.WINDOWS, 'False'), eq(variables.testType, 'integ_test'))
       continueOnError: true
 
-    - script: make install-olive INSTALL_EXTRAS=[$(device)]
+    - script: |
+        pip install .
       displayName: Install Olive
 
-    - task: AzureCLI@1
-      inputs:
-        azureSubscription: $(OLIVE_RG_SERVICE_CONNECTION)
-        scriptLocation: 'inlineScript'
-        inlineScript: make $(testType)
-      displayName: Test Olive
-      env:
-        OLIVEWHEELS_STORAGE_CONNECTION_STRING: $(olive-wheels-storage-connection-string)
-        WORKSPACE_SUBSCRIPTION_ID: $(workspace-subscription-id)
-        WORKSPACE_RESOURCE_GROUP: $(workspace-resource-group)
-        WORKSPACE_NAME: $(workspace-name)
-        AZURE_TENANT_ID: $(azure-tenant-id)
-        AZURE_CLIENT_ID: $(olive-rg-sp-id)
-        AZURE_CLIENT_SECRET: $(olive-rg-sp-secret)
+    - ${{ if startsWith(parameters.onnxruntime, 'ort-nightly') }}:
+      - script: |
+          pip install onnxruntime
+          pip uninstall -y onnxruntime
+          pip install ${{ parameters.onnxruntime }} --index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/pypi/simple/
+        displayName: Install ${{ parameters.onnxruntime }}
+    - ${{ else }}:
+      - script: |
+          pip install ${{ parameters.onnxruntime }}
+        displayName: Install ${{ parameters.onnxruntime }}
+
+    - ${{ if and(eq(variables.WINDOWS, 'True'), eq(variables.testType, 'multiple_ep')) }}:
+      - task: AzureCLI@1
+        inputs:
+          azureSubscription: $(OLIVE_RG_SERVICE_CONNECTION)
+          scriptLocation: 'inlineScript'
+          inlineScript: |
+            call python -m pip install pytest
+            call curl --output openvino_toolkit.zip https://storage.openvinotoolkit.org/repositories/openvino/packages/2023.0.1/windows/w_openvino_toolkit_windows_2023.0.1.11005.fa1c41994f3_x86_64.zip
+            call 7z x openvino_toolkit.zip
+            call w_openvino_toolkit_windows_2023.0.1.11005.fa1c41994f3_x86_64\\setupvars.bat
+            call python -m pip install numpy psutil coverage protobuf==3.20.3
+            call coverage run --source=$(Build.SourcesDirectory)/olive -m pytest -v -s --log-cli-level=WARNING --junitxml=$(Build.SourcesDirectory)/logs/test-TestOlive.xml $(Build.SourcesDirectory)/test/$(testType)
+            call coverage xml
+        displayName: Test Olive
+        env:
+          OLIVEWHEELS_STORAGE_CONNECTION_STRING: $(olive-wheels-storage-connection-string)
+          WORKSPACE_SUBSCRIPTION_ID: $(workspace-subscription-id)
+          WORKSPACE_RESOURCE_GROUP: $(workspace-resource-group)
+          WORKSPACE_NAME: $(workspace-name)
+          AZURE_TENANT_ID: $(azure-tenant-id)
+          AZURE_CLIENT_ID: $(olive-rg-sp-id)
+          AZURE_CLIENT_SECRET: $(olive-rg-sp-secret)
+    - ${{ else }}:
+      - task: AzureCLI@1
+        inputs:
+          azureSubscription: $(OLIVE_RG_SERVICE_CONNECTION)
+          scriptLocation: 'inlineScript'
+          inlineScript: |
+            python -m pip install pytest
+            python -m pip install -r $(Build.SourcesDirectory)/test/requirements-test.txt
+
+            coverage run --source=$(Build.SourcesDirectory)/olive -m pytest -v -s --log-cli-level=WARNING --junitxml=$(Build.SourcesDirectory)/logs/test-TestOlive.xml $(Build.SourcesDirectory)/test/$(testType)
+            coverage xml
+
+        displayName: Test Olive
+        env:
+          OLIVEWHEELS_STORAGE_CONNECTION_STRING: $(olive-wheels-storage-connection-string)
+          WORKSPACE_SUBSCRIPTION_ID: $(workspace-subscription-id)
+          WORKSPACE_RESOURCE_GROUP: $(workspace-resource-group)
+          WORKSPACE_NAME: $(workspace-name)
+          AZURE_TENANT_ID: $(azure-tenant-id)
+          AZURE_CLIENT_ID: $(olive-rg-sp-id)
+          AZURE_CLIENT_SECRET: $(olive-rg-sp-secret)
 
     - task: CredScan@3
       displayName: 'Run CredScan'
@@ -72,6 +114,6 @@ jobs:
         codeCoverageTool: 'Cobertura'
         summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml'
 
-    - script: make clean WINDOWS=$(WINDOWS)
+    - script: git clean -dfX
       condition: always()
       displayName: Clean remaining artifacts

.azure_pipelines/olive-ci.yaml

@@ -32,6 +32,7 @@ jobs:
     name: Linux_CI
     pool: $(OLIVE_POOL_UBUNTU2004)
     windows: False
+    onnxruntime: onnxruntime
     examples:
       bert_ptq_cpu:
         exampleFolder: bert
@@ -63,6 +64,7 @@ jobs:
     name: Windows_CI
     pool: $(OLIVE_POOL_WIN2019)
     windows: True
+    onnxruntime: onnxruntime
     examples:
       bert_ptq_cpu:
         exampleFolder: bert
@@ -89,6 +91,7 @@ jobs:
     pool: $(OLIVE_POOL_UBUNTU2004)
     windows: False
     device: gpu
+    onnxruntime: onnxruntime-gpu
     examples:
       bert_cuda_gpu:
         exampleFolder: bert

.azure_pipelines/olive-ort-nightly.yaml

@@ -0,0 +1,84 @@
+trigger: none
+
+pr: none
+
+schedules:
+- cron: '0 0 * * *'
+  displayName: Daily build
+  branches:
+    include:
+    - main
+
+variables:
+    ComponentDetection.Timeout: 2400
+
+jobs:
+- template: job_templates/olive-build-template.yaml
+  parameters:
+    name: Linux_CI
+    pool: $(OLIVE_POOL_UBUNTU2004)
+    windows: False
+    onnxruntime: ort-nightly
+    examples:
+      bert_ptq_cpu:
+        exampleFolder: bert
+        exampleName: bert_ptq_cpu
+      bert_ptq_cpu_aml:
+        exampleFolder: bert
+        exampleName: bert_ptq_cpu_aml
+      bert_ptq_cpu_docker:
+        exampleFolder: bert
+        exampleName: bert_ptq_cpu_docker
+      cifar10_openvino_intel_hw:
+        exampleFolder: cifar10_openvino_intel_hw
+        exampleName: cifar10_openvino_intel_hw
+      resnet_ptq_cpu:
+        exampleFolder: resnet
+        exampleName: resnet_ptq_cpu
+      resnet_vitis_ai_ptq_cpu:
+        exampleFolder: resnet
+        exampleName: resnet_vitis_ai_ptq_cpu
+      resnet_qat:
+        exampleFolder: resnet
+        exampleName: resnet_qat
+      whisper:
+        exampleFolder: whisper
+        exampleName: whisper
+
+- template: job_templates/olive-build-template.yaml
+  parameters:
+    name: Windows_CI
+    pool: $(OLIVE_POOL_WIN2019)
+    windows: True
+    onnxruntime: ort-nightly
+    examples:
+      bert_ptq_cpu:
+        exampleFolder: bert
+        exampleName: bert_ptq_cpu
+      bert_ptq_cpu_aml:
+        exampleFolder: bert
+        exampleName: bert_ptq_cpu_aml
+      cifar10_openvino_intel_hw:
+        exampleFolder: cifar10_openvino_intel_hw
+        exampleName: cifar10_openvino_intel_hw
+      resnet_ptq_cpu:
+        exampleFolder: resnet
+        exampleName: resnet_ptq_cpu
+      resnet_qat:
+        exampleFolder: resnet
+        exampleName: resnet_qat
+      whisper:
+        exampleFolder: whisper
+        exampleName: whisper
+
+- template: job_templates/olive-build-template.yaml
+  parameters:
+    name: Linux_GPU_CI
+    pool: $(OLIVE_POOL_UBUNTU2004)
+    windows: False
+    device: gpu
+    onnxruntime: ort-nightly-gpu
+    examples:
+      bert_cuda_gpu:
+        exampleFolder: bert
+        exampleName: bert_cuda_gpu

examples/resnet/requirements.txt

@@ -1,7 +1,6 @@
 azure-ai-ml
 azure-identity
 azureml-fsspec
-onnxruntime<=1.15.1
 pytorch-lightning
 scipy
 tabulate

examples/test/test_resnet_ptq_cpu.py

@@ -6,6 +6,8 @@
 from pathlib import Path
 
 import pytest
+from onnxruntime import __version__ as OrtVersion
+from packaging import version
 from utils import check_output, patch_config
 
 from olive.common.utils import retry_func, run_subprocess
@@ -30,6 +32,10 @@ def setup():
 @pytest.mark.parametrize("execution_order", ["pass-by-pass"])
 @pytest.mark.parametrize("system", ["local_system", "aml_system"])
 @pytest.mark.parametrize("olive_json", ["resnet_ptq_cpu.json", "resnet_ptq_cpu_aml_dataset.json"])
+@pytest.mark.skipif(
+    version.parse(OrtVersion) == version.parse("1.16.0"),
+    reason="resnet is not supported in ORT 1.16.0 caused by https://github.com/microsoft/onnxruntime/issues/17627",
+)
 def test_resnet(search_algorithm, execution_order, system, olive_json):
     # TODO: add gpu e2e test
     from olive.workflows import run as olive_run

examples/test/test_resnet_vitis_ai_ptq_cpu.py

@@ -33,8 +33,8 @@ def setup():
 @pytest.mark.parametrize("system", ["local_system", "aml_system"])
 @pytest.mark.parametrize("olive_json", ["resnet_vitis_ai_ptq_cpu.json"])
 @pytest.mark.skipif(
-    version.parse(OrtVersion) >= version.parse("1.16.0"),
-    reason="VitisAIQuantization is not supported in ORT 1.16.0 with TensorsData",
+    version.parse(OrtVersion) == version.parse("1.16.0"),
+    reason="resnet is not supported in ORT 1.16.0 caused by https://github.com/microsoft/onnxruntime/issues/17627",
 )
 def test_resnet(search_algorithm, execution_order, system, olive_json):
     # TODO: add gpu e2e test

examples/whisper/requirements.txt

@@ -1,6 +1,5 @@
 neural-compressor
 onnx==1.14.0
-onnxruntime>=1.15.0
 onnxruntime-extensions==0.8.0
 tabulate
 torch>=1.13.1

test/unit_test/passes/vitis_ai/test_vitis_ai_quantization.py

@@ -38,7 +38,7 @@ def dummy_calibration_reader(data_dir=None, batch_size=1, *args, **kwargs):
 
 
 @pytest.mark.skipif(
-    version.parse(OrtVersion) >= version.parse("1.16.0"),
+    version.parse(OrtVersion) == version.parse("1.16.0"),
     reason="VitisAIQuantization is not supported in ORT 1.16.0 with TensorsData",
 )
 def test_vitis_ai_quantization_pass(tmp_path):