Merged PR 5411855: Manual pull from onnxruntime upstream GitHub maste…

…r into DmlDev I'm just doing this manually this once because some commits were squashed, and I want to be sure they merge cleanly for the nightly automatic merge. "git merge upstream/master"
microsoft · fdwr · Nov 20, 2021 · Sep 21, 2020 · Sep 22, 2020 · Sep 22, 2020
commit 502f85c8839a7b48d0984a405e6ec810e3e1841b
diff --git a/BUILD.md b/BUILD.md
@@ -308,9 +308,23 @@ These instructions are for JetPack SDK 4.4.
 See more information on DNNL and MKL-ML [here](./docs/execution_providers/DNNL-ExecutionProvider.md).
 
 #### Build Instructions
+The DNNL execution provider can be built for Intel CPU or GPU. To build for Intel GPU, install [Intel SDK for OpenCL Applications](https://software.intel.com/content/www/us/en/develop/tools/opencl-sdk.html). Install the latest GPU driver - [Windows graphics driver](https://downloadcenter.intel.com/product/80939/Graphics), [Linux graphics compute runtime and OpenCL driver](https://github.com/intel/compute-runtime/releases).
+
+##### Windows
+`.\build.bat --use_dnnl`
+
+##### Linux
+`./build.sh --use_dnnl`
+
+To build for Intel GPU, replace dnnl_opencl_root with the path of the Intel SDK for OpenCL Applications.
+
+##### Windows 
+
+`.\build.bat --use_dnnl --dnnl_gpu_runtime ocl --dnnl_opencl_root "c:\program files (x86)\intelswtools\sw_dev_tools\opencl\sdk"`
+
 ##### Linux
 
-DNNL: `./build.sh --use_dnnl`
+`./build.sh --use_dnnl --dnnl_gpu_runtime ocl --dnnl_opencl_root "/opt/intel/sw_dev_tools/opencl-sdk"`
 
 ---
 
@@ -405,16 +419,18 @@ See more information on the OpenVINO Execution Provider [here](./docs/execution_
 | <code>VAD-M_FP16</code> | Intel<sup>®</sup> Vision Accelerator Design based on 8 Movidius<sup>TM</sup> MyriadX VPUs |
 | <code>VAD-F_FP32</code> | Intel<sup>®</sup> Vision Accelerator Design with an Intel<sup>®</sup> Arria<sup>®</sup> 10 FPGA |
 | <code>HETERO:<DEVICE_TYPE_1>,<DEVICE_TYPE_2>,<DEVICE_TYPE_3>...</code> | All Intel<sup>®</sup> silicons mentioned above |
+| <code>MULTI:<DEVICE_TYPE_1>,<DEVICE_TYPE_2>,<DEVICE_TYPE_3>...</code> | All Intel<sup>®</sup> silicons mentioned above |
 
-Specifying Hardware Target for HETERO Build:
+Specifying Hardware Target for HETERO or Multi-Device Build:
 
 HETERO:<DEVICE_TYPE_1>,<DEVICE_TYPE_2>,<DEVICE_TYPE_3>...
+MULTI:<DEVICE_TYPE_1>,<DEVICE_TYPE_2>,<DEVICE_TYPE_3>...
 The <DEVICE_TYPE> can be any of these devices from this list ['CPU','GPU','MYRIAD','FPGA','HDDL']
 
-A minimum of two DEVICE_TYPE'S should be specified for a valid HETERO Build.
+A minimum of two DEVICE_TYPE'S should be specified for a valid HETERO or Multi-Device Build.
 
 Example:
-HETERO:MYRIAD,CPU        HETERO:HDDL,GPU,CPU
+HETERO:MYRIAD,CPU  HETERO:HDDL,GPU,CPU  MULTI:MYRIAD,GPU,CPU
 
 For more information on OpenVINO Execution Provider&#39;s ONNX Layer support, Topology support, and Intel hardware enabled, please refer to the document [OpenVINO-ExecutionProvider.md](./docs/execution_providers/OpenVINO-ExecutionProvider.md) in <code>$onnxruntime_root/docs/execution_providers</code>
 
@@ -547,29 +563,24 @@ onnxruntime_perf_test
 onnxruntime_test_all
 ```
 
-#### Build Instructions (Jetson Nano)
+#### Native Build Instructions (validated on Jetson Nano and Jetson Xavier)
 
 1. Build ACL Library (skip if already built)
 ```
 cd ~
-git clone https://github.com/Arm-software/ComputeLibrary.git
+git clone -b v20.02 https://github.com/Arm-software/ComputeLibrary.git
 cd ComputeLibrary
-sudo apt install scons
-sudo apt install g++-arm-linux-gnueabihf
+sudo apt-get install -y scons g++-arm-linux-gnueabihf
 scons -j8 arch=arm64-v8a  Werror=1 debug=0 asserts=0 neon=1 opencl=1 examples=1 build=native
 ```
-2. Set environment variables to set include directory and shared object library path.
-```
-export CPATH=~/ComputeLibrary/include/:~/ComputeLibrary/
-export LD_LIBRARY_PATH=~/ComputeLibrary/build/
-```
-3. Build onnxruntime with --use_acl flag
-```
-./build.sh --use_acl
-```
-To use a library outside the normal environment you can set a custom path by using --acl_home and --acl_libs tags that defines the path to the ComputeLibrary directory and the build directory respectively.
+
+2. Cmake is needed to build ONNX Runtime. Because the minimum required version is 3.13,
+   it is necessary to build CMake from source. Download Unix/Linux sources from https://cmake.org/download/
+   and follow https://cmake.org/install/ to build from source. Version 3.17.5 and 3.18.4 have been tested on Jetson.
+
+3. Build onnxruntime with --use_acl flag with one of the supported ACL version flags. (ACL_1902 | ACL_1905 | ACL_1908 | ACL_2002)
 ```
-./build.sh --use_acl --acl_home /path/to/ComputeLibrary --acl_libs /path/to/build
+./build.sh --config RelWithDebInfo --use_acl ACL_2002 --update --build --build_wheel --parallel --acl_home ~/ComputeLibrary --acl_libs ~/ComputeLibrary/build
 ```
 
 ---

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -25,7 +25,6 @@ Note that if an issue isn't a high priority or has many open questions then it m
 The ONNX Runtime team will review the proposal and either approve or close the issue based on whether it broadly aligns with the [Onnx Runtime Roadmap - High Level Goals section](./docs/Roadmap.md) and contribution guidelines.
 
 4. **API Review**  
-
 If the feature adds new APIs then we'll start an API review. All new public APIs must be reviewed before merging. 
 For making changes to the C API refer to guidance [here](./docs/C_API_Guidelines.md).
 For making changes to the Windows Machine Learning WinRT API, please label your request as *api:WinML*, and someone from the Windows ML team will work with you. 

diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt
@@ -128,7 +128,7 @@ option(onnxruntime_ENABLE_NVTX_PROFILE "Enable NVTX profile." OFF)
 option(onnxruntime_ENABLE_TRAINING "Enable training functionality." OFF)
 option(onnxruntime_ENABLE_TRAINING_E2E_TESTS "Enable training end-to-end tests." OFF)
 option(onnxruntime_USE_HOROVOD "Build with HOROVOD support" OFF)
-option(onnxruntime_USE_NCCL "Build with NCCL support" ON)
+option(onnxruntime_USE_NCCL "Build with NCCL support" OFF)
 #A special option just for debugging and sanitize check. Please do not enable in option in retail builds.
 #The option has no effect on Windows.
 option(onnxruntime_USE_VALGRIND "Build with valgrind hacks" OFF)
@@ -975,8 +975,8 @@ if(onnxruntime_USE_OPENVINO)
 
   add_definitions(-DUSE_OPENVINO=1)
 
-  if (EXISTS "$ENV{INTEL_OPENVINO_DIR}/inference_engine/version.txt")
-    file(READ $ENV{INTEL_OPENVINO_DIR}/inference_engine/version.txt VER)
+  if (EXISTS "$ENV{INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/version.txt")
+    file(READ $ENV{INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/version.txt VER)
   endif()
 
   if (NOT DEFINED ENV{INTEL_OPENVINO_DIR})
@@ -1037,6 +1037,11 @@ if(onnxruntime_USE_OPENVINO)
     add_definitions(-DDEVICE_NAME="${onnxruntime_USE_OPENVINO_DEVICE}")
   endif()
 
+  if(onnxruntime_USE_OPENVINO_MULTI)
+    add_definitions(-DOPENVINO_CONFIG_MULTI=1)
+    add_definitions(-DDEVICE_NAME="${onnxruntime_USE_OPENVINO_DEVICE}")
+  endif()
+
 endif()
 
 if (onnxruntime_USE_VITISAI)
@@ -1216,6 +1221,7 @@ if (onnxruntime_ENABLE_TRAINING)
     include(FindPackageHandleStandardArgs)
     find_package_handle_standard_args(MPI DEFAULT_MSG MPI_INCLUDE_DIR MPI_LIBRARY)
 
+    set(onnxruntime_USE_MPI ON)
     if (MPI_FOUND)
       if(NOT DEFINED onnxruntime_MPI_HOME)
         execute_process(COMMAND mpirun --version OUTPUT_VARIABLE MPIRUN_OUTPUT)
@@ -1231,7 +1237,14 @@ if (onnxruntime_ENABLE_TRAINING)
       mark_as_advanced(MPI_INCLUDE_DIRS MPI_LIBRARIES)
 
       list(APPEND onnxruntime_EXTERNAL_LIBRARIES ${MPI_LIBRARIES} ${MPI_CXX_LINK_FLAGS})
+
+      # Enable NCCL only when using CUDA or ROCM.
+      if(${onnxruntime_USE_CUDA} OR ${onnxruntime_USE_ROCM})
+        set(onnxruntime_USE_NCCL ON)
+      endif()
+
     else ()
+      set(onnxruntime_USE_MPI OFF)
       set(onnxruntime_USE_NCCL OFF)
       set(onnxruntime_USE_HOROVOD OFF)
       message( WARNING "MPI is not found. Please use --mpi_home to specify the path of MPI. Otherwise, NCCL or HOROVOD will be disabled." )
@@ -1302,6 +1315,10 @@ if (onnxruntime_ENABLE_TRAINING)
     endif()
   endif()
 
+  if (onnxruntime_USE_MPI AND MPI_FOUND)
+    add_definitions(-DUSE_MPI=1)
+  endif()
+
   if (onnxruntime_USE_HOROVOD AND MPI_FOUND)
     if (WIN32)
       message( FATAL_ERROR "Horovod is not supported on Windows." )

diff --git a/cmake/external/dnnl.cmake b/cmake/external/dnnl.cmake
@@ -1,8 +1,8 @@
 include (ExternalProject)
 
-set(DNNL_URL https://github.com/intel/mkl-dnn.git)
-# If DNNL_TAG is updated, check if platform.cmake.patch need to be updated.
-set(DNNL_TAG v1.1.1)
+set(DNNL_URL https://github.com/oneapi-src/onednn)
+# If DNNL_TAG is updated, check if MKLML_VERSION and platform.cmake.patch need to be updated.
+set(DNNL_TAG v1.7)
 
 if(WIN32)
   set(MKLML_OS_VERSION_STR "win")
@@ -20,6 +20,14 @@ else()
   endif()  
 endif()
 
+if (onnxruntime_USE_DNNL AND onnxruntime_DNNL_GPU_RUNTIME STREQUAL "ocl" AND onnxruntime_DNNL_OPENCL_ROOT STREQUAL "")
+  message(FATAL_ERROR "onnxruntime_DNNL_OPENCL_ROOT required for onnxruntime_DNNL_GPU_RUNTIME")
+elseif(onnxruntime_USE_DNNL AND onnxruntime_DNNL_GPU_RUNTIME STREQUAL "ocl")
+  file(TO_CMAKE_PATH ${onnxruntime_DNNL_OPENCL_ROOT} onnxruntime_DNNL_OPENCL_ROOT)
+  set(DNNL_OCL_INCLUDE_DIR ${onnxruntime_DNNL_OPENCL_ROOT}/include)
+  set(DNNL_GPU_CMAKE_ARGS "-DDNNL_GPU_RUNTIME=OCL " "-DOPENCLROOT=${onnxruntime_DNNL_OPENCL_ROOT}")
+endif()
+
 if (onnxruntime_USE_DNNL)
   set(DNNL_SOURCE ${CMAKE_CURRENT_BINARY_DIR}/dnnl/src/dnnl/src)
   set(DNNL_INSTALL ${CMAKE_CURRENT_BINARY_DIR}/dnnl/install)
@@ -46,7 +54,7 @@ if (onnxruntime_USE_DNNL)
     GIT_TAG ${DNNL_TAG}
     # PATCH_COMMAND ${MKLDNN_PATCH_DISCARD_COMMAND} COMMAND ${DNNL_PATCH_COMMAND}
     SOURCE_DIR ${DNNL_SOURCE}
-    CMAKE_ARGS -DDNNL_BUILD_TESTS=OFF -DDNNL_BUILD_EXAMPLES=OFF -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${DNNL_INSTALL}
+    CMAKE_ARGS -DDNNL_BUILD_TESTS=OFF -DDNNL_BUILD_EXAMPLES=OFF -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${DNNL_INSTALL} ${DNNL_GPU_CMAKE_ARGS}
   )
   link_directories(${DNNL_LIB_DIR})
 endif()
diff --git a/cmake/external/onnx_minimal.cmake b/cmake/external/onnx_minimal.cmake
@@ -14,7 +14,7 @@ endif()
 
 set(ONNX_SOURCE_ROOT ${PROJECT_SOURCE_DIR}/external/onnx)
 
-add_library(onnx_proto ${ONNX_SOURCE_ROOT}/onnx/onnx-ml.proto ${ONNX_SOURCE_ROOT}/onnx/onnx-operators-ml.proto)
+add_library(onnx_proto ${ONNX_SOURCE_ROOT}/onnx/onnx-ml.proto ${ONNX_SOURCE_ROOT}/onnx/onnx-operators-ml.proto ${ONNX_SOURCE_ROOT}/onnx/onnx-data.proto)
 
 target_include_directories(onnx_proto PUBLIC $<TARGET_PROPERTY:protobuf::libprotobuf,INTERFACE_INCLUDE_DIRECTORIES> "${CMAKE_CURRENT_BINARY_DIR}")
 target_compile_definitions(onnx_proto PUBLIC $<TARGET_PROPERTY:protobuf::libprotobuf,INTERFACE_COMPILE_DEFINITIONS>)

diff --git a/cmake/onnxruntime_mlas.cmake b/cmake/onnxruntime_mlas.cmake
@@ -20,6 +20,7 @@ set(mlas_common_srcs
   ${ONNXRUNTIME_ROOT}/core/mlas/lib/quantize.cpp
   ${ONNXRUNTIME_ROOT}/core/mlas/lib/qladd.cpp
   ${ONNXRUNTIME_ROOT}/core/mlas/lib/qlmul.cpp
+  ${ONNXRUNTIME_ROOT}/core/mlas/lib/qpostprocessor.cpp
 )
 
 if(MSVC)