Skip to content

Commit

Permalink
webcodecs: Activate GPU side resizing for video encoding on Windows
Browse files Browse the repository at this point in the history 
There are 2 very different code paths that are called GPU side resizing:

1. Frames backed by D3D11 texture are scaled by PerformD3DScaling(),
   such frames usually come from web-cameras and this is supposed to be
   the most efficient way of scaling.

2. Frames backed by mappable GPU memory buffers in I420/NV12 formats
   are scaled by libyuv/ConvertAndScaleFrame() while being copied to
   CPU memory. This should also be more efficient than the status quo
   because we eliminate one extra memcpy from the GPU buffer to
   a mojo buffer. For example such frames are coming from
   capturing offscreen canvas.

Bug: 1212955
Change-Id: I4ca62555736c009cc4f1893158759ee4f93430cf
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3612726
Reviewed-by: Dale Curtis <dalecurtis@chromium.org>
Commit-Queue: Eugene Zemtsov <eugene@chromium.org>
Cr-Commit-Position: refs/heads/main@{#997474}
  • Loading branch information
@Djuffin
Djuffin authored and Chromium LUCI CQ committed Apr 28, 2022
1 parent 427ad92 commit 2f7b3c5
Show file tree
Hide file tree
Showing 3 changed files with 33 additions and 60 deletions.
88 changes: 29 additions & 59 deletions media/gpu/windows/media_foundation_video_encode_accelerator_win.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
#include "media/base/media_log.h"
#include "media/base/media_switches.h"
#include "media/base/video_frame.h"
#include "media/base/video_util.h"
#include "media/base/win/mf_helpers.h"
#include "media/base/win/mf_initializer.h"
#include "media/gpu/gpu_video_encode_accelerator_helpers.h"
Expand Down Expand Up @@ -1029,17 +1030,8 @@ HRESULT MediaFoundationVideoEncodeAccelerator::PopulateInputSampleBuffer(
return MF_E_INVALID_STREAM_DATA;
}

const uint8_t* src_y = nullptr;
const uint8_t* src_uv = nullptr;
base::ScopedClosureRunner scoped_unmap_gmb;

if (frame->storage_type() ==
VideoFrame::StorageType::STORAGE_GPU_MEMORY_BUFFER) {
if (frame->format() != PIXEL_FORMAT_NV12) {
DLOG(ERROR) << "GMB video frame is not NV12";
return MF_E_INVALID_STREAM_DATA;
}

gfx::GpuMemoryBuffer* gmb = frame->GetGpuMemoryBuffer();
if (!gmb) {
DLOG(ERROR) << "Failed to get GMB for input frame";
Expand All @@ -1057,32 +1049,28 @@ HRESULT MediaFoundationVideoEncodeAccelerator::PopulateInputSampleBuffer(
return PopulateInputSampleBufferGpu(std::move(frame));
}

// Shared memory GMB case.
if (!gmb->Map()) {
// ConvertToMemoryMappedFrame() doesn't copy pixel data,
// it just maps GPU buffer owned by |frame| and presents it as mapped
// view in CPU memory. |frame| will unmap the buffer when destructed.
frame = ConvertToMemoryMappedFrame(std::move(frame));
if (!frame) {
DLOG(ERROR) << "Failed to map shared memory GMB";
return E_FAIL;
}

scoped_unmap_gmb.ReplaceClosure(
base::BindOnce([](gfx::GpuMemoryBuffer* gmb) { gmb->Unmap(); }, gmb));

src_y = reinterpret_cast<const uint8_t*>(gmb->memory(VideoFrame::kYPlane));
src_uv =
reinterpret_cast<const uint8_t*>(gmb->memory(VideoFrame::kUVPlane));
}

const auto kTargetPixelFormat = PIXEL_FORMAT_NV12;
Microsoft::WRL::ComPtr<IMFMediaBuffer> input_buffer;
HRESULT hr = input_sample_->GetBufferByIndex(0, &input_buffer);
if (FAILED(hr)) {
// Allocate a new buffer.
MFT_INPUT_STREAM_INFO input_stream_info;
hr = encoder_->GetInputStreamInfo(input_stream_id_, &input_stream_info);
RETURN_ON_HR_FAILURE(hr, "Couldn't get input stream info", hr);

hr = MFCreateAlignedMemoryBuffer(
input_stream_info.cbSize ? input_stream_info.cbSize
: VideoFrame::AllocationSize(
PIXEL_FORMAT_NV12, input_visible_size_),
kTargetPixelFormat, input_visible_size_),
input_stream_info.cbAlignment == 0 ? input_stream_info.cbAlignment
: input_stream_info.cbAlignment - 1,
&input_buffer);
Expand All @@ -1091,54 +1079,36 @@ HRESULT MediaFoundationVideoEncodeAccelerator::PopulateInputSampleBuffer(
RETURN_ON_HR_FAILURE(hr, "Failed to add buffer to sample", hr);
}

// Establish plain pointers into the input buffer, where we will copy pixel
// data to.
MediaBufferScopedPointer scoped_buffer(input_buffer.Get());
DCHECK(scoped_buffer.get());
uint8_t* dst_y = scoped_buffer.get();
size_t dst_y_stride = VideoFrame::RowBytes(
VideoFrame::kYPlane, kTargetPixelFormat, input_visible_size_.width());
uint8_t* dst_uv =
scoped_buffer.get() +
frame->row_bytes(VideoFrame::kYPlane) * frame->rows(VideoFrame::kYPlane);
uint8_t* end = dst_uv + frame->row_bytes(VideoFrame::kUVPlane) *
frame->rows(VideoFrame::kUVPlane);
dst_y_stride * VideoFrame::Rows(VideoFrame::kYPlane, kTargetPixelFormat,
input_visible_size_.height());
size_t dst_uv_stride = VideoFrame::RowBytes(
VideoFrame::kUVPlane, kTargetPixelFormat, input_visible_size_.width());
uint8_t* end = dst_uv + dst_uv_stride * frame->rows(VideoFrame::kUVPlane);
DCHECK_GE(static_cast<ptrdiff_t>(scoped_buffer.max_length()),
end - scoped_buffer.get());

if (frame->format() == PIXEL_FORMAT_NV12) {
// Copy NV12 pixel data from |frame| to |input_buffer|.
if (frame->IsMappable()) {
src_y = frame->visible_data(VideoFrame::kYPlane);
src_uv = frame->visible_data(VideoFrame::kUVPlane);
}
int error = libyuv::NV12Copy(src_y, frame->stride(VideoFrame::kYPlane),
src_uv, frame->stride(VideoFrame::kUVPlane),
dst_y, frame->row_bytes(VideoFrame::kYPlane),
dst_uv, frame->row_bytes(VideoFrame::kUVPlane),
input_visible_size_.width(),
input_visible_size_.height());
if (error) {
DLOG(ERROR) << "NV12Copy failed";
return E_FAIL;
}
} else if (frame->format() == PIXEL_FORMAT_I420) {
DCHECK(frame->IsMappable());
// Convert I420 to NV12 as input.
int error = libyuv::I420ToNV12(
frame->visible_data(VideoFrame::kYPlane),
frame->stride(VideoFrame::kYPlane),
frame->visible_data(VideoFrame::kUPlane),
frame->stride(VideoFrame::kUPlane),
frame->visible_data(VideoFrame::kVPlane),
frame->stride(VideoFrame::kVPlane), dst_y,
frame->row_bytes(VideoFrame::kYPlane), dst_uv,
frame->row_bytes(VideoFrame::kUPlane) * 2, input_visible_size_.width(),
input_visible_size_.height());
if (error) {
DLOG(ERROR) << "I420ToNV12 failed";
return E_FAIL;
}
} else {
NOTREACHED();
}
// Set up a VideoFrame with the data pointing into the input buffer.
// We need it to ease copying and scaling by reusing ConvertAndScaleFrame()
auto frame_in_buffer = VideoFrame::WrapExternalYuvData(
kTargetPixelFormat, input_visible_size_, gfx::Rect(input_visible_size_),
input_visible_size_, dst_y_stride, dst_uv_stride, dst_y, dst_uv,
frame->timestamp());

auto status = ConvertAndScaleFrame(*frame, *frame_in_buffer, resize_buffer_);
if (!status.is_ok()) {
DLOG(ERROR) << "ConvertAndScaleFrame failed with error code: "
<< static_cast<uint32_t>(status.code());
return E_FAIL;
}
return S_OK;
}

Expand Down
3 changes: 3 additions & 0 deletions media/gpu/windows/media_foundation_video_encode_accelerator_win.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,9 @@ class MEDIA_GPU_EXPORT MediaFoundationVideoEncodeAccelerator
// DXGI device manager for handling hardware input textures
scoped_refptr<DXGIDeviceManager> dxgi_device_manager_;

// A buffer used as a scratch space for I420 to NV12 conversion
std::vector<uint8_t> resize_buffer_;

// Declared last to ensure that all weak pointers are invalidated before
// other destructors run.
base::WeakPtr<MediaFoundationVideoEncodeAccelerator> encoder_weak_ptr_;
Expand Down
2 changes: 1 addition & 1 deletion media/video/video_encode_accelerator.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ bool VideoEncodeAccelerator::IsFlushSupported() {
}

bool VideoEncodeAccelerator::IsGpuFrameResizeSupported() {
#if BUILDFLAG(IS_CHROMEOS_ASH)
#if BUILDFLAG(IS_CHROMEOS_ASH) || BUILDFLAG(IS_WIN)
// TODO(crbug.com/1166889) Add proper method overrides in
// MojoVideoEncodeAccelerator and other subclasses that might return true.
return true;
Expand Down

0 comments on commit 2f7b3c5

Please sign in to comment.