Skip to content

Commit

Permalink
Add undo capability to CursorRenderer.
Browse files Browse the repository at this point in the history 
For the upcoming tab capture implementation replacement, it is necessary
to be able to undo the changes being made to VideoFrames before
"returning" the frame to its source. This change adds and tests that new
functionality.

Bug: 754872
Change-Id: I30aef805644820d2a01de1b8f6e9d1b9da341b80
Reviewed-on: https://chromium-review.googlesource.com/836147
Commit-Queue: Yuri Wiitala <miu@chromium.org>
Reviewed-by: Xiangjun Zhang <xjz@chromium.org>
Cr-Commit-Position: refs/heads/master@{#525613}
  • Loading branch information
@miu-chromium
miu-chromium authored and Commit Bot committed Dec 21, 2017
1 parent 81646ad commit 24e984d
Show file tree
Hide file tree
Showing 6 changed files with 140 additions and 37 deletions.
2 changes: 1 addition & 1 deletion content/browser/media/capture/aura_window_capture_machine.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -391,7 +391,7 @@ void AuraWindowCaptureMachine::CopyOutputFinishedForVideo(
if (machine) {
if (machine->cursor_renderer_ && result)
machine->cursor_renderer_->RenderOnVideoFrame(target.get(),
region_in_frame);
region_in_frame, nullptr);
} else {
VLOG(1) << "Aborting capture: AuraWindowCaptureMachine has gone away.";
result = false;
Expand Down
87 changes: 80 additions & 7 deletions content/browser/media/capture/cursor_renderer.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,8 @@ void CursorRenderer::SnapshotCursorState() {
}

bool CursorRenderer::RenderOnVideoFrame(media::VideoFrame* frame,
const gfx::Rect& region_in_frame) {
const gfx::Rect& region_in_frame,
CursorRendererUndoer* undoer) {
DCHECK_CALLED_ON_VALID_SEQUENCE(render_sequence_checker_);
DCHECK(frame);

Expand Down Expand Up @@ -152,17 +153,22 @@ bool CursorRenderer::RenderOnVideoFrame(media::VideoFrame* frame,
gfx::Rect(cursor_position, gfx::Size(scaled_cursor_bitmap_.width(),
scaled_cursor_bitmap_.height())),
frame->visible_rect());
if (rect.IsEmpty())
return false;

if (undoer)
undoer->TakeSnapshot(*frame, rect);

// Render the cursor in the video frame. This loop also performs a simple
// RGB→YUV color space conversion, with alpha-blended compositing.
for (int y = rect.y(); y < rect.bottom(); ++y) {
int cursor_y = y - cursor_position.y();
uint8_t* yplane = frame->data(media::VideoFrame::kYPlane) +
y * frame->row_bytes(media::VideoFrame::kYPlane);
uint8_t* uplane = frame->data(media::VideoFrame::kUPlane) +
(y / 2) * frame->row_bytes(media::VideoFrame::kUPlane);
uint8_t* vplane = frame->data(media::VideoFrame::kVPlane) +
(y / 2) * frame->row_bytes(media::VideoFrame::kVPlane);
uint8_t* yplane = frame->visible_data(media::VideoFrame::kYPlane) +
y * frame->stride(media::VideoFrame::kYPlane);
uint8_t* uplane = frame->visible_data(media::VideoFrame::kUPlane) +
(y / 2) * frame->stride(media::VideoFrame::kUPlane);
uint8_t* vplane = frame->visible_data(media::VideoFrame::kVPlane) +
(y / 2) * frame->stride(media::VideoFrame::kVPlane);
for (int x = rect.x(); x < rect.right(); ++x) {
int cursor_x = x - cursor_position.x();
SkColor color = scaled_cursor_bitmap_.getColor(cursor_x, cursor_y);
Expand All @@ -175,6 +181,8 @@ bool CursorRenderer::RenderOnVideoFrame(media::VideoFrame* frame,
yplane[x] = alpha_blend(alpha, color_y, yplane[x]);

// Only sample U and V at even coordinates.
// TODO(miu): This isn't right. We should be blending four cursor pixels
// into each U or V output pixel.
if ((x % 2 == 0) && (y % 2 == 0)) {
int color_u = clip_byte(
((color_r * -38 + color_g * -74 + color_b * 112 + 128) >> 8) + 128);
Expand Down Expand Up @@ -273,4 +281,69 @@ void CursorRenderer::OnMouseHasGoneIdle() {
}
}

CursorRendererUndoer::CursorRendererUndoer() = default;

CursorRendererUndoer::~CursorRendererUndoer() = default;

namespace {

// Returns the rect of pixels in a Chroma plane affected by the given |rect| in
// the Luma plane.
gfx::Rect ToEncompassingChromaRect(const gfx::Rect& rect) {
const int left = rect.x() / 2;
const int top = rect.y() / 2;
const int right = (rect.right() + 1) / 2;
const int bottom = (rect.bottom() + 1) / 2;
return gfx::Rect(left, top, right - left, bottom - top);
}

constexpr size_t kYuvPlanes[] = {media::VideoFrame::kYPlane,
media::VideoFrame::kUPlane,
media::VideoFrame::kVPlane};

} // namespace

void CursorRendererUndoer::TakeSnapshot(const media::VideoFrame& frame,
const gfx::Rect& rect) {
DCHECK(frame.visible_rect().Contains(rect));

rect_ = rect;
const gfx::Rect chroma_rect = ToEncompassingChromaRect(rect_);
snapshot_.resize(rect_.size().GetArea() + 2 * chroma_rect.size().GetArea());

uint8_t* dst = snapshot_.data();
for (auto plane : kYuvPlanes) {
const gfx::Rect& plane_rect =
(plane == media::VideoFrame::kYPlane) ? rect_ : chroma_rect;
const int stride = frame.stride(plane);
const uint8_t* src =
frame.visible_data(plane) + plane_rect.y() * stride + plane_rect.x();
for (int row = 0; row < plane_rect.height(); ++row) {
memcpy(dst, src, plane_rect.width());
src += stride;
dst += plane_rect.width();
}
}
}

void CursorRendererUndoer::Undo(media::VideoFrame* frame) const {
DCHECK(frame->visible_rect().Contains(rect_));

const gfx::Rect chroma_rect = ToEncompassingChromaRect(rect_);

const uint8_t* src = snapshot_.data();
for (auto plane : kYuvPlanes) {
const gfx::Rect& plane_rect =
(plane == media::VideoFrame::kYPlane) ? rect_ : chroma_rect;
const int stride = frame->stride(plane);
uint8_t* dst =
frame->visible_data(plane) + plane_rect.y() * stride + plane_rect.x();
for (int row = 0; row < plane_rect.height(); ++row) {
memcpy(dst, src, plane_rect.width());
src += plane_rect.width();
dst += stride;
}
}
}

} // namespace content
27 changes: 25 additions & 2 deletions content/browser/media/capture/cursor_renderer.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@

namespace content {

class CursorRendererUndoer;

// CursorRenderer is an abstract base class that handles all the
// non-platform-specific common cursor rendering functionality. In order to
// track the cursor, the platform-specific implementation will listen to
Expand All @@ -49,9 +51,12 @@ class CONTENT_EXPORT CursorRenderer {
virtual void SetTargetView(gfx::NativeView view) = 0;

// Renders cursor on the given video frame within the content region,
// returning true if |frame| was modified.
// returning true if |frame| was modified. |undoer| is optional: If provided,
// it will be updated with state necessary for later undoing the cursor
// rendering.
bool RenderOnVideoFrame(media::VideoFrame* frame,
const gfx::Rect& region_in_frame);
const gfx::Rect& region_in_frame,
CursorRendererUndoer* undoer);

// Sets a callback that will be run whenever RenderOnVideoFrame() should be
// called soon, to update the mouse cursor location or image in the video.
Expand Down Expand Up @@ -173,6 +178,24 @@ class CONTENT_EXPORT CursorRenderer {
DISALLOW_COPY_AND_ASSIGN(CursorRenderer);
};

// Restores the original content of a VideoFrame, to the point before cursor
// rendering modified it. See CursorRenderer::RenderOnVideoFrame().
class CONTENT_EXPORT CursorRendererUndoer {
public:
CursorRendererUndoer();
~CursorRendererUndoer();

void TakeSnapshot(const media::VideoFrame& frame, const gfx::Rect& rect);

// Restores the frame content to the point where TakeSnapshot() was last
// called.
void Undo(media::VideoFrame* frame) const;

private:
gfx::Rect rect_;
std::vector<uint8_t> snapshot_;
};

} // namespace content

#endif // CONTENT_BROWSER_MEDIA_CAPTURE_CURSOR_RENDERER_H_
29 changes: 16 additions & 13 deletions content/browser/media/capture/cursor_renderer_aura_unittest.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,13 +76,13 @@ class CursorRendererAuraTest : public AuraTestBase {
media::PIXEL_FORMAT_YV12, dummy_frame_size,
gfx::Rect(dummy_frame_size), dummy_frame_size, base::TimeDelta());
}
return cursor_renderer_ &&
cursor_renderer_->RenderOnVideoFrame(dummy_frame_.get(),
dummy_frame_->visible_rect());
return RenderCursorOnVideoFrame(dummy_frame_.get(), nullptr);
}

bool RenderCursorOnVideoFrame(media::VideoFrame* frame) {
return cursor_renderer_->RenderOnVideoFrame(frame, frame->visible_rect());
bool RenderCursorOnVideoFrame(media::VideoFrame* frame,
CursorRendererUndoer* undoer) {
return cursor_renderer_->RenderOnVideoFrame(frame, frame->visible_rect(),
undoer);
}

bool IsUserInteractingWithView() {
Expand Down Expand Up @@ -119,12 +119,12 @@ class CursorRendererAuraTest : public AuraTestBase {
gfx::Rect rect) {
bool y_found = false, u_found = false, v_found = false;
for (int y = rect.y(); y < rect.bottom(); ++y) {
uint8_t* yplane = frame->data(media::VideoFrame::kYPlane) +
y * frame->row_bytes(media::VideoFrame::kYPlane);
uint8_t* uplane = frame->data(media::VideoFrame::kUPlane) +
(y / 2) * frame->row_bytes(media::VideoFrame::kUPlane);
uint8_t* vplane = frame->data(media::VideoFrame::kVPlane) +
(y / 2) * frame->row_bytes(media::VideoFrame::kVPlane);
uint8_t* yplane = frame->visible_data(media::VideoFrame::kYPlane) +
y * frame->stride(media::VideoFrame::kYPlane);
uint8_t* uplane = frame->visible_data(media::VideoFrame::kUPlane) +
(y / 2) * frame->stride(media::VideoFrame::kUPlane);
uint8_t* vplane = frame->visible_data(media::VideoFrame::kVPlane) +
(y / 2) * frame->stride(media::VideoFrame::kVPlane);
for (int x = rect.x(); x < rect.right(); ++x) {
if (yplane[x] != 0)
y_found = true;
Expand Down Expand Up @@ -233,9 +233,12 @@ TEST_F(CursorRendererAuraTest, CursorRenderedOnFrame) {
MoveMouseCursorWithinWindow();
EXPECT_TRUE(CursorDisplayed());

EXPECT_FALSE(NonZeroPixelsInRegion(frame, gfx::Rect(50, 50, 70, 70)));
EXPECT_TRUE(RenderCursorOnVideoFrame(frame.get()));
EXPECT_FALSE(NonZeroPixelsInRegion(frame, frame->visible_rect()));
CursorRendererUndoer undoer;
EXPECT_TRUE(RenderCursorOnVideoFrame(frame.get(), &undoer));
EXPECT_TRUE(NonZeroPixelsInRegion(frame, gfx::Rect(50, 50, 70, 70)));
undoer.Undo(frame.get());
EXPECT_FALSE(NonZeroPixelsInRegion(frame, frame->visible_rect()));
}

TEST_F(CursorRendererAuraTest, CursorRenderedOnRootWindow) {
Expand Down
29 changes: 16 additions & 13 deletions content/browser/media/capture/cursor_renderer_mac_unittest.mm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,13 +65,13 @@ bool CursorDisplayed() {
media::PIXEL_FORMAT_YV12, dummy_frame_size,
gfx::Rect(dummy_frame_size), dummy_frame_size, base::TimeDelta());
}
return cursor_renderer_ &&
cursor_renderer_->RenderOnVideoFrame(dummy_frame_.get(),
dummy_frame_->visible_rect());
return RenderCursorOnVideoFrame(dummy_frame_.get(), nullptr);
}

bool RenderCursorOnVideoFrame(media::VideoFrame* frame) {
return cursor_renderer_->RenderOnVideoFrame(frame, frame->visible_rect());
bool RenderCursorOnVideoFrame(media::VideoFrame* frame,
CursorRendererUndoer* undoer) {
return cursor_renderer_->RenderOnVideoFrame(frame, frame->visible_rect(),
undoer);
}

bool IsUserInteractingWithView() {
Expand Down Expand Up @@ -110,12 +110,12 @@ bool NonZeroPixelsInRegion(scoped_refptr<media::VideoFrame> frame,
gfx::Rect rect) {
bool y_found = false, u_found = false, v_found = false;
for (int y = rect.y(); y < rect.bottom(); ++y) {
uint8_t* yplane = frame->data(media::VideoFrame::kYPlane) +
y * frame->row_bytes(media::VideoFrame::kYPlane);
uint8_t* uplane = frame->data(media::VideoFrame::kUPlane) +
(y / 2) * frame->row_bytes(media::VideoFrame::kUPlane);
uint8_t* vplane = frame->data(media::VideoFrame::kVPlane) +
(y / 2) * frame->row_bytes(media::VideoFrame::kVPlane);
uint8_t* yplane = frame->visible_data(media::VideoFrame::kYPlane) +
y * frame->stride(media::VideoFrame::kYPlane);
uint8_t* uplane = frame->visible_data(media::VideoFrame::kUPlane) +
(y / 2) * frame->stride(media::VideoFrame::kUPlane);
uint8_t* vplane = frame->visible_data(media::VideoFrame::kVPlane) +
(y / 2) * frame->stride(media::VideoFrame::kVPlane);
for (int x = rect.x(); x < rect.right(); ++x) {
if (yplane[x] != 0)
y_found = true;
Expand Down Expand Up @@ -217,9 +217,12 @@ void StartEventTap() {
MoveMouseCursorWithinWindow();
EXPECT_TRUE(CursorDisplayed());

EXPECT_FALSE(NonZeroPixelsInRegion(frame, gfx::Rect(50, 50, 70, 70)));
EXPECT_TRUE(RenderCursorOnVideoFrame(frame.get()));
EXPECT_FALSE(NonZeroPixelsInRegion(frame, frame->visible_rect()));
CursorRendererUndoer undoer;
EXPECT_TRUE(RenderCursorOnVideoFrame(frame.get(), &undoer));
EXPECT_TRUE(NonZeroPixelsInRegion(frame, gfx::Rect(50, 50, 70, 70)));
undoer.Undo(frame.get());
EXPECT_FALSE(NonZeroPixelsInRegion(frame, frame->visible_rect()));
}

} // namespace content
3 changes: 2 additions & 1 deletion content/browser/media/capture/web_contents_video_capture_device.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -255,7 +255,8 @@ void FrameSubscriber::DidCaptureFrame(
if (frame_subscriber_ && frame_subscriber_->cursor_renderer_) {
CursorRenderer* cursor_renderer =
frame_subscriber_->cursor_renderer_.get();
cursor_renderer->RenderOnVideoFrame(frame.get(), region_in_frame);
cursor_renderer->RenderOnVideoFrame(frame.get(), region_in_frame,
nullptr);
// Signal downstream consumers of this frame that encoding/transmission
// should be optimized for image quality over smoothness if: a) The user
// appears to be interacting with the source; and b) A significant amount
Expand Down

0 comments on commit 24e984d

Please sign in to comment.