Teach ImageCapture API how to handle non-YUV media::VideoFrames.

This uses the shared main thread context to convert these frames into
SkImages. Since there's no reason to create a new IO-thread context,
the entire conversion process is moved to the main thread.

Fixed: 1161825
Test: https://plnkr.co/edit/k6Itrjzzj60vT2fC?preview
Change-Id: I5e161cfc1e498442409f577d279a85c8c1322522
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2611341
Reviewed-by: Reilly Grant <reillyg@chromium.org>
Reviewed-by: Jeremy Roman <jbroman@chromium.org>
Commit-Queue: Jeremy Roman <jbroman@chromium.org>
Commit-Queue: Dale Curtis <dalecurtis@chromium.org>
Auto-Submit: Dale Curtis <dalecurtis@chromium.org>
Cr-Commit-Position: refs/heads/master@{#840299}

third_party/blink/renderer/modules/imagecapture/DEPS

@@ -2,7 +2,8 @@ include_rules = [
   "+media/capture/mojom/image_capture.mojom-blink.h",
   "+media/capture/mojom/image_capture.mojom-blink-forward.h",
 
-  "+media/base",
+  "+cc/paint/skia_paint_canvas.h",
+  "+media",
   "+skia/ext/legacy_display_globals.h",
   "+skia/ext/platform_canvas.h",
   "+third_party/libyuv",

third_party/blink/renderer/modules/imagecapture/image_capture_frame_grabber.cc

@@ -4,12 +4,16 @@
 
 #include "third_party/blink/renderer/modules/imagecapture/image_capture_frame_grabber.h"
 
+#include "cc/paint/skia_paint_canvas.h"
 #include "media/base/bind_to_current_loop.h"
 #include "media/base/video_frame.h"
 #include "media/base/video_types.h"
 #include "media/base/video_util.h"
+#include "media/renderers/paint_canvas_video_renderer.h"
 #include "skia/ext/legacy_display_globals.h"
 #include "skia/ext/platform_canvas.h"
+#include "third_party/blink/public/platform/platform.h"
+#include "third_party/blink/public/platform/web_graphics_context_3d_provider.h"
 #include "third_party/blink/renderer/core/imagebitmap/image_bitmap.h"
 #include "third_party/blink/renderer/platform/mediastream/media_stream_component.h"
 #include "third_party/blink/renderer/platform/mediastream/media_stream_source.h"
@@ -50,7 +54,7 @@ void OnError(std::unique_ptr<ImageCaptureGrabFrameCallbacks> callbacks) {
 }  // anonymous namespace
 
 // Ref-counted class to receive a single VideoFrame on IO thread, convert it and
-// send it to |main_task_runner_|, where this class is created and destroyed.
+// send it to |task_runner|, where this class is created and destroyed.
 class ImageCaptureFrameGrabber::SingleShotFrameHandler
     : public WTF::ThreadSafeRefCounted<SingleShotFrameHandler> {
  public:
@@ -68,6 +72,10 @@ class ImageCaptureFrameGrabber::SingleShotFrameHandler
  private:
   friend class WTF::ThreadSafeRefCounted<SingleShotFrameHandler>;
 
+  // Converts the media::VideoFrame into a SkImage on the |task_runner|.
+  void ConvertAndDeliverFrame(SkImageDeliverCB callback,
+                              scoped_refptr<media::VideoFrame> frame);
+
   // Flag to indicate that the first frames has been processed, and subsequent
   // ones can be safely discarded.
   bool first_frame_received_;
@@ -80,14 +88,19 @@ void ImageCaptureFrameGrabber::SingleShotFrameHandler::OnVideoFrameOnIOThread(
     scoped_refptr<base::SingleThreadTaskRunner> task_runner,
     scoped_refptr<media::VideoFrame> frame,
     base::TimeTicks /* current_time */) {
-  DCHECK(frame->format() == media::PIXEL_FORMAT_I420 ||
-         frame->format() == media::PIXEL_FORMAT_I420A ||
-         frame->format() == media::PIXEL_FORMAT_NV12);
-
   if (first_frame_received_)
     return;
   first_frame_received_ = true;
+  task_runner->PostTask(FROM_HERE,
+                        ConvertToBaseOnceCallback(CrossThreadBindOnce(
+                            &SingleShotFrameHandler::ConvertAndDeliverFrame,
+                            scoped_refptr<SingleShotFrameHandler>(this),
+                            std::move(callback), std::move(frame))));
+}
 
+void ImageCaptureFrameGrabber::SingleShotFrameHandler::ConvertAndDeliverFrame(
+    SkImageDeliverCB callback,
+    scoped_refptr<media::VideoFrame> frame) {
   const SkAlphaType alpha = media::IsOpaque(frame->format())
                                 ? kOpaque_SkAlphaType
                                 : kPremul_SkAlphaType;
@@ -98,14 +111,33 @@ void ImageCaptureFrameGrabber::SingleShotFrameHandler::OnVideoFrameOnIOThread(
   sk_sp<SkSurface> surface = SkSurface::MakeRaster(info, &props);
   DCHECK(surface);
 
-  auto wrapper_callback =
-      media::BindToLoop(std::move(task_runner),
-                        ConvertToBaseRepeatingCallback(std::move(callback)));
+  // If a frame is GPU backed, we need to use PaintCanvasVideoRenderer to read
+  // it back from the GPU.
+  const bool is_readable = frame->format() == media::PIXEL_FORMAT_I420 ||
+                           frame->format() == media::PIXEL_FORMAT_I420A ||
+                           (frame->format() == media::PIXEL_FORMAT_NV12 &&
+                            frame->HasGpuMemoryBuffer());
+  if (!is_readable) {
+    // |context_provider| is null if the GPU process has crashed or isn't there
+    auto context_provider =
+        Platform::Current()->CreateSharedOffscreenGraphicsContext3DProvider();
+    if (!context_provider) {
+      DLOG(ERROR) << "Failed to create GPU context for GPU backed frame.";
+      std::move(callback).Run(sk_sp<SkImage>());
+      return;
+    }
+
+    cc::SkiaPaintCanvas canvas(surface->getCanvas());
+    media::PaintCanvasVideoRenderer pcvr;
+    context_provider->CopyVideoFrame(&pcvr, frame.get(), &canvas);
+    std::move(callback).Run(surface->makeImageSnapshot());
+    return;
+  }
 
   SkPixmap pixmap;
   if (!skia::GetWritablePixels(surface->getCanvas(), &pixmap)) {
     DLOG(ERROR) << "Error trying to map SkSurface's pixels";
-    std::move(wrapper_callback).Run(sk_sp<SkImage>());
+    std::move(callback).Run(sk_sp<SkImage>());
     return;
   }
 
@@ -120,10 +152,11 @@ void ImageCaptureFrameGrabber::SingleShotFrameHandler::OnVideoFrameOnIOThread(
   int destination_height = pixmap.height();
 
   if (frame->storage_type() == media::VideoFrame::STORAGE_GPU_MEMORY_BUFFER) {
+    DCHECK_EQ(frame->format(), media::PIXEL_FORMAT_NV12);
     auto* gmb = frame->GetGpuMemoryBuffer();
     if (!gmb->Map()) {
       DLOG(ERROR) << "Error mapping GpuMemoryBuffer video frame";
-      std::move(wrapper_callback).Run(sk_sp<SkImage>());
+      std::move(callback).Run(sk_sp<SkImage>());
       return;
     }
 
@@ -178,7 +211,7 @@ void ImageCaptureFrameGrabber::SingleShotFrameHandler::OnVideoFrameOnIOThread(
     }
   }
 
-  std::move(wrapper_callback).Run(surface->makeImageSnapshot());
+  std::move(callback).Run(surface->makeImageSnapshot());
 }
 
 ImageCaptureFrameGrabber::ImageCaptureFrameGrabber()

third_party/blink/tools/blinkpy/presubmit/audit_non_blink_usage.py

@@ -893,8 +893,9 @@
             'third_party/blink/renderer/modules/imagecapture/',
         ],
         'allowed': [
-            'media::.+',
+            'cc::SkiaPaintCanvas',
             'libyuv::.+',
+            'media::.+',
         ]
     },
     {