Predict delegated ink points

This CL adds prediction using a Kalman Predictor to the delegated ink
trail feature. When possible, it will predict a point 12ms ahead of the
newest real point available. Number of points and timeframe of
prediction is likely to change as future experiments are run based on
data found through the histogram.

Bug: 1052145
Change-Id: If471d274c17d6b7048e261479739df025dc938ac
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2429261
Commit-Queue: Mario Bianucci <mabian@microsoft.com>
Reviewed-by: Jesse Doherty <jwd@chromium.org>
Reviewed-by: Sadrul Chowdhury <sadrul@chromium.org>
Reviewed-by: weiliangc <weiliangc@chromium.org>
Reviewed-by: Daniel Libby <dlibby@microsoft.com>
Cr-Commit-Position: refs/heads/master@{#822814}

components/viz/service/BUILD.gn

@@ -219,6 +219,7 @@ viz_component("service") {
     "//gpu/command_buffer/client:gles2_interface",
     "//services/viz/privileged/mojom/compositing",
     "//services/viz/public/mojom",
+    "//ui/base/prediction",
     "//ui/gfx",
     "//ui/gfx/geometry",
     "//ui/latency",

components/viz/service/display/delegated_ink_point_renderer_base.cc

@@ -6,10 +6,17 @@
 
 #include "base/trace_event/trace_event.h"
 #include "components/viz/common/delegated_ink_metadata.h"
+#include "ui/base/prediction/kalman_predictor.h"
 
 namespace viz {
 
-DelegatedInkPointRendererBase::DelegatedInkPointRendererBase() = default;
+DelegatedInkPointRendererBase::DelegatedInkPointRendererBase()
+    : metrics_handler_("Renderer.DelegatedInkTrail.Prediction") {
+  unsigned int predictor_options =
+      ui::KalmanPredictor::PredictionOptions::kHeuristicsEnabled |
+      ui::KalmanPredictor::PredictionOptions::kDirectionCutOffEnabled;
+  predictor_ = std::make_unique<ui::KalmanPredictor>(predictor_options);
+}
 DelegatedInkPointRendererBase::~DelegatedInkPointRendererBase() = default;
 
 void DelegatedInkPointRendererBase::InitMessagePipeline(
@@ -87,6 +94,8 @@ std::vector<DelegatedInkPoint> DelegatedInkPointRendererBase::FilterPoints() {
     } else {
       if (it->first == metadata_->timestamp() || points_to_draw.size() > 0) {
         points_to_draw.emplace_back(it->second, it->first);
+        metrics_handler_.AddRealEvent(it->second, it->first,
+                                      metadata_->frame_time());
         it++;
       } else {
         // If we find a point that is later than |metadata_|'s timestamp before
@@ -101,12 +110,60 @@ std::vector<DelegatedInkPoint> DelegatedInkPointRendererBase::FilterPoints() {
   return points_to_draw;
 }
 
+void DelegatedInkPointRendererBase::PredictPoints(
+    std::vector<DelegatedInkPoint>* ink_points_to_draw) {
+  DCHECK(metadata_);
+  int points_predicted = 0;
+
+  // |ink_points_to_draw| needs to have at least one point in it already as a
+  // reference to know what timestamp to start predicting points at. This single
+  // point may just match |metadata_|.
+  if (predictor_->HasPrediction() && ink_points_to_draw->size() > 0) {
+    for (int i = 0; i < kNumberOfPointsToPredict; ++i) {
+      base::TimeTicks timestamp =
+          ink_points_to_draw->back().timestamp() +
+          base::TimeDelta::FromMilliseconds(
+              kNumberOfMillisecondsIntoFutureToPredictPerPoint);
+      std::unique_ptr<ui::InputPredictor::InputData> predicted_point =
+          predictor_->GeneratePrediction(timestamp);
+      if (predicted_point) {
+        ink_points_to_draw->emplace_back(predicted_point->pos,
+                                         predicted_point->time_stamp);
+        metrics_handler_.AddPredictedEvent(predicted_point->pos,
+                                           predicted_point->time_stamp,
+                                           metadata_->frame_time());
+        points_predicted++;
+      } else {
+        // HasPrediction() can return true while GeneratePrediction() fails to
+        // produce a prediction if the predicted point would go in to the
+        // opposite direction of most recently stored points. If this happens,
+        // don't continue trying to generate more predicted points.
+        break;
+      }
+    }
+  }
+
+  TRACE_EVENT_INSTANT1("viz", "DelegatedInkPointRendererBase::PredictPoints",
+                       TRACE_EVENT_SCOPE_THREAD, "predicted points",
+                       points_predicted);
+
+  metrics_handler_.EvaluatePrediction();
+}
+
+void DelegatedInkPointRendererBase::ResetPrediction() {
+  predictor_->Reset();
+  metrics_handler_.Reset();
+}
+
 void DelegatedInkPointRendererBase::StoreDelegatedInkPoint(
     const DelegatedInkPoint& point) {
   TRACE_EVENT_INSTANT1("viz",
                        "DelegatedInkPointRendererImpl::StoreDelegatedInkPoint",
                        TRACE_EVENT_SCOPE_THREAD, "point", point.ToString());
 
+  predictor_->Update(
+      ui::InputPredictor::InputData(point.point(), point.timestamp()));
+
   // Fail-safe to prevent storing excessive points if they are being sent but
   // never filtered and used, like if the renderer has stalled during a long
   // running script.

components/viz/service/display/delegated_ink_point_renderer_base.h

@@ -13,6 +13,8 @@
 #include "components/viz/service/viz_service_export.h"
 #include "mojo/public/cpp/bindings/receiver.h"
 #include "services/viz/public/mojom/compositing/delegated_ink_point.mojom.h"
+#include "ui/base/prediction/input_predictor.h"
+#include "ui/base/prediction/prediction_metrics_handler.h"
 
 namespace viz {
 class DelegatedInkMetadata;
@@ -22,6 +24,14 @@ class DelegatedInkMetadata;
 // added.
 constexpr int kMaximumDelegatedInkPointsStored = 10;
 
+// The number of points to predict into the future, when prediction is
+// available.
+constexpr int kNumberOfPointsToPredict = 1;
+
+// The time that each predicted point should be ahead of the previous point,
+// in milliseconds.
+constexpr int kNumberOfMillisecondsIntoFutureToPredictPerPoint = 12;
+
 // This is the base class used for rendering delegated ink trails on the end of
 // strokes to reduce user perceived latency. On initialization, it binds the
 // mojo interface required for receiving delegated ink points that are made and
@@ -55,6 +65,9 @@ class VIZ_SERVICE_EXPORT DelegatedInkPointRendererBase
   // therefore should be removed from |points_| before drawing.
   std::vector<DelegatedInkPoint> FilterPoints();
 
+  void PredictPoints(std::vector<DelegatedInkPoint>* ink_points_to_draw);
+  void ResetPrediction();
+
   std::unique_ptr<DelegatedInkMetadata> metadata_;
 
  private:
@@ -74,6 +87,13 @@ class VIZ_SERVICE_EXPORT DelegatedInkPointRendererBase
   // of the ink trail.
   std::map<base::TimeTicks, gfx::PointF> points_;
 
+  // Kalman predictor that is used for generating predicted points.
+  std::unique_ptr<ui::InputPredictor> predictor_;
+
+  // Handler for calculating useful metrics for evaluating predicted points
+  // and populating the histograms with those metrics.
+  ui::PredictionMetricsHandler metrics_handler_;
+
   mojo::Receiver<mojom::DelegatedInkPointRenderer> receiver_{this};
 };
 

components/viz/service/display/delegated_ink_point_renderer_skia.cc

@@ -4,8 +4,6 @@
 
 #include "components/viz/service/display/delegated_ink_point_renderer_skia.h"
 
-#include <vector>
-
 #include "base/metrics/histogram_macros.h"
 #include "base/trace_event/trace_event.h"
 #include "components/viz/common/delegated_ink_metadata.h"
@@ -60,47 +58,61 @@ gfx::Rect DelegatedInkPointRendererSkia::GetDamageRect() {
   return gfx::ToEnclosingRect(damage_rect_f);
 }
 
+base::TimeDelta GetImprovement(
+    const std::vector<DelegatedInkPoint>* points_to_draw,
+    const DelegatedInkMetadata* metadata) {
+  if (points_to_draw->size() == 0)
+    return base::TimeDelta::FromMilliseconds(0);
+
+  return points_to_draw->back().timestamp() - metadata->timestamp();
+}
+
+std::vector<SkPoint> DelegatedInkPointRendererSkia::GetPointsToDraw() {
+  std::vector<DelegatedInkPoint> ink_points_to_draw = FilterPoints();
+  UMA_HISTOGRAM_TIMES(
+      "Renderer.DelegatedInkTrail.LatencyImprovement.Skia.WithoutPrediction",
+      GetImprovement(&ink_points_to_draw, metadata_.get()));
+
+  PredictPoints(&ink_points_to_draw);
+  UMA_HISTOGRAM_TIMES(
+      "Renderer.DelegatedInkTrail.LatencyImprovement.Skia.WithPrediction",
+      GetImprovement(&ink_points_to_draw, metadata_.get()));
+
+  std::vector<SkPoint> sk_points;
+  for (DelegatedInkPoint ink_point : ink_points_to_draw)
+    sk_points.push_back(gfx::PointFToSkPoint(ink_point.point()));
+
+  return sk_points;
+}
+
 void DelegatedInkPointRendererSkia::FinalizePathForDraw() {
   // Always rewind the path first so that a path isn't drawn twice.
   path_.rewind();
 
   // Setting the damage rect to empty ensures that the damage rect is cleared
-  // when trails are not being drawn so that extra drawing doesn't occur.
+  // when trails are not being drawn so that extra drawing doesn't occur. If
+  // there isn't metadata, that also indicates that the previous trail has
+  // finished so the predictor should be reset as well.
   if (!metadata_) {
     SetDamageRect(gfx::RectF());
+    ResetPrediction();
     return;
   }
 
-  // First, filter the delegated ink points so that only ones that have a
-  // timestamp that is equal to or later than the metadata still exist.
-  std::vector<DelegatedInkPoint> points_to_draw = FilterPoints();
+  std::vector<SkPoint> sk_points = GetPointsToDraw();
 
-  TRACE_EVENT_INSTANT1("viz", "Filtered points for delegated ink trail",
-                       TRACE_EVENT_SCOPE_THREAD, "points",
-                       points_to_draw.size());
-
-  // TODO(1052145): Predict points.
-
-  base::TimeDelta improvement =
-      static_cast<int>(points_to_draw.size()) > 0
-          ? points_to_draw.back().timestamp() - metadata_->timestamp()
-          : base::TimeDelta::FromMilliseconds(0);
-  UMA_HISTOGRAM_TIMES(
-      "Renderer.DelegatedInkTrail.LatencyImprovement.Skia.WithoutPrediction",
-      improvement);
+  TRACE_EVENT_INSTANT1("viz",
+                       "Filtered and predicted points for delegated ink trail",
+                       TRACE_EVENT_SCOPE_THREAD, "points", sk_points.size());
 
   // If there is only one point total after filtering and predicting, then it
   // will match the metadata point and therefore doesn't need to be drawn in
   // this way, as it will be rendered normally.
-  if (points_to_draw.size() <= 1) {
+  if (sk_points.size() <= 1) {
     SetDamageRect(gfx::RectF());
     return;
   }
 
-  std::vector<SkPoint> sk_points;
-  for (auto ink_point : points_to_draw)
-    sk_points.emplace_back(gfx::PointFToSkPoint(ink_point.point()));
-
   path_.moveTo(sk_points[0]);
   switch (sk_points.size()) {
     case 2:

components/viz/service/display/delegated_ink_point_renderer_skia.h

@@ -5,6 +5,8 @@
 #ifndef COMPONENTS_VIZ_SERVICE_DISPLAY_DELEGATED_INK_POINT_RENDERER_SKIA_H_
 #define COMPONENTS_VIZ_SERVICE_DISPLAY_DELEGATED_INK_POINT_RENDERER_SKIA_H_
 
+#include <vector>
+
 #include "components/viz/service/display/delegated_ink_point_renderer_base.h"
 #include "components/viz/service/viz_service_export.h"
 #include "third_party/skia/include/core/SkPath.h"
@@ -71,6 +73,11 @@ class VIZ_SERVICE_EXPORT DelegatedInkPointRendererSkia
  private:
   void SetDamageRect(gfx::RectF);
 
+  // First filters the points that are stored to only keep points with a
+  // timestamp equal to or later than |metadata_|'s, then predict points if
+  // possible. Then converts those points into SkPoints and returns them.
+  std::vector<SkPoint> GetPointsToDraw();
+
   int GetPathPointCountForTest() const override;
 
   // The path that will be drawn in DrawDelegatedInkTrail(). See class comments