[UKM] Add navigation metadata

This CL collects a few new booleans about main frame navigations:
Same origin, renderer-initiated, is error page and is main frame.

The first two aim to facilitate research into landing pages vs.
journey navigations.
The latter two are destined to filter out irrelevant navigations.

Bug: 1198057
Change-Id: I32838f58fa999a99e380f8e7aa59f6356228c3f3
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2794958
Commit-Queue: Yoav Weiss <yoavweiss@chromium.org>
Reviewed-by: Robert Kaplow <rkaplow@chromium.org>
Reviewed-by: Arthur Sonzogni <arthursonzogni@chromium.org>
Cr-Commit-Position: refs/heads/master@{#878069}

components/ukm/content/source_url_recorder.cc

@@ -316,6 +316,22 @@ void SourceUrlRecorderWebContentsObserver::MaybeRecordUrl(
 
   navigation_data.is_same_document_navigation =
       navigation_handle->IsSameDocument();
+
+  navigation_data.same_origin_status =
+      UkmSource::NavigationData::SameOriginStatus::UNSET;
+  // Only set the same origin flag for committed non-error,
+  // non-same-document navigations.
+  if (navigation_handle->HasCommitted() && !navigation_handle->IsErrorPage() &&
+      !navigation_handle->IsSameDocument()) {
+    navigation_data.same_origin_status =
+        navigation_handle->IsSameOrigin()
+            ? UkmSource::NavigationData::SameOriginStatus::SAME_ORIGIN
+            : UkmSource::NavigationData::SameOriginStatus::CROSS_ORIGIN;
+  }
+  navigation_data.is_renderer_initiated =
+      navigation_handle->IsRendererInitiated();
+  navigation_data.is_error_page = navigation_handle->IsErrorPage();
+
   navigation_data.previous_source_id =
       last_committed_full_navigation_source_id_;
 

components/ukm/content/source_url_recorder_test.cc

@@ -96,7 +96,7 @@ TEST_F(SourceUrlRecorderWebContentsObserverTest, IgnoreUrlInSubframe) {
   const auto& sources = test_ukm_recorder_.GetSources();
 
   // Expect two sources created, one for navigation and one for document, both
-  // should have the URL of the main frame .
+  // should have the URL of the main frame.
   EXPECT_EQ(2ul, sources.size());
   for (const auto& kv : sources) {
     EXPECT_EQ(main_frame_url, kv.second->url());
@@ -223,3 +223,93 @@ TEST_F(SourceUrlRecorderWebContentsObserverTest,
 
   EXPECT_EQ(url, GetAssociatedURLForWebContentsDocument());
 }
+
+TEST_F(SourceUrlRecorderWebContentsObserverTest, NavigationMetadata) {
+  GURL url1("https://www.example.com/1");
+  GURL same_origin_url1("https://www.example.com/same_origin");
+  GURL url2("https://test.example.com/2");
+  GURL cross_origin_url2("https://test1.example.com/cross_origin");
+  GURL error_url("chrome://error");
+  NavigationSimulator::NavigateAndCommitFromBrowser(web_contents(), url1);
+  NavigationSimulator::CreateRendererInitiated(same_origin_url1, main_rfh())
+      ->Commit();
+  NavigationSimulator::NavigateAndCommitFromBrowser(web_contents(), url2);
+  NavigationSimulator::CreateRendererInitiated(cross_origin_url2, main_rfh())
+      ->Commit();
+  NavigationSimulator::NavigateAndFailFromBrowser(web_contents(), error_url,
+                                                  net::ERR_FAILED);
+
+  EXPECT_EQ(error_url, web_contents()->GetLastCommittedURL());
+
+  // Serialize each source so we can verify expectations below.
+  ukm::Source full_nav_source1;
+  ukm::Source full_nav_source2;
+  ukm::Source same_origin_source1;
+  ukm::Source cross_origin_source2;
+  ukm::Source error_page_source;
+
+  for (auto& kv : test_ukm_recorder_.GetSources()) {
+    // Populate protos from the navigation sources.
+    if (ukm::GetSourceIdType(kv.first) != ukm::SourceIdObj::Type::NAVIGATION_ID)
+      continue;
+
+    if (kv.second->url() == url1) {
+      kv.second->PopulateProto(&full_nav_source1);
+    } else if (kv.second->url() == url2) {
+      kv.second->PopulateProto(&full_nav_source2);
+    } else if (kv.second->url() == same_origin_url1) {
+      kv.second->PopulateProto(&same_origin_source1);
+    } else if (kv.second->url() == cross_origin_url2) {
+      kv.second->PopulateProto(&cross_origin_source2);
+    } else if (kv.second->url() == error_url) {
+      kv.second->PopulateProto(&error_page_source);
+    } else {
+      FAIL() << "Encountered unexpected source.";
+    }
+  }
+
+  // The first navigation was a browser initiated navigation to url1.
+  EXPECT_EQ(url1, full_nav_source1.urls(0).url());
+  EXPECT_TRUE(full_nav_source1.has_id());
+  EXPECT_FALSE(full_nav_source1.navigation_metadata().is_renderer_initiated());
+  EXPECT_EQ(full_nav_source1.navigation_metadata().same_origin_status(),
+            ukm::Source::CROSS_ORIGIN);
+  EXPECT_FALSE(full_nav_source1.navigation_metadata().is_error_page());
+
+  // The second navigation was a same-origin renderer-initiated navigation to
+  // same_origin_url1.
+  EXPECT_EQ(same_origin_url1, same_origin_source1.urls(0).url());
+  EXPECT_TRUE(same_origin_source1.has_id());
+  EXPECT_TRUE(
+      same_origin_source1.navigation_metadata().is_renderer_initiated());
+  EXPECT_EQ(same_origin_source1.navigation_metadata().same_origin_status(),
+            ukm::Source::SAME_ORIGIN);
+  EXPECT_FALSE(same_origin_source1.navigation_metadata().is_error_page());
+
+  // The third navigation was a browser initiated navigation to url2.
+  EXPECT_EQ(url2, full_nav_source2.urls(0).url());
+  EXPECT_TRUE(full_nav_source2.has_id());
+  EXPECT_FALSE(full_nav_source2.navigation_metadata().is_renderer_initiated());
+  EXPECT_EQ(full_nav_source2.navigation_metadata().same_origin_status(),
+            ukm::Source::CROSS_ORIGIN);
+  EXPECT_FALSE(full_nav_source2.navigation_metadata().is_error_page());
+
+  // The fourth navigation was a cross-origin renderer-initiated navigation to
+  // url2.
+  EXPECT_EQ(cross_origin_url2, cross_origin_source2.urls(0).url());
+  EXPECT_TRUE(cross_origin_source2.has_id());
+  EXPECT_TRUE(
+      cross_origin_source2.navigation_metadata().is_renderer_initiated());
+  EXPECT_EQ(cross_origin_source2.navigation_metadata().same_origin_status(),
+            ukm::Source::CROSS_ORIGIN);
+  EXPECT_FALSE(cross_origin_source2.navigation_metadata().is_error_page());
+
+  // The fifth navigation was an error page. Make sure it's is_error_page flag
+  // is set.
+  EXPECT_EQ(error_url, error_page_source.urls(0).url());
+  EXPECT_TRUE(error_page_source.has_id());
+  EXPECT_FALSE(error_page_source.navigation_metadata().is_renderer_initiated());
+  EXPECT_EQ(error_page_source.navigation_metadata().same_origin_status(),
+            ukm::Source::UNSET);
+  EXPECT_TRUE(error_page_source.navigation_metadata().is_error_page());
+}

content/browser/navigation_browsertest.cc

@@ -224,6 +224,23 @@ class DidStartNavigationCallback : public WebContentsObserver {
   base::OnceCallback<void(NavigationHandle*)> callback_;
 };
 
+// Helper class. Immediately run a callback when a navigation finishes.
+class DidFinishNavigationCallback : public WebContentsObserver {
+ public:
+  explicit DidFinishNavigationCallback(
+      WebContents* web_contents,
+      base::OnceCallback<void(NavigationHandle*)> callback)
+      : WebContentsObserver(web_contents), callback_(std::move(callback)) {}
+  ~DidFinishNavigationCallback() final = default;
+
+ private:
+  void DidFinishNavigation(NavigationHandle* navigation_handle) final {
+    if (callback_)
+      std::move(callback_).Run(navigation_handle);
+  }
+  base::OnceCallback<void(NavigationHandle*)> callback_;
+};
+
 const char* non_cacheable_html_response =
     "HTTP/1.1 200 OK\n"
     "cache-control: no-cache, no-store, must-revalidate\n"
@@ -4872,6 +4889,208 @@ IN_PROC_BROWSER_TEST_F(
   VerifyResultsOfAboutBlankNavigation(subframe, main_frame);
 }
 
+IN_PROC_BROWSER_TEST_F(NavigationBrowserTest,
+                       SameOriginFlagOfSameOriginAboutBlankNavigation) {
+  GURL parent_url(embedded_test_server()->GetURL("a.com", "/empty.html"));
+  GURL iframe_url(embedded_test_server()->GetURL("a.com", "/empty.html"));
+  EXPECT_TRUE(NavigateToURL(shell(), parent_url));
+
+  EXPECT_TRUE(ExecJs(current_frame_host(), JsReplace(R"(
+    let iframe = document.createElement('iframe');
+    iframe.src = $1;
+    document.body.appendChild(iframe);
+  )",
+                                                     iframe_url)));
+  WaitForLoadStop(shell()->web_contents());
+
+  base::RunLoop loop;
+  DidFinishNavigationCallback callback(
+      shell()->web_contents(),
+      base::BindLambdaForTesting([&](NavigationHandle* handle) {
+        ASSERT_TRUE(handle->HasCommitted());
+        EXPECT_TRUE(handle->IsSameOrigin());
+        loop.Quit();
+      }));
+
+  // Changing the src to trigger DidFinishNavigationCallback
+  EXPECT_TRUE(ExecJs(current_frame_host(), R"(
+    document.querySelector("iframe").src = 'about:blank';
+  )"));
+  loop.Run();
+}
+
+IN_PROC_BROWSER_TEST_F(NavigationBrowserTest,
+                       SameOriginFlagOfCrossOriginAboutBlankNavigation) {
+  GURL parent_url(embedded_test_server()->GetURL("a.com", "/empty.html"));
+  GURL iframe_url(embedded_test_server()->GetURL("b.com", "/empty.html"));
+  EXPECT_TRUE(NavigateToURL(shell(), parent_url));
+
+  EXPECT_TRUE(ExecJs(current_frame_host(), JsReplace(R"(
+    let iframe = document.createElement('iframe');
+    iframe.src = $1;
+    document.body.appendChild(iframe);
+  )",
+                                                     iframe_url)));
+  WaitForLoadStop(shell()->web_contents());
+
+  base::RunLoop loop;
+  DidFinishNavigationCallback callback(
+      shell()->web_contents(),
+      base::BindLambdaForTesting([&](NavigationHandle* handle) {
+        ASSERT_TRUE(handle->HasCommitted());
+        EXPECT_FALSE(handle->IsSameOrigin());
+        loop.Quit();
+      }));
+
+  // Changing the src to trigger DidFinishNavigationCallback
+  EXPECT_TRUE(ExecJs(current_frame_host(), R"(
+    document.querySelector("iframe").src = 'about:blank';
+  )"));
+  loop.Run();
+}
+
+IN_PROC_BROWSER_TEST_F(NavigationBrowserTest,
+                       SameOriginFlagOfSrcdocNavigation) {
+  GURL url = embedded_test_server()->GetURL("a.com", "/empty.html");
+  GURL cross_origin = embedded_test_server()->GetURL("b.com", "/empty.html");
+  EXPECT_TRUE(NavigateToURL(shell(), url));
+
+  // Navigating to about:srcdoc from the initial empty document is always a
+  // same-origin navigation:
+  // - about:srcdoc is same-origin with the parent.
+  // - the initial empty document is same-origin with the parent.
+  {
+    base::RunLoop loop;
+    DidFinishNavigationCallback callback(
+        shell()->web_contents(),
+        base::BindLambdaForTesting([&](NavigationHandle* handle) {
+          ASSERT_TRUE(handle->HasCommitted());
+          EXPECT_TRUE(handle->IsSameOrigin());
+          loop.Quit();
+        }));
+    EXPECT_TRUE(ExecJs(current_frame_host(), R"(
+      let iframe = document.createElement('iframe');
+      iframe.srcdoc = "dummy content";
+      document.body.appendChild(iframe);
+    )"));
+    loop.Run();
+  }
+
+  // Now, navigate cross-origin, and back to about:srcdoc with a brand new
+  // iframe. The navigation is now considered cross-origin.
+  // - the previous document is cross-origin with the parent.
+  // - about:srcdoc is same-origin with the parent.
+  {
+    EXPECT_TRUE(ExecJs(current_frame_host(), JsReplace(R"(
+      let iframe2 = document.createElement('iframe');
+      iframe2.src = $1;
+      iframe2.id = 'iframe2';
+      document.body.appendChild(iframe2);
+    )",
+                                                       cross_origin)));
+    WaitForLoadStop(shell()->web_contents());
+
+    base::RunLoop loop;
+    DidFinishNavigationCallback callback(
+        shell()->web_contents(),
+        base::BindLambdaForTesting([&](NavigationHandle* handle) {
+          ASSERT_TRUE(handle->HasCommitted());
+          EXPECT_FALSE(handle->IsSameOrigin());
+          loop.Quit();
+        }));
+    EXPECT_TRUE(ExecJs(current_frame_host(), R"(
+      document.getElementById("iframe2").srcdoc = "dummy content";
+    )"));
+    loop.Run();
+  }
+}
+
+IN_PROC_BROWSER_TEST_F(NavigationBrowserTest,
+                       SameOriginFlagOfAboutBlankToAboutBlankNavigation) {
+  GURL parent_url(embedded_test_server()->GetURL("a.com", "/empty.html"));
+  GURL iframe_url(embedded_test_server()->GetURL("b.com", "/empty.html"));
+  EXPECT_TRUE(NavigateToURL(shell(), parent_url));
+
+  EXPECT_TRUE(ExecJs(main_frame(), JsReplace(R"(
+    let iframe = document.createElement('iframe');
+    iframe.src = $1;
+    document.body.appendChild(iframe);
+  )",
+                                             iframe_url)));
+  WaitForLoadStop(shell()->web_contents());
+
+  // Test a same-origin about:blank navigation
+  {
+    base::RunLoop loop;
+    DidFinishNavigationCallback callback(
+        shell()->web_contents(),
+        base::BindLambdaForTesting([&](NavigationHandle* handle) {
+          ASSERT_TRUE(handle->HasCommitted());
+          EXPECT_TRUE(handle->IsSameOrigin());
+          loop.Quit();
+        }));
+    RenderFrameHostImpl* child_document =
+        current_frame_host()->child_at(0)->current_frame_host();
+    EXPECT_TRUE(ExecJs(child_document, R"(location.href = "about:blank";)"));
+    loop.Run();
+  }
+
+  // Test another same-origin about:blank navigation
+  {
+    base::RunLoop loop;
+    DidFinishNavigationCallback callback(
+        shell()->web_contents(),
+        base::BindLambdaForTesting([&](NavigationHandle* handle) {
+          ASSERT_TRUE(handle->HasCommitted());
+          EXPECT_TRUE(handle->IsSameOrigin());
+          loop.Quit();
+        }));
+    RenderFrameHostImpl* child_document =
+        current_frame_host()->child_at(0)->current_frame_host();
+    EXPECT_TRUE(ExecJs(child_document, R"(location.href = "about:blank";)"));
+    loop.Run();
+  }
+
+  // Test a cross-origin about:blank navigation
+  {
+    base::RunLoop loop;
+    DidFinishNavigationCallback callback(
+        shell()->web_contents(),
+        base::BindLambdaForTesting([&](NavigationHandle* handle) {
+          ASSERT_TRUE(handle->HasCommitted());
+          EXPECT_FALSE(handle->IsSameOrigin());
+          loop.Quit();
+        }));
+    EXPECT_TRUE(ExecJs(current_frame_host(), R"(
+      document.querySelector('iframe').src = "about:blank";
+    )"));
+    loop.Run();
+  }
+}
+
+IN_PROC_BROWSER_TEST_F(NavigationBrowserTest, SameOriginOfSandboxedIframe) {
+  EXPECT_TRUE(NavigateToURL(
+      shell(), embedded_test_server()->GetURL("a.com", "/empty.html")));
+
+  base::RunLoop loop;
+  DidFinishNavigationCallback callback(
+      shell()->web_contents(),
+      base::BindLambdaForTesting([&](NavigationHandle* handle) {
+        ASSERT_TRUE(handle->HasCommitted());
+        // TODO(https://crbug.com/888079) Take sandbox into account. Same Origin
+        // should be true
+        EXPECT_FALSE(handle->IsSameOrigin());
+        loop.Quit();
+      }));
+  EXPECT_TRUE(ExecJs(current_frame_host(), R"(
+    let iframe = document.createElement('iframe');
+    iframe.sandbox = "allow-scripts";
+    iframe.src = "/empty.html";
+    document.body.appendChild(iframe);
+  )"));
+  loop.Run();
+}
+
 // The test below verifies that an initial empty document has a functional
 // URLLoaderFactory.  Note that the same behavior is expected in the
 // ...NewPopupToEmptyUrl and in the ...NewPopupToAboutBlank testcases - the