Make network requests for prefetches.

This CL adds making network requests for prefetches to the content/
refactor of the prefetch code. The following files were copied from
chrome/browser/prefetch/prefetch_proxy/ with only minimal changes:
 - prefetch_proxy_network_context* --> prefetch_network_context*
 - prefetch_proxy_network_context_client* -->
     prefetch_network_context_client*
 - Relevant functions from prefetch_proxy_params* --> prefetch_params*
 - prefetch_proxy_proxy_configurator* --> prefetch_proxy_configurator*

Bug: 1299059
Change-Id: Ic43222bc2987130035574edd4acd0dd9efcf3122
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3586126
Commit-Queue: Max Curran <curranmax@chromium.org>
Reviewed-by: Ryan Sturm <ryansturm@chromium.org>
Reviewed-by: Nicolas Ouellet-Payeur <nicolaso@chromium.org>
Cr-Commit-Position: refs/heads/main@{#998622}

content/browser/BUILD.gn

@@ -1909,6 +1909,14 @@ source_set("browser") {
     "speculation_rules/prefetch/prefetch_document_manager.h",
     "speculation_rules/prefetch/prefetch_features.cc",
     "speculation_rules/prefetch/prefetch_features.h",
+    "speculation_rules/prefetch/prefetch_network_context.cc",
+    "speculation_rules/prefetch/prefetch_network_context.h",
+    "speculation_rules/prefetch/prefetch_network_context_client.cc",
+    "speculation_rules/prefetch/prefetch_network_context_client.h",
+    "speculation_rules/prefetch/prefetch_params.cc",
+    "speculation_rules/prefetch/prefetch_params.h",
+    "speculation_rules/prefetch/prefetch_proxy_configurator.cc",
+    "speculation_rules/prefetch/prefetch_proxy_configurator.h",
     "speculation_rules/prefetch/prefetch_service.cc",
     "speculation_rules/prefetch/prefetch_service.h",
     "speculation_rules/prefetch/prefetch_status.h",

content/browser/speculation_rules/prefetch/prefetch_container.cc

@@ -4,20 +4,28 @@
 
 #include "content/browser/speculation_rules/prefetch/prefetch_container.h"
 
+#include <memory>
+
+#include "content/browser/speculation_rules/prefetch/prefetch_document_manager.h"
+#include "content/browser/speculation_rules/prefetch/prefetch_network_context.h"
+#include "content/browser/speculation_rules/prefetch/prefetch_service.h"
 #include "content/browser/speculation_rules/prefetch/prefetch_status.h"
 #include "content/browser/speculation_rules/prefetch/prefetch_type.h"
 #include "content/public/browser/global_routing_id.h"
+#include "services/network/public/cpp/simple_url_loader.h"
 #include "url/gurl.h"
 
 namespace content {
 
 PrefetchContainer::PrefetchContainer(
     const GlobalRenderFrameHostId& referring_render_frame_host_id,
     const GURL& url,
-    const PrefetchType& prefetch_type)
+    const PrefetchType& prefetch_type,
+    base::WeakPtr<PrefetchDocumentManager> prefetch_document_manager)
     : referring_render_frame_host_id_(referring_render_frame_host_id),
       url_(url),
-      prefetch_type_(prefetch_type) {}
+      prefetch_type_(prefetch_type),
+      prefetch_document_manager_(prefetch_document_manager) {}
 
 PrefetchContainer::~PrefetchContainer() = default;
 
@@ -26,4 +34,28 @@ PrefetchStatus PrefetchContainer::GetPrefetchStatus() const {
   return prefetch_status_.value();
 }
 
+PrefetchNetworkContext* PrefetchContainer::GetOrCreateNetworkContext(
+    PrefetchService* prefetch_service) {
+  if (!network_context_) {
+    network_context_ = std::make_unique<PrefetchNetworkContext>(
+        prefetch_service, prefetch_type_);
+  }
+  return network_context_.get();
+}
+
+PrefetchDocumentManager* PrefetchContainer::GetPrefetchDocumentManager() const {
+  return prefetch_document_manager_.get();
+}
+
+void PrefetchContainer::TakeURLLoader(
+    std::unique_ptr<network::SimpleURLLoader> loader) {
+  DCHECK(!loader_);
+  loader_ = std::move(loader);
+}
+
+void PrefetchContainer::ResetURLLoader() {
+  DCHECK(loader_);
+  loader_.reset();
+}
+
 }  // namespace content

content/browser/speculation_rules/prefetch/prefetch_container.h

@@ -15,15 +15,24 @@
 #include "third_party/abseil-cpp/absl/types/optional.h"
 #include "url/gurl.h"
 
+namespace network {
+class SimpleURLLoader;
+}  // namespace network
+
 namespace content {
 
+class PrefetchService;
+class PrefetchDocumentManager;
+class PrefetchNetworkContext;
+
 // This class contains the state for a request to prefetch a specific URL.
 class CONTENT_EXPORT PrefetchContainer {
  public:
   PrefetchContainer(
       const GlobalRenderFrameHostId& referring_render_frame_host_id,
       const GURL& url,
-      const PrefetchType& prefetch_type);
+      const PrefetchType& prefetch_type,
+      base::WeakPtr<PrefetchDocumentManager> prefetch_document_manager);
   ~PrefetchContainer();
 
   PrefetchContainer(const PrefetchContainer&) = delete;
@@ -58,6 +67,23 @@ class CONTENT_EXPORT PrefetchContainer {
   bool HasPrefetchStatus() const { return prefetch_status_.has_value(); }
   PrefetchStatus GetPrefetchStatus() const;
 
+  // Whether this prefetch is a decoy. Decoy prefetches will not store the
+  // response, and not serve any prefetched resources.
+  void SetIsDecoy(bool is_decoy) { is_decoy_ = is_decoy; }
+  bool IsDecoy() const { return is_decoy_; }
+
+  // The network context used to make network requests for this prefetch.
+  PrefetchNetworkContext* GetOrCreateNetworkContext(
+      PrefetchService* prefetch_service);
+  PrefetchNetworkContext* GetNetworkContext() { return network_context_.get(); }
+
+  // The URL loader used to make the network requests for this prefetch.
+  void TakeURLLoader(std::unique_ptr<network::SimpleURLLoader> loader);
+  void ResetURLLoader();
+
+  // The |PrefetchDocumentManager| that requested |this|.
+  PrefetchDocumentManager* GetPrefetchDocumentManager() const;
+
  private:
   // The ID of the render frame host that triggered the prefetch.
   GlobalRenderFrameHostId referring_render_frame_host_id_;
@@ -72,9 +98,24 @@ class CONTENT_EXPORT PrefetchContainer {
   // prefetched.
   PrefetchType prefetch_type_;
 
+  // The |PrefetchDocumentManager| that requested |this|. Initially it owns
+  // |this|, but once the network request for the prefetch is started,
+  // ownernship is transferred to |PrefetchService|.
+  base::WeakPtr<PrefetchDocumentManager> prefetch_document_manager_;
+
   // The current status, if any, of the prefetch.
   absl::optional<PrefetchStatus> prefetch_status_;
 
+  // Whether this prefetch is a decoy or not. If the prefetch is a decoy then
+  // any prefetched resources will not be served.
+  bool is_decoy_ = false;
+
+  // The network context used to prefetch |url_|.
+  std::unique_ptr<PrefetchNetworkContext> network_context_;
+
+  // The URL loader used to prefetch |url_|.
+  std::unique_ptr<network::SimpleURLLoader> loader_;
+
   base::WeakPtrFactory<PrefetchContainer> weak_method_factory_{this};
 };
 

content/browser/speculation_rules/prefetch/prefetch_container_unittest.cc

@@ -3,6 +3,7 @@
 // found in the LICENSE file.
 
 #include "content/browser/speculation_rules/prefetch/prefetch_container.h"
+
 #include "content/browser/speculation_rules/prefetch/prefetch_status.h"
 #include "content/browser/speculation_rules/prefetch/prefetch_type.h"
 #include "content/public/browser/global_routing_id.h"
@@ -17,7 +18,8 @@ TEST_F(PrefetchContainerTest, CreatePrefetchContainer) {
   PrefetchContainer prefetch_container(
       GlobalRenderFrameHostId(1234, 5678), GURL("https://test.com"),
       PrefetchType(/*use_isolated_network_context=*/true,
-                   /*use_prefetch_proxy=*/true));
+                   /*use_prefetch_proxy=*/true),
+      nullptr);
 
   EXPECT_EQ(prefetch_container.GetReferringRenderFrameHostId(),
             GlobalRenderFrameHostId(1234, 5678));
@@ -35,7 +37,8 @@ TEST_F(PrefetchContainerTest, PrefetchStatus) {
   PrefetchContainer prefetch_container(
       GlobalRenderFrameHostId(1234, 5678), GURL("https://test.com"),
       PrefetchType(/*use_isolated_network_context=*/true,
-                   /*use_prefetch_proxy=*/true));
+                   /*use_prefetch_proxy=*/true),
+      nullptr);
 
   EXPECT_FALSE(prefetch_container.HasPrefetchStatus());
 
@@ -46,5 +49,18 @@ TEST_F(PrefetchContainerTest, PrefetchStatus) {
             PrefetchStatus::kPrefetchNotStarted);
 }
 
+TEST_F(PrefetchContainerTest, IsDecoy) {
+  PrefetchContainer prefetch_container(
+      GlobalRenderFrameHostId(1234, 5678), GURL("https://test.com"),
+      PrefetchType(/*use_isolated_network_context=*/true,
+                   /*use_prefetch_proxy=*/true),
+      nullptr);
+
+  EXPECT_FALSE(prefetch_container.IsDecoy());
+
+  prefetch_container.SetIsDecoy(true);
+  EXPECT_TRUE(prefetch_container.IsDecoy());
+}
+
 }  // namespace
 }  // namespace content

content/browser/speculation_rules/prefetch/prefetch_document_manager.cc

@@ -16,6 +16,10 @@
 
 namespace content {
 
+namespace {
+static PrefetchService* g_prefetch_service_for_testing = nullptr;
+}  // namespace
+
 PrefetchDocumentManager::PrefetchDocumentManager(RenderFrameHost* rfh)
     : DocumentUserData(rfh) {}
 
@@ -81,9 +85,16 @@ void PrefetchDocumentManager::PrefetchUrl(const GURL& url,
 
   // Create a new |PrefetchContainer| and take ownership of it
   owned_prefetches_[url] = std::make_unique<PrefetchContainer>(
-      render_frame_host().GetGlobalId(), url, prefetch_type);
+      render_frame_host().GetGlobalId(), url, prefetch_type,
+      weak_method_factory_.GetWeakPtr());
   all_prefetches_[url] = owned_prefetches_[url]->GetWeakPtr();
 
+  if (g_prefetch_service_for_testing) {
+    g_prefetch_service_for_testing->PrefetchUrl(
+        owned_prefetches_[url]->GetWeakPtr());
+    return;
+  }
+
   // Send a reference of the new |PrefetchContainer| to |PrefetchService| to
   // start the prefetch process.
   DCHECK(BrowserContextImpl::From(render_frame_host().GetBrowserContext())
@@ -95,6 +106,21 @@ void PrefetchDocumentManager::PrefetchUrl(const GURL& url,
   // TODO(https://crbug.com/1299059): Track metrics about the prefetches.
 }
 
+std::unique_ptr<PrefetchContainer>
+PrefetchDocumentManager::ReleasePrefetchContainer(const GURL& url) {
+  DCHECK(owned_prefetches_.find(url) != owned_prefetches_.end());
+  std::unique_ptr<PrefetchContainer> prefetch_container =
+      std::move(owned_prefetches_[url]);
+  owned_prefetches_.erase(url);
+  return prefetch_container;
+}
+
+// static
+void PrefetchDocumentManager::SetPrefetchServiceForTesting(
+    PrefetchService* prefetch_service) {
+  g_prefetch_service_for_testing = prefetch_service;
+}
+
 DOCUMENT_USER_DATA_KEY_IMPL(PrefetchDocumentManager);
 
 }  // namespace content

content/browser/speculation_rules/prefetch/prefetch_document_manager.h

@@ -9,7 +9,7 @@
 #include <memory>
 #include <vector>
 
-#include "content/browser/speculation_rules/prefetch/prefetch_container.h"
+#include "base/memory/weak_ptr.h"
 #include "content/browser/speculation_rules/prefetch/prefetch_type.h"
 #include "content/common/content_export.h"
 #include "content/public/browser/document_user_data.h"
@@ -18,6 +18,9 @@
 
 namespace content {
 
+class PrefetchContainer;
+class PrefetchService;
+
 // Manages the state of and tracks metrics about prefetches for a single page
 // load.
 class CONTENT_EXPORT PrefetchDocumentManager
@@ -38,6 +41,13 @@ class CONTENT_EXPORT PrefetchDocumentManager
   // Starts the process to prefetch |url| with the given |prefetch_type|.
   void PrefetchUrl(const GURL& url, const PrefetchType& prefetch_type);
 
+  // Releases ownership of the |PrefetchContainer| associated with |url|. The
+  // prefetch is removed from |owned_prefetches_|, but a pointer to it remains
+  // in |all_prefetches_|.
+  std::unique_ptr<PrefetchContainer> ReleasePrefetchContainer(const GURL& url);
+
+  static void SetPrefetchServiceForTesting(PrefetchService* prefetch_service);
+
  private:
   explicit PrefetchDocumentManager(RenderFrameHost* rfh);
   friend DocumentUserData;
@@ -52,6 +62,8 @@ class CONTENT_EXPORT PrefetchDocumentManager
   // which point |PrefetchService| takes ownership.
   std::map<GURL, std::unique_ptr<PrefetchContainer>> owned_prefetches_;
 
+  base::WeakPtrFactory<PrefetchDocumentManager> weak_method_factory_{this};
+
   DOCUMENT_USER_DATA_KEY_DECL();
 };
 

content/browser/speculation_rules/prefetch/prefetch_document_manager_unittest.cc

@@ -9,20 +9,37 @@
 
 #include "base/test/scoped_feature_list.h"
 #include "content/browser/speculation_rules/prefetch/prefetch_features.h"
+#include "content/browser/speculation_rules/prefetch/prefetch_service.h"
 #include "content/public/test/test_browser_context.h"
 #include "content/test/test_web_contents.h"
+#include "services/network/public/cpp/weak_wrapper_shared_url_loader_factory.h"
+#include "services/network/test/test_url_loader_factory.h"
 #include "testing/gtest/include/gtest/gtest.h"
 #include "third_party/blink/public/mojom/loader/referrer.mojom.h"
 #include "third_party/blink/public/mojom/speculation_rules/speculation_rules.mojom.h"
 
 namespace content {
 namespace {
 
+class TestPrefetchService : public PrefetchService {
+ public:
+  explicit TestPrefetchService(BrowserContext* browser_context)
+      : PrefetchService(browser_context) {}
+
+  void PrefetchUrl(
+      base::WeakPtr<PrefetchContainer> prefetch_container) override {
+    prefetches_.push_back(prefetch_container);
+  }
+
+  std::vector<base::WeakPtr<PrefetchContainer>> prefetches_;
+};
+
 class PrefetchDocumentManagerTest : public RenderViewHostTestHarness {
  public:
   PrefetchDocumentManagerTest() {
-    scoped_feature_list_.InitAndEnableFeature(
-        content::features::kPrefetchUseContentRefactor);
+    scoped_feature_list_.InitAndEnableFeatureWithParameters(
+        content::features::kPrefetchUseContentRefactor,
+        {{"proxy_host", "https://testproxyhost.com"}});
   }
 
   void SetUp() override {
@@ -33,11 +50,17 @@ class PrefetchDocumentManagerTest : public RenderViewHostTestHarness {
         browser_context_.get(),
         SiteInstanceImpl::Create(browser_context_.get()));
     web_contents_->NavigateAndCommit(GetSameOriginUrl("/"));
+
+    prefetch_service_ =
+        std::make_unique<TestPrefetchService>(browser_context_.get());
+    PrefetchDocumentManager::SetPrefetchServiceForTesting(
+        prefetch_service_.get());
   }
 
   void TearDown() override {
     web_contents_.reset();
     browser_context_.reset();
+    PrefetchDocumentManager::SetPrefetchServiceForTesting(nullptr);
     RenderViewHostTestHarness::TearDown();
   }
 
@@ -51,11 +74,16 @@ class PrefetchDocumentManagerTest : public RenderViewHostTestHarness {
     return GURL("https://other.example.com" + path);
   }
 
+  const std::vector<base::WeakPtr<PrefetchContainer>>& GetPrefetches() {
+    return prefetch_service_->prefetches_;
+  }
+
  private:
   base::test::ScopedFeatureList scoped_feature_list_;
 
   std::unique_ptr<TestBrowserContext> browser_context_;
   std::unique_ptr<TestWebContents> web_contents_;
+  std::unique_ptr<TestPrefetchService> prefetch_service_;
 };
 
 TEST_F(PrefetchDocumentManagerTest, ProcessSpeculationCandidates) {
@@ -108,8 +136,25 @@ TEST_F(PrefetchDocumentManagerTest, ProcessSpeculationCandidates) {
   PrefetchDocumentManager::GetOrCreateForCurrentDocument(GetMainFrame())
       ->ProcessCandidates(candidates);
 
-  // Check that the candidates that should be prefetched were removed, and the
-  // others were kept.
+  // Check that the candidates that should be prefetched were sent to
+  // |PrefetchService|.
+  const auto& prefetch_urls = GetPrefetches();
+  ASSERT_EQ(prefetch_urls.size(), 3U);
+  EXPECT_EQ(prefetch_urls[0]->GetURL(), GetCrossOriginUrl("/candidate1.html"));
+  EXPECT_EQ(prefetch_urls[0]->GetPrefetchType(),
+            PrefetchType(/*use_isolated_network_context=*/true,
+                         /*use_prefetch_proxy=*/true));
+  EXPECT_EQ(prefetch_urls[1]->GetURL(), GetCrossOriginUrl("/candidate2.html"));
+  EXPECT_EQ(prefetch_urls[1]->GetPrefetchType(),
+            PrefetchType(/*use_isolated_network_context=*/true,
+                         /*use_prefetch_proxy=*/false));
+  EXPECT_EQ(prefetch_urls[2]->GetURL(), GetSameOriginUrl("/candidate3.html"));
+  EXPECT_EQ(prefetch_urls[2]->GetPrefetchType(),
+            PrefetchType(/*use_isolated_network_context=*/false,
+                         /*use_prefetch_proxy=*/false));
+
+  // Check that the only remaining entries in candidates are those that
+  // shouldn't be prefetched by |PrefetchService|.
   ASSERT_EQ(candidates.size(), 2U);
   EXPECT_EQ(candidates[0]->url, GetCrossOriginUrl("/candidate4.html"));
   EXPECT_EQ(candidates[1]->url, GetCrossOriginUrl("/candidate5.html"));