headless: Make it possible to configure an HTTP proxy

Add a configuration option for setting a proxy to be used for all
HTTP/HTTPS connections.

BUG=546953

Review URL: https://codereview.chromium.org/1781193004

Cr-Commit-Position: refs/heads/master@{#381208}

headless/BUILD.gn

@@ -137,6 +137,8 @@ executable("headless_shell") {
 
   sources = [
     "app/headless_shell.cc",
+    "app/headless_shell_switches.cc",
+    "app/headless_shell_switches.h",
   ]
 
   deps = [

headless/README.md

@@ -49,11 +49,13 @@ The main embedder API classes are:
 - `HeadlessBrowser::Options::Builder` - Defines the embedding options, e.g.:
   - `SetMessagePump` - Replaces the default base message pump. See
     `base::MessagePump`.
+  - `SetProxyServer` - Configures an HTTP/HTTPS proxy server to be used for
+    accessing the network.
 
-## Headless API
+## Client API
 
-The headless API is used to drive the browser and interact with the loaded web
-pages. Its main classes are:
+The headless client API is used to drive the browser and interact with loaded
+web pages. Its main classes are:
 
 - `HeadlessBrowser` - Represents the global headless browser instance.
 - `HeadlessWebContents` - Represents a single "tab" within the browser.

headless/app/headless_shell.cc

@@ -10,6 +10,7 @@
 #include "base/numerics/safe_conversions.h"
 #include "base/strings/string_number_conversions.h"
 #include "content/public/common/content_switches.h"
+#include "headless/app/headless_shell_switches.h"
 #include "headless/public/headless_browser.h"
 #include "headless/public/headless_web_contents.h"
 #include "net/base/ip_address.h"
@@ -68,6 +69,8 @@ class HeadlessShell : public HeadlessWebContents::Observer {
     ShutdownIfNeeded();
   }
 
+  void DidFinishNavigation(bool success) override {}
+
  private:
   HeadlessBrowser* browser_;  // Not owned.
   scoped_ptr<HeadlessWebContents> web_contents_;
@@ -96,6 +99,18 @@ int main(int argc, const char** argv) {
     }
   }
 
+  if (command_line.HasSwitch(headless::switches::kProxyServer)) {
+    std::string proxy_server =
+        command_line.GetSwitchValueASCII(headless::switches::kProxyServer);
+    net::HostPortPair parsed_proxy_server =
+        net::HostPortPair::FromString(proxy_server);
+    if (parsed_proxy_server.host().empty() || !parsed_proxy_server.port()) {
+      LOG(ERROR) << "Malformed proxy server url";
+      return EXIT_FAILURE;
+    }
+    builder.SetProxyServer(parsed_proxy_server);
+  }
+
   return HeadlessBrowserMain(
       builder.Build(),
       base::Bind(&HeadlessShell::OnStart, base::Unretained(&shell)));

headless/app/headless_shell_switches.cc

@@ -0,0 +1,15 @@
+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "headless/app/headless_shell_switches.h"
+
+namespace headless {
+namespace switches {
+
+// Uses a specified proxy server, overrides system settings. This switch only
+// affects HTTP and HTTPS requests.
+const char kProxyServer[] = "proxy-server";
+
+}  // namespace switches
+}  // namespace headless

headless/app/headless_shell_switches.h

@@ -0,0 +1,14 @@
+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef HEADLESS_APP_HEADLESS_SHELL_SWITCHES_H_
+#define HEADLESS_APP_HEADLESS_SHELL_SWITCHES_H_
+
+namespace headless {
+namespace switches {
+extern const char kProxyServer[];
+}  // namespace switches
+}  // namespace headless
+
+#endif  // HEADLESS_APP_HEADLESS_SHELL_SWITCHES_H_

headless/lib/browser/headless_browser_context.cc

@@ -156,6 +156,11 @@ HeadlessBrowserContext::GetBackgroundSyncController() {
   return nullptr;
 }
 
+void HeadlessBrowserContext::SetOptionsForTesting(
+    const HeadlessBrowser::Options& options) {
+  options_ = options;
+}
+
 void HeadlessBrowserContext::SetURLRequestContextGetter(
     scoped_refptr<net::URLRequestContextGetter> url_request_context_getter) {
   resource_context_->set_url_request_context_getter(url_request_context_getter);

headless/lib/browser/headless_browser_context.h

@@ -44,6 +44,7 @@ class HeadlessBrowserContext : public content::BrowserContext {
   content::BackgroundSyncController* GetBackgroundSyncController() override;
 
   const HeadlessBrowser::Options& options() const { return options_; }
+  void SetOptionsForTesting(const HeadlessBrowser::Options& options);
 
   // Configure the URL request context getter to be used for serving URL
   // requests in this browser instance.

headless/lib/browser/headless_browser_impl.cc

@@ -74,6 +74,12 @@ void HeadlessBrowserImpl::RunOnStartCallback() {
   on_start_callback_ = base::Callback<void(HeadlessBrowser*)>();
 }
 
+void HeadlessBrowserImpl::SetOptionsForTesting(
+    const HeadlessBrowser::Options& options) {
+  options_ = options;
+  browser_context()->SetOptionsForTesting(options);
+}
+
 int HeadlessBrowserMain(
     const HeadlessBrowser::Options& options,
     const base::Callback<void(HeadlessBrowser*)>& on_browser_start_callback) {

headless/lib/browser/headless_browser_impl.h

@@ -44,6 +44,11 @@ class HeadlessBrowserImpl : public HeadlessBrowser {
 
   const HeadlessBrowser::Options& options() const { return options_; }
 
+  // Customize the options used by this headless browser instance. Note that
+  // options which take effect before the message loop has been started (e.g.,
+  // custom message pumps) cannot be set via this method.
+  void SetOptionsForTesting(const HeadlessBrowser::Options& options);
+
  protected:
   base::Callback<void(HeadlessBrowser*)> on_start_callback_;
   HeadlessBrowser::Options options_;

headless/lib/browser/headless_url_request_context_getter.cc

@@ -73,7 +73,8 @@ HeadlessURLRequestContextGetter::HeadlessURLRequestContextGetter(
   // We must create the proxy config service on the UI loop on Linux because it
   // must synchronously run on the glib message loop. This will be passed to
   // the URLRequestContextStorage on the IO thread in GetURLRequestContext().
-  proxy_config_service_ = GetProxyConfigService();
+  if (options_.proxy_server.IsEmpty())
+    proxy_config_service_ = GetProxyConfigService();
 }
 
 HeadlessURLRequestContextGetter::~HeadlessURLRequestContextGetter() {}
@@ -91,6 +92,8 @@ HeadlessURLRequestContextGetter::GetProxyConfigService() {
 
 scoped_ptr<net::ProxyService>
 HeadlessURLRequestContextGetter::GetProxyService() {
+  if (!options_.proxy_server.IsEmpty())
+    return net::ProxyService::CreateFixed(options_.proxy_server.ToString());
   return net::ProxyService::CreateUsingSystemProxyResolver(
       std::move(proxy_config_service_), 0, url_request_context_->net_log());
 }

headless/lib/browser/headless_web_contents_impl.cc

@@ -7,6 +7,7 @@
 #include "base/bind.h"
 #include "base/memory/weak_ptr.h"
 #include "base/trace_event/trace_event.h"
+#include "content/public/browser/navigation_handle.h"
 #include "content/public/browser/render_frame_host.h"
 #include "content/public/browser/render_process_host.h"
 #include "content/public/browser/render_view_host.h"
@@ -33,6 +34,12 @@ class WebContentsObserverAdapter : public content::WebContentsObserver {
     observer_->DocumentOnLoadCompletedInMainFrame();
   }
 
+  void DidFinishNavigation(
+      content::NavigationHandle* navigation_handle) override {
+    observer_->DidFinishNavigation(navigation_handle->HasCommitted() &&
+                                   !navigation_handle->IsErrorPage());
+  }
+
  private:
   HeadlessWebContents::Observer* observer_;  // Not owned.
 

headless/lib/headless_browser_browsertest.cc

@@ -6,6 +6,7 @@
 #include "headless/public/headless_browser.h"
 #include "headless/public/headless_web_contents.h"
 #include "headless/test/headless_browser_test.h"
+#include "net/test/spawned_test_server/spawned_test_server.h"
 #include "testing/gtest/include/gtest/gtest.h"
 #include "ui/gfx/geometry/size.h"
 
@@ -19,4 +20,42 @@ IN_PROC_BROWSER_TEST_F(HeadlessBrowserTest, CreateAndDestroyWebContents) {
   web_contents.reset();
 }
 
+class HeadlessBrowserTestWithProxy : public HeadlessBrowserTest {
+ public:
+  HeadlessBrowserTestWithProxy()
+      : proxy_server_(net::SpawnedTestServer::TYPE_HTTP,
+                      net::SpawnedTestServer::kLocalhost,
+                      base::FilePath(FILE_PATH_LITERAL("headless/test/data"))) {
+  }
+
+  void SetUp() override {
+    ASSERT_TRUE(proxy_server_.Start());
+    HeadlessBrowserTest::SetUp();
+  }
+
+  void TearDown() override {
+    proxy_server_.Stop();
+    HeadlessBrowserTest::TearDown();
+  }
+
+  net::SpawnedTestServer* proxy_server() { return &proxy_server_; }
+
+ private:
+  net::SpawnedTestServer proxy_server_;
+};
+
+IN_PROC_BROWSER_TEST_F(HeadlessBrowserTestWithProxy, SetProxyServer) {
+  HeadlessBrowser::Options::Builder builder;
+  builder.SetProxyServer(proxy_server()->host_port_pair());
+  SetBrowserOptions(builder.Build());
+
+  scoped_ptr<HeadlessWebContents> web_contents =
+      browser()->CreateWebContents(gfx::Size(800, 600));
+
+  // Load a page which doesn't actually exist, but for which the our proxy
+  // returns valid content anyway.
+  EXPECT_TRUE(NavigateAndWaitForLoad(
+      web_contents.get(), GURL("http://not-an-actual-domain.tld/hello.html")));
+}
+
 }  // namespace headless

headless/lib/headless_web_contents_browsertest.cc

@@ -2,7 +2,6 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-#include "base/run_loop.h"
 #include "content/public/test/browser_test.h"
 #include "headless/public/headless_browser.h"
 #include "headless/public/headless_web_contents.h"
@@ -15,37 +14,12 @@ namespace headless {
 
 class HeadlessWebContentsTest : public HeadlessBrowserTest {};
 
-class WaitForNavigationObserver : public HeadlessWebContents::Observer {
- public:
-  WaitForNavigationObserver(base::RunLoop* run_loop,
-                            HeadlessWebContents* web_contents)
-      : run_loop_(run_loop), web_contents_(web_contents) {
-    web_contents_->AddObserver(this);
-  }
-
-  ~WaitForNavigationObserver() override { web_contents_->RemoveObserver(this); }
-
-  void DocumentOnLoadCompletedInMainFrame() override { run_loop_->Quit(); }
-
- private:
-  base::RunLoop* run_loop_;            // Not owned.
-  HeadlessWebContents* web_contents_;  // Not owned.
-
-  DISALLOW_COPY_AND_ASSIGN(WaitForNavigationObserver);
-};
-
 IN_PROC_BROWSER_TEST_F(HeadlessWebContentsTest, Navigation) {
   EXPECT_TRUE(embedded_test_server()->Start());
   scoped_ptr<HeadlessWebContents> web_contents =
       browser()->CreateWebContents(gfx::Size(800, 600));
-
-  base::RunLoop run_loop;
-  base::MessageLoop::ScopedNestableTaskAllower nestable_allower(
-      base::MessageLoop::current());
-  WaitForNavigationObserver observer(&run_loop, web_contents.get());
-
-  web_contents->OpenURL(embedded_test_server()->GetURL("/hello.html"));
-  run_loop.Run();
+  EXPECT_TRUE(NavigateAndWaitForLoad(
+      web_contents.get(), embedded_test_server()->GetURL("/hello.html")));
 }
 
 IN_PROC_BROWSER_TEST_F(HeadlessWebContentsTest, NavigationWithBadURL) {

headless/public/headless_browser.cc

@@ -26,6 +26,8 @@ Options::~Options() {}
 
 Builder::Builder(int argc, const char** argv) : options_(argc, argv) {}
 
+Builder::Builder() : options_(0, nullptr) {}
+
 Builder::~Builder() {}
 
 Builder& Builder::SetUserAgent(const std::string& user_agent) {
@@ -43,6 +45,11 @@ Builder& Builder::SetMessagePump(base::MessagePump* message_pump) {
   return *this;
 }
 
+Builder& Builder::SetProxyServer(const net::HostPortPair& proxy_server) {
+  options_.proxy_server = proxy_server;
+  return *this;
+}
+
 Options Builder::Build() {
   return options_;
 }

headless/public/headless_browser.h

@@ -12,6 +12,7 @@
 #include "base/memory/ref_counted.h"
 #include "base/memory/scoped_ptr.h"
 #include "headless/public/headless_export.h"
+#include "net/base/host_port_pair.h"
 #include "net/base/ip_endpoint.h"
 
 namespace base {
@@ -67,8 +68,14 @@ struct HeadlessBrowser::Options {
   std::string user_agent;
   std::string navigator_platform;
 
+  // Address at which DevTools should listen for connections. Disabled by
+  // default.
   net::IPEndPoint devtools_endpoint;
 
+  // Address of the HTTP/HTTPS proxy server to use. The system proxy settings
+  // are used by default.
+  net::HostPortPair proxy_server;
+
   // Optional message pump that overrides the default. Must outlive the browser.
   base::MessagePump* message_pump;
 
@@ -79,11 +86,13 @@ struct HeadlessBrowser::Options {
 class HeadlessBrowser::Options::Builder {
  public:
   Builder(int argc, const char** argv);
+  Builder();
   ~Builder();
 
   Builder& SetUserAgent(const std::string& user_agent);
   Builder& EnableDevToolsServer(const net::IPEndPoint& endpoint);
   Builder& SetMessagePump(base::MessagePump* message_pump);
+  Builder& SetProxyServer(const net::HostPortPair& proxy_server);
 
   Options Build();
 

headless/public/headless_web_contents.h

@@ -24,6 +24,7 @@ class HEADLESS_EXPORT HeadlessWebContents {
    public:
     // Will be called on browser thread.
     virtual void DocumentOnLoadCompletedInMainFrame() = 0;
+    virtual void DidFinishNavigation(bool success) = 0;
 
    protected:
     Observer() {}