Move DownloadWorker from content/ to components/

This CL moves DownloadWorker from content/browser/download to
components/download.
DownloadWorker is used by ParallelDownloadJob for creating parallel
download requests.
To remove the dependencies on content/, following changes are made:
1. Introduces URLDownloadHandlerFactory class, this class allows content/
to inject its own URLDownloadHandler implementation (UrlDownloader) when
network service is disabled.
2. DownloadManager will always pass a SharedURLLoaderFactory to
ResourceDownloader. ResourceDownloader will later return it back to
DownloadManager, and DownloadManager will then pass it to the
ParallelDownloadJob through DownloadItemImpl.

Bug: 803135
Change-Id: I96dfb75daa438049df0dc9fd0fbe74abd5ff1d7a
Reviewed-on: https://chromium-review.googlesource.com/974120
Commit-Queue: Min Qin <qinmin@chromium.org>
Reviewed-by: David Trainor <dtrainor@chromium.org>
Reviewed-by: John Abd-El-Malek <jam@chromium.org>
Reviewed-by: Matt Menke <mmenke@chromium.org>
Cr-Commit-Position: refs/heads/master@{#545570}

components/download/internal/common/BUILD.gn

@@ -25,10 +25,12 @@ source_set("internal") {
     "download_task_runner.cc",
     "download_ukm_helper.cc",
     "download_utils.cc",
+    "download_worker.cc",
     "parallel_download_utils.cc",
     "rate_estimator.cc",
     "resource_downloader.cc",
     "stream_handle_input_stream.cc",
+    "url_download_handler_factory.cc",
   ]
 
   public_deps = [

components/download/internal/common/DEPS

@@ -6,9 +6,11 @@ include_rules = [
   "+crypto",
   "+mojo/public/c/system",
   "+net/base/filename_util.h",
+  "+net/base/load_flags.h",
   "+net/base/io_buffer.h",
   "+net/base/net_errors.h",
   "+net/http/http_content_disposition.h",
+  "+net/http/http_request_headers.h",
   "+net/http/http_response_headers.h",
   "+net/http/http_status_code.h",
   "+net/http/http_util.h",

components/download/internal/common/download_utils.cc

@@ -11,10 +11,24 @@
 #include "components/download/public/common/download_item.h"
 #include "components/download/public/common/download_save_info.h"
 #include "components/download/public/common/download_stats.h"
+#include "components/download/public/common/download_url_parameters.h"
+#include "net/base/load_flags.h"
+#include "net/http/http_request_headers.h"
 #include "net/http/http_status_code.h"
+#include "services/network/public/cpp/resource_request.h"
 
 namespace download {
 
+namespace {
+
+void AppendExtraHeaders(net::HttpRequestHeaders* headers,
+                        DownloadUrlParameters* params) {
+  for (const auto& header : params->request_headers())
+    headers->SetHeaderIfMissing(header.first, header.second);
+}
+
+}  // namespace
+
 const uint32_t DownloadItem::kInvalidId = 0;
 
 DownloadInterruptReason HandleRequestCompletionStatus(
@@ -208,4 +222,129 @@ void HandleResponseHeaders(const net::HttpResponseHeaders* headers,
        headers->response_code() == net::HTTP_PARTIAL_CONTENT);
 }
 
+std::unique_ptr<network::ResourceRequest> CreateResourceRequest(
+    DownloadUrlParameters* params) {
+  DCHECK(params->offset() >= 0);
+
+  std::unique_ptr<network::ResourceRequest> request(
+      new network::ResourceRequest);
+  request->method = params->method();
+  request->url = params->url();
+  request->request_initiator = params->initiator();
+  request->do_not_prompt_for_login = params->do_not_prompt_for_login();
+  request->site_for_cookies = params->url();
+  request->referrer = params->referrer();
+  request->referrer_policy = params->referrer_policy();
+  request->allow_download = true;
+  request->is_main_frame = true;
+
+  if (params->render_process_host_id() >= 0)
+    request->render_frame_id = params->render_frame_host_routing_id();
+
+  bool has_upload_data = false;
+  if (params->post_body()) {
+    request->request_body = params->post_body();
+    has_upload_data = true;
+  }
+
+  if (params->post_id() >= 0) {
+    // The POST in this case does not have an actual body, and only works
+    // when retrieving data from cache. This is done because we don't want
+    // to do a re-POST without user consent, and currently don't have a good
+    // plan on how to display the UI for that.
+    DCHECK(params->prefer_cache());
+    DCHECK_EQ("POST", params->method());
+    request->request_body = new network::ResourceRequestBody();
+    request->request_body->set_identifier(params->post_id());
+    has_upload_data = true;
+  }
+
+  request->load_flags = GetLoadFlags(params, has_upload_data);
+
+  // Add additional request headers.
+  std::unique_ptr<net::HttpRequestHeaders> headers =
+      GetAdditionalRequestHeaders(params);
+  request->headers.Swap(headers.get());
+
+  return request;
+}
+
+int GetLoadFlags(DownloadUrlParameters* params, bool has_upload_data) {
+  int load_flags = 0;
+  if (params->prefer_cache()) {
+    // If there is upload data attached, only retrieve from cache because there
+    // is no current mechanism to prompt the user for their consent for a
+    // re-post. For GETs, try to retrieve data from the cache and skip
+    // validating the entry if present.
+    if (has_upload_data)
+      load_flags |= net::LOAD_ONLY_FROM_CACHE | net::LOAD_SKIP_CACHE_VALIDATION;
+    else
+      load_flags |= net::LOAD_SKIP_CACHE_VALIDATION;
+  } else {
+    load_flags |= net::LOAD_DISABLE_CACHE;
+  }
+  return load_flags;
+}
+
+std::unique_ptr<net::HttpRequestHeaders> GetAdditionalRequestHeaders(
+    DownloadUrlParameters* params) {
+  auto headers = std::make_unique<net::HttpRequestHeaders>();
+  if (params->offset() == 0 &&
+      params->length() == DownloadSaveInfo::kLengthFullContent) {
+    AppendExtraHeaders(headers.get(), params);
+    return headers;
+  }
+
+  bool has_last_modified = !params->last_modified().empty();
+  bool has_etag = !params->etag().empty();
+
+  // Strong validator(i.e. etag or last modified) is required in range requests
+  // for download resumption and parallel download.
+  DCHECK(has_etag || has_last_modified);
+  if (!has_etag && !has_last_modified) {
+    DVLOG(1) << "Creating partial request without strong validators.";
+    AppendExtraHeaders(headers.get(), params);
+    return headers;
+  }
+
+  // Add "Range" header.
+  std::string range_header =
+      (params->length() == DownloadSaveInfo::kLengthFullContent)
+          ? base::StringPrintf("bytes=%" PRId64 "-", params->offset())
+          : base::StringPrintf("bytes=%" PRId64 "-%" PRId64, params->offset(),
+                               params->offset() + params->length() - 1);
+  headers->SetHeader(net::HttpRequestHeaders::kRange, range_header);
+
+  // Add "If-Range" headers.
+  if (params->use_if_range()) {
+    // In accordance with RFC 7233 Section 3.2, use If-Range to specify that
+    // the server return the entire entity if the validator doesn't match.
+    // Last-Modified can be used in the absence of ETag as a validator if the
+    // response headers satisfied the HttpUtil::HasStrongValidators()
+    // predicate.
+    //
+    // This function assumes that HasStrongValidators() was true and that the
+    // ETag and Last-Modified header values supplied are valid.
+    headers->SetHeader(net::HttpRequestHeaders::kIfRange,
+                       has_etag ? params->etag() : params->last_modified());
+    AppendExtraHeaders(headers.get(), params);
+    return headers;
+  }
+
+  // Add "If-Match"/"If-Unmodified-Since" headers.
+  if (has_etag)
+    headers->SetHeader(net::HttpRequestHeaders::kIfMatch, params->etag());
+
+  // According to RFC 7232 section 3.4, "If-Unmodified-Since" is mainly for
+  // old servers that didn't implement "If-Match" and must be ignored when
+  // "If-Match" presents.
+  if (has_last_modified) {
+    headers->SetHeader(net::HttpRequestHeaders::kIfUnmodifiedSince,
+                       params->last_modified());
+  }
+
+  AppendExtraHeaders(headers.get(), params);
+  return headers;
+}
+
 }  // namespace download

content/browser/download/download_worker.cc → components/download/internal/common/download_worker.cc

@@ -2,81 +2,56 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-#include "content/browser/download/download_worker.h"
+#include "components/download/public/common/download_worker.h"
 
 #include "base/message_loop/message_loop.h"
 #include "components/download/public/common/download_create_info.h"
 #include "components/download/public/common/download_interrupt_reasons.h"
 #include "components/download/public/common/download_task_runner.h"
+#include "components/download/public/common/download_utils.h"
 #include "components/download/public/common/input_stream.h"
 #include "components/download/public/common/resource_downloader.h"
-#include "content/browser/download/download_utils.h"
-#include "content/browser/download/url_downloader.h"
-#include "content/public/common/content_features.h"
+#include "components/download/public/common/url_download_handler_factory.h"
 #include "services/network/public/cpp/features.h"
 #include "services/network/public/cpp/shared_url_loader_factory.h"
 
-namespace content {
+namespace download {
 namespace {
 
 const int kWorkerVerboseLevel = 1;
 
-class CompletedInputStream : public download::InputStream {
+class CompletedInputStream : public InputStream {
  public:
-  CompletedInputStream(download::DownloadInterruptReason status)
-      : status_(status){};
+  CompletedInputStream(DownloadInterruptReason status) : status_(status){};
   ~CompletedInputStream() override = default;
 
-  // download::InputStream
+  // InputStream
   bool IsEmpty() override { return false; }
-  download::InputStream::StreamState Read(scoped_refptr<net::IOBuffer>* data,
-                                          size_t* length) override {
+  InputStream::StreamState Read(scoped_refptr<net::IOBuffer>* data,
+                                size_t* length) override {
     *length = 0;
     return InputStream::StreamState::COMPLETE;
   }
 
-  download::DownloadInterruptReason GetCompletionStatus() override {
-    return status_;
-  }
+  DownloadInterruptReason GetCompletionStatus() override { return status_; }
 
  private:
-  download::DownloadInterruptReason status_;
+  DownloadInterruptReason status_;
   DISALLOW_COPY_AND_ASSIGN(CompletedInputStream);
 };
 
 void CreateUrlDownloadHandler(
-    std::unique_ptr<download::DownloadUrlParameters> params,
-    base::WeakPtr<download::UrlDownloadHandler::Delegate> delegate,
-    scoped_refptr<URLLoaderFactoryGetter> url_loader_factory_getter,
+    std::unique_ptr<DownloadUrlParameters> params,
+    base::WeakPtr<UrlDownloadHandler::Delegate> delegate,
+    scoped_refptr<network::SharedURLLoaderFactory> shared_url_loader_factory,
     const scoped_refptr<base::SingleThreadTaskRunner>& task_runner) {
-  download::UrlDownloadHandler::UniqueUrlDownloadHandlerPtr downloader(
-      nullptr, base::OnTaskRunnerDeleter(base::ThreadTaskRunnerHandle::Get()));
-
-  if (base::FeatureList::IsEnabled(network::features::kNetworkService)) {
-    std::unique_ptr<network::ResourceRequest> request =
-        CreateResourceRequest(params.get());
-    downloader.reset(download::ResourceDownloader::BeginDownload(
-                         delegate, std::move(params), std::move(request),
-                         url_loader_factory_getter->GetNetworkFactory(), GURL(),
-                         GURL(), GURL(), download::DownloadItem::kInvalidId,
-                         true, task_runner)
-                         .release());
-  } else {
-    // Build the URLRequest, BlobDataHandle is hold in original request for
-    // image download.
-    std::unique_ptr<net::URLRequest> url_request =
-        DownloadRequestCore::CreateRequestOnIOThread(
-            download::DownloadItem::kInvalidId, params.get());
-
-    downloader.reset(UrlDownloader::BeginDownload(
-                         delegate, std::move(url_request), params.get(), true)
-                         .release());
-  }
+  auto downloader = UrlDownloadHandlerFactory::Create(
+      std::move(params), delegate, std::move(shared_url_loader_factory),
+      task_runner);
   task_runner->PostTask(
       FROM_HERE,
-      base::BindOnce(
-          &download::UrlDownloadHandler::Delegate::OnUrlDownloadHandlerCreated,
-          delegate, std::move(downloader)));
+      base::BindOnce(&UrlDownloadHandler::Delegate::OnUrlDownloadHandlerCreated,
+                     delegate, std::move(downloader)));
 }
 
 }  // namespace
@@ -98,13 +73,13 @@ DownloadWorker::DownloadWorker(DownloadWorker::Delegate* delegate,
 DownloadWorker::~DownloadWorker() = default;
 
 void DownloadWorker::SendRequest(
-    std::unique_ptr<download::DownloadUrlParameters> params,
-    scoped_refptr<URLLoaderFactoryGetter> url_loader_factory_getter) {
-  download::GetIOTaskRunner()->PostTask(
-      FROM_HERE,
-      base::BindOnce(&CreateUrlDownloadHandler, std::move(params),
-                     weak_factory_.GetWeakPtr(), url_loader_factory_getter,
-                     base::ThreadTaskRunnerHandle::Get()));
+    std::unique_ptr<DownloadUrlParameters> params,
+    scoped_refptr<network::SharedURLLoaderFactory> shared_url_loader_factory) {
+  GetIOTaskRunner()->PostTask(
+      FROM_HERE, base::BindOnce(&CreateUrlDownloadHandler, std::move(params),
+                                weak_factory_.GetWeakPtr(),
+                                std::move(shared_url_loader_factory),
+                                base::ThreadTaskRunnerHandle::Get()));
 }
 
 void DownloadWorker::Pause() {
@@ -127,9 +102,10 @@ void DownloadWorker::Cancel(bool user_cancel) {
 }
 
 void DownloadWorker::OnUrlDownloadStarted(
-    std::unique_ptr<download::DownloadCreateInfo> create_info,
-    std::unique_ptr<download::InputStream> input_stream,
-    const download::DownloadUrlParameters::OnStartedCallback& callback) {
+    std::unique_ptr<DownloadCreateInfo> create_info,
+    std::unique_ptr<InputStream> input_stream,
+    scoped_refptr<network::SharedURLLoaderFactory> shared_url_loader_factory,
+    const DownloadUrlParameters::OnStartedCallback& callback) {
   // |callback| is not used in subsequent requests.
   DCHECK(callback.is_null());
 
@@ -142,7 +118,7 @@ void DownloadWorker::OnUrlDownloadStarted(
   }
 
   // TODO(xingliu): Add metric for error handling.
-  if (create_info->result != download::DOWNLOAD_INTERRUPT_REASON_NONE) {
+  if (create_info->result != DOWNLOAD_INTERRUPT_REASON_NONE) {
     VLOG(kWorkerVerboseLevel)
         << "Parallel download sub-request failed. reason = "
         << create_info->result;
@@ -161,16 +137,15 @@ void DownloadWorker::OnUrlDownloadStarted(
   delegate_->OnInputStreamReady(this, std::move(input_stream));
 }
 
-void DownloadWorker::OnUrlDownloadStopped(
-    download::UrlDownloadHandler* downloader) {
+void DownloadWorker::OnUrlDownloadStopped(UrlDownloadHandler* downloader) {
   // Release the |url_download_handler_|, the object will be deleted on IO
   // thread.
   url_download_handler_.reset();
 }
 
 void DownloadWorker::OnUrlDownloadHandlerCreated(
-    download::UrlDownloadHandler::UniqueUrlDownloadHandlerPtr downloader) {
+    UrlDownloadHandler::UniqueUrlDownloadHandlerPtr downloader) {
   url_download_handler_ = std::move(downloader);
 }
 
-}  // namespace content
+}  // namespace download