Skip to content

Commit

Permalink
Verify checksum for downloaded updates and discard the update if the …
Browse files Browse the repository at this point in the history 
…checks fails to match.

BUG=543161

Review-Url: https://codereview.chromium.org/2206733002
Cr-Commit-Position: refs/heads/master@{#411225}
  • Loading branch information
@aawc
aawc authored and Commit bot committed Aug 11, 2016
1 parent af18219 commit 0facbb7
Show file tree
Hide file tree
Showing 6 changed files with 274 additions and 80 deletions.
2 changes: 2 additions & 0 deletions components/safe_browsing_db/BUILD.gn
View file
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,7 @@ source_set("v4_store") {
":v4_protocol_manager_util",
":v4_rice",
"//base",
"//crypto",
]
}

Expand Down Expand Up @@ -304,6 +305,7 @@ source_set("unit_tests") {
":v4_update_protocol_manager",
"//base",
"//content/test:test_support",
"//crypto",
"//net",
"//net:test_support",
"//testing/gtest",
Expand Down
12 changes: 9 additions & 3 deletions components/safe_browsing_db/v4_rice.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ namespace safe_browsing {

namespace {

const unsigned int kMaxBitIndex = 8 * sizeof(uint32_t);
const int kBitsPerByte = 8;
const unsigned int kMaxBitIndex = kBitsPerByte * sizeof(uint32_t);

void GetBytesFromUInt32(uint32_t word, char* bytes) {
const size_t mask = 0xFF;
Expand Down Expand Up @@ -264,10 +265,15 @@ void V4RiceDecoder::GetBitsFromCurrentWord(unsigned int num_requested_bits,
};

std::string V4RiceDecoder::DebugString() const {
// Calculates the total number of bits that we have read from the buffer,
// excluding those that have been read into current_word_ but not yet
// consumed byt GetNextBits().
unsigned bits_read = (data_byte_index_ - sizeof(uint32_t)) * kBitsPerByte +
current_word_bit_index_;
return base::StringPrintf(
"current_word_: %x; data_byte_index_; %x, "
"bits_read: %x; current_word_: %x; data_byte_index_; %x, "
"current_word_bit_index_: %x; rice_parameter_: %x",
current_word_, data_byte_index_, current_word_bit_index_,
bits_read, current_word_, data_byte_index_, current_word_bit_index_,
rice_parameter_);
}

Expand Down
133 changes: 97 additions & 36 deletions components/safe_browsing_db/v4_store.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,13 @@
#include "base/memory/ptr_util.h"
#include "base/metrics/histogram_macros.h"
#include "base/metrics/sparse_histogram.h"
#include "base/stl_util.h"
#include "base/strings/stringprintf.h"
#include "components/safe_browsing_db/v4_rice.h"
#include "components/safe_browsing_db/v4_store.h"
#include "components/safe_browsing_db/v4_store.pb.h"
#include "crypto/secure_hash.h"
#include "crypto/sha2.h"

namespace safe_browsing {

Expand Down Expand Up @@ -113,28 +116,44 @@ bool V4Store::Reset() {
return true;
}

ApplyUpdateResult V4Store::ProcessFullUpdate(
std::unique_ptr<ListUpdateResponse> response,
const std::unique_ptr<V4Store>& new_store) {
HashPrefixMap hash_prefix_map;
ApplyUpdateResult apply_update_result =
UpdateHashPrefixMapFromAdditions(response->additions(), &hash_prefix_map);
if (apply_update_result == APPLY_UPDATE_SUCCESS) {
new_store->hash_prefix_map_ = hash_prefix_map;
RecordStoreWriteResult(new_store->WriteToDisk(std::move(response)));
ApplyUpdateResult V4Store::ProcessPartialUpdateAndWriteToDisk(
const HashPrefixMap& hash_prefix_map_old,
std::unique_ptr<ListUpdateResponse> response) {
DCHECK(response->has_response_type());
DCHECK_EQ(ListUpdateResponse::PARTIAL_UPDATE, response->response_type());

ApplyUpdateResult result = ProcessUpdate(hash_prefix_map_old, response);
if (result == APPLY_UPDATE_SUCCESS) {
// TODO(vakh): Create a ListUpdateResponse containing RICE encoded
// hash prefixes and response_type as FULL_UPDATE, and write that to disk.
}
return apply_update_result;
return result;
}

ApplyUpdateResult V4Store::ProcessPartialUpdate(
std::unique_ptr<ListUpdateResponse> response,
const std::unique_ptr<V4Store>& new_store) {
// TODO(vakh):
// 1. Done: Merge the old store and the new update in new_store.
// 2. Create a ListUpdateResponse containing RICE encoded hash-prefixes and
// response_type as FULL_UPDATE, and write that to disk.
// 3. Remove this if condition after completing 1. and 2.
ApplyUpdateResult V4Store::ProcessFullUpdateAndWriteToDisk(
std::unique_ptr<ListUpdateResponse> response) {
ApplyUpdateResult result = ProcessFullUpdate(response);
if (result == APPLY_UPDATE_SUCCESS) {
RecordStoreWriteResult(WriteToDisk(std::move(response)));
}
return result;
}

ApplyUpdateResult V4Store::ProcessFullUpdate(
const std::unique_ptr<ListUpdateResponse>& response) {
DCHECK(response->has_response_type());
DCHECK_EQ(ListUpdateResponse::FULL_UPDATE, response->response_type());
// TODO(vakh): For a full update, we don't need to process the update in
// lexographical order to store it, but we do need to do that for calculating
// checksum. It might save some CPU cycles to store the full update as-is and
// walk the list of hash prefixes in lexographical order only for checksum
// calculation.
return ProcessUpdate(HashPrefixMap(), response);
}

ApplyUpdateResult V4Store::ProcessUpdate(
const HashPrefixMap& hash_prefix_map_old,
const std::unique_ptr<ListUpdateResponse>& response) {
const RepeatedField<int32>* raw_removals = nullptr;
RepeatedField<int32> rice_removals;
size_t removals_size = response->removals_size();
Expand Down Expand Up @@ -167,12 +186,23 @@ ApplyUpdateResult V4Store::ProcessPartialUpdate(
HashPrefixMap hash_prefix_map;
ApplyUpdateResult apply_update_result =
UpdateHashPrefixMapFromAdditions(response->additions(), &hash_prefix_map);
if (apply_update_result != APPLY_UPDATE_SUCCESS) {
return apply_update_result;
}

if (apply_update_result == APPLY_UPDATE_SUCCESS) {
apply_update_result =
new_store->MergeUpdate(hash_prefix_map_, hash_prefix_map, raw_removals);
std::string expected_checksum;
if (response->has_checksum() && response->checksum().has_sha256()) {
expected_checksum = response->checksum().sha256();
}
return apply_update_result;

apply_update_result = MergeUpdate(hash_prefix_map_old, hash_prefix_map,
raw_removals, expected_checksum);
if (apply_update_result != APPLY_UPDATE_SUCCESS) {
return apply_update_result;
}

state_ = response->new_client_state();
return APPLY_UPDATE_SUCCESS;
}

void V4Store::ApplyUpdate(
Expand All @@ -181,13 +211,14 @@ void V4Store::ApplyUpdate(
UpdatedStoreReadyCallback callback) {
std::unique_ptr<V4Store> new_store(
new V4Store(this->task_runner_, this->store_path_));
new_store->state_ = response->new_client_state();

ApplyUpdateResult apply_update_result;
if (response->response_type() == ListUpdateResponse::PARTIAL_UPDATE) {
apply_update_result = ProcessPartialUpdate(std::move(response), new_store);
apply_update_result = new_store->ProcessPartialUpdateAndWriteToDisk(
hash_prefix_map_, std::move(response));
} else if (response->response_type() == ListUpdateResponse::FULL_UPDATE) {
apply_update_result = ProcessFullUpdate(std::move(response), new_store);
apply_update_result =
new_store->ProcessFullUpdateAndWriteToDisk(std::move(response));
} else {
apply_update_result = UNEXPECTED_RESPONSE_TYPE_FAILURE;
NOTREACHED() << "Unexpected response type: " << response->response_type();
Expand All @@ -198,6 +229,8 @@ void V4Store::ApplyUpdate(
callback_task_runner->PostTask(
FROM_HERE, base::Bind(callback, base::Passed(&new_store)));
} else {
DVLOG(1) << "ApplyUpdate failed: reason: " << apply_update_result
<< "; store: " << *this;
// new_store failed updating. Pass a nullptr to the callback.
callback_task_runner->PostTask(FROM_HERE, base::Bind(callback, nullptr));
}
Expand Down Expand Up @@ -321,10 +354,10 @@ void V4Store::ReserveSpaceInPrefixMap(const HashPrefixMap& other_prefixes_map,
}
}

ApplyUpdateResult V4Store::MergeUpdate(
const HashPrefixMap& old_prefixes_map,
const HashPrefixMap& additions_map,
const RepeatedField<int32>* raw_removals) {
ApplyUpdateResult V4Store::MergeUpdate(const HashPrefixMap& old_prefixes_map,
const HashPrefixMap& additions_map,
const RepeatedField<int32>* raw_removals,
const std::string& expected_checksum) {
DCHECK(hash_prefix_map_.empty());
hash_prefix_map_.clear();
ReserveSpaceInPrefixMap(old_prefixes_map, &hash_prefix_map_);
Expand All @@ -347,6 +380,10 @@ ApplyUpdateResult V4Store::MergeUpdate(
// At least one of the maps still has elements that need to be merged into the
// new store.

bool calculate_checksum = !expected_checksum.empty();
std::unique_ptr<crypto::SecureHash> checksum_ctx(
crypto::SecureHash::Create(crypto::SecureHash::SHA256));

// Keep track of the number of elements picked from the old map. This is used
// to determine which elements to drop based on the raw_removals. Note that
// picked is not the same as merged. A picked element isn't merged if its
Expand Down Expand Up @@ -380,6 +417,11 @@ ApplyUpdateResult V4Store::MergeUpdate(
*removals_iter != total_picked_from_old) {
// Append the smallest hash to the appropriate list.
hash_prefix_map_[next_smallest_prefix_size] += next_smallest_prefix_old;

if (calculate_checksum) {
checksum_ctx->Update(string_as_array(&next_smallest_prefix_old),
next_smallest_prefix_size);
}
} else {
// Element not added to new map. Move the removals iterator forward.
removals_iter++;
Expand All @@ -397,6 +439,11 @@ ApplyUpdateResult V4Store::MergeUpdate(
hash_prefix_map_[next_smallest_prefix_size] +=
next_smallest_prefix_additions;

if (calculate_checksum) {
checksum_ctx->Update(string_as_array(&next_smallest_prefix_additions),
next_smallest_prefix_size);
}

// Update the iterator map, which means that we have merged one hash
// prefix of size |next_smallest_prefix_size| from the update.
additions_iterator_map[next_smallest_prefix_size] +=
Expand All @@ -409,9 +456,24 @@ ApplyUpdateResult V4Store::MergeUpdate(
}
}

return (!raw_removals || removals_iter == raw_removals->end())
? APPLY_UPDATE_SUCCESS
: REMOVALS_INDEX_TOO_LARGE_FAILURE;
if (raw_removals && removals_iter != raw_removals->end()) {
return REMOVALS_INDEX_TOO_LARGE_FAILURE;
}

if (calculate_checksum) {
std::string checksum(crypto::kSHA256Length, 0);
checksum_ctx->Finish(string_as_array(&checksum), checksum.size());
if (checksum != expected_checksum) {
std::string checksum_base64, expected_checksum_base64;
base::Base64Encode(checksum, &checksum_base64);
base::Base64Encode(expected_checksum, &expected_checksum_base64);
DVLOG(1) << "Checksum failed: calculated: " << checksum_base64
<< "expected: " << expected_checksum_base64;
return CHECKSUM_MISMATCH_FAILURE;
}
}

return APPLY_UPDATE_SUCCESS;
}

StoreReadResult V4Store::ReadFromDisk() {
Expand Down Expand Up @@ -450,16 +512,15 @@ StoreReadResult V4Store::ReadFromDisk() {
return HASH_PREFIX_INFO_MISSING_FAILURE;
}

const ListUpdateResponse& response = file_format.list_update_response();
ApplyUpdateResult apply_update_result = UpdateHashPrefixMapFromAdditions(
response.additions(), &hash_prefix_map_);
std::unique_ptr<ListUpdateResponse> response(new ListUpdateResponse);
response->Swap(file_format.mutable_list_update_response());
ApplyUpdateResult apply_update_result = ProcessFullUpdate(response);
RecordApplyUpdateResultWhenReadingFromDisk(apply_update_result);
if (apply_update_result != APPLY_UPDATE_SUCCESS) {
hash_prefix_map_.clear();
return HASH_PREFIX_MAP_GENERATION_FAILURE;
}

state_ = response.new_client_state();
return READ_SUCCESS;
}

Expand Down
50 changes: 37 additions & 13 deletions components/safe_browsing_db/v4_store.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,10 @@ enum ApplyUpdateResult {
// Compression type other than RAW and RICE for removals.
UNEXPECTED_COMPRESSION_TYPE_REMOVALS_FAILURE = 10,

// The state of the store did not match the expected checksum sent by the
// server.
CHECKSUM_MISMATCH_FAILURE = 11,

// Memory space for histograms is determined by the max. ALWAYS
// ADD NEW VALUES BEFORE THIS ONE.
APPLY_UPDATE_RESULT_MAX
Expand Down Expand Up @@ -245,6 +249,7 @@ class V4Store {
TestAdditionsWithRiceEncodingFailsWithInvalidInput);
FRIEND_TEST_ALL_PREFIXES(V4StoreTest, TestAdditionsWithRiceEncodingSucceeds);
FRIEND_TEST_ALL_PREFIXES(V4StoreTest, TestRemovalsWithRiceEncodingSucceeds);
FRIEND_TEST_ALL_PREFIXES(V4StoreTest, TestMergeUpdatesFailsChecksum);
friend class V4StoreTest;

// If |prefix_size| is within expected range, and |raw_hashes| is not invalid,
Expand Down Expand Up @@ -290,24 +295,43 @@ class V4Store {
// Merges the prefix map from the old store (|old_hash_prefix_map|) and the
// update (additions_map) to populate the prefix map for the current store.
// The indices in the |raw_removals| list, which may be NULL, are not merged.
// The SHA256 checksum of the final list of hash prefixes, in lexographically
// sorted order, must match |expected_checksum| (if it's not empty).
ApplyUpdateResult MergeUpdate(const HashPrefixMap& old_hash_prefix_map,
const HashPrefixMap& additions_map,
const ::google::protobuf::RepeatedField<
::google::protobuf::int32>* raw_removals);

// Processes the FULL_UPDATE |response| from the server and updates the
// V4Store in |new_store| and writes it to disk. If processing the |response|
// succeeds, it returns APPLY_UPDATE_SUCCESS.
ApplyUpdateResult ProcessFullUpdate(
std::unique_ptr<ListUpdateResponse> response,
const std::unique_ptr<V4Store>& new_store);
::google::protobuf::int32>* raw_removals,
const std::string& expected_checksum);

// Processes the PARTIAL_UPDATE |response| from the server and updates the
// V4Store in |new_store|. If processing the |response| succeeds, it returns
// Processes the FULL_UPDATE |response| from the server, and writes the
// merged V4Store to disk. If processing the |response| succeeds, it returns
// APPLY_UPDATE_SUCCESS.
ApplyUpdateResult ProcessPartialUpdate(
std::unique_ptr<ListUpdateResponse> response,
const std::unique_ptr<V4Store>& new_store);
// This method is only called when we receive a FULL_UPDATE from the server.
ApplyUpdateResult ProcessFullUpdateAndWriteToDisk(
std::unique_ptr<ListUpdateResponse> response);

// Processes a FULL_UPDATE |response| and updates the V4Store. If processing
// the |response| succeeds, it returns APPLY_UPDATE_SUCCESS.
// This method is called when we receive a FULL_UPDATE from the server, and
// when we read a store file from disk on startup.
ApplyUpdateResult ProcessFullUpdate(
const std::unique_ptr<ListUpdateResponse>& response);

// Merges the hash prefixes in |hash_prefix_map_old| and |response|, updates
// the |hash_prefix_map_| and |state_| in the V4Store, and writes the merged
// store to disk. If processing succeeds, it returns APPLY_UPDATE_SUCCESS.
// This method is only called when we receive a PARTIAL_UPDATE from the
// server.
ApplyUpdateResult ProcessPartialUpdateAndWriteToDisk(
const HashPrefixMap& hash_prefix_map_old,
std::unique_ptr<ListUpdateResponse> response);

// Merges the hash prefixes in |hash_prefix_map_old| and |response|, and
// updates the |hash_prefix_map_| and |state_| in the V4Store. If processing
// succeeds, it returns APPLY_UPDATE_SUCCESS.
ApplyUpdateResult ProcessUpdate(
const HashPrefixMap& hash_prefix_map_old,
const std::unique_ptr<ListUpdateResponse>& response);

// Reads the state of the store from the file on disk and returns the reason
// for the failure or reports success.
Expand Down
Loading

0 comments on commit 0facbb7

Please sign in to comment.