[leveldb] tracking memory used by in-memory Env's.

leveldb::InMemoryEnv is not exposed with no official support for
calculating memory usage. However, this is a pretty decent estimate
and will be helpful for tracking in-memory Env RAM use.

Bug: 750803
Change-Id: I4e07327bf4480bdb786290c82d8f66412868f36a
Reviewed-on: https://chromium-review.googlesource.com/601207
Reviewed-by: Kinuko Yasuda <kinuko@chromium.org>
Reviewed-by: Jochen Eisinger <jochen@chromium.org>
Reviewed-by: Siddhartha S <ssid@chromium.org>
Cr-Commit-Position: refs/heads/master@{#552361}

PRESUBMIT.py

@@ -402,7 +402,8 @@
       'leveldb::NewMemEnv',
       (
         'Instead of leveldb::NewMemEnv() use leveldb_chrome::NewMemEnv() from',
-        'third_party/leveldatabase/leveldb_chrome.h.',
+        'third_party/leveldatabase/leveldb_chrome.h. It exposes environments',
+        "to Chrome's tracing, making their memory usage visible.",
       ),
       True,
       (

base/trace_event/memory_infra_background_whitelist.cc

@@ -106,6 +106,7 @@ const char* const kAllocatorDumpNameWhitelist[] = {
     "leveldatabase/block_cache/web",
     "leveldatabase/db_0x?",
     "leveldatabase/db_0x?/block_cache",
+    "leveldatabase/memenv_0x?",
     "malloc",
     "malloc/allocated_objects",
     "malloc/metadata_fragmentation_caches",
@@ -259,7 +260,8 @@ const char* const kAllocatorDumpNameWhitelist[] = {
     "v8/isolate_0x?/malloc",
     "v8/isolate_0x?/zapped_for_debug",
     "site_storage/blob_storage/0x?",
-    "site_storage/index_db/0x?",
+    "site_storage/index_db/db_0x?",
+    "site_storage/index_db/memenv_0x?",
     "site_storage/localstorage/0x?/cache_size",
     "site_storage/localstorage/0x?/leveldb",
     "site_storage/session_storage/0x?",

chrome/browser/extensions/api/sync_file_system/sync_file_system_browsertest.cc

@@ -69,7 +69,7 @@ class SyncFileSystemTest : public extensions::PlatformAppBrowserTest,
   }
 
   void SetUpOnMainThread() override {
-    in_memory_env_.reset(leveldb_chrome::NewMemEnv(leveldb::Env::Default()));
+    in_memory_env_ = leveldb_chrome::NewMemEnv("SyncFileSystemTest");
     extensions::PlatformAppBrowserTest::SetUpOnMainThread();
     ASSERT_TRUE(base_dir_.CreateUniqueTempDir());
 

chrome/browser/sync_file_system/drive_backend/conflict_resolver_unittest.cc

@@ -58,7 +58,7 @@ class ConflictResolverTest : public testing::Test {
 
   void SetUp() override {
     ASSERT_TRUE(database_dir_.CreateUniqueTempDir());
-    in_memory_env_.reset(leveldb_chrome::NewMemEnv(leveldb::Env::Default()));
+    in_memory_env_ = leveldb_chrome::NewMemEnv("ConflictResolverTest");
 
     std::unique_ptr<FakeDriveServiceWrapper> fake_drive_service(
         new FakeDriveServiceWrapper);

chrome/browser/sync_file_system/drive_backend/drive_backend_sync_unittest.cc

@@ -87,7 +87,7 @@ class DriveBackendSyncTest : public testing::Test,
 
   void SetUp() override {
     ASSERT_TRUE(base_dir_.CreateUniqueTempDir());
-    in_memory_env_.reset(leveldb_chrome::NewMemEnv(leveldb::Env::Default()));
+    in_memory_env_ = leveldb_chrome::NewMemEnv("DriveBackendSyncTest");
 
     io_task_runner_ = content::BrowserThread::GetTaskRunnerForThread(
         content::BrowserThread::IO);

chrome/browser/sync_file_system/drive_backend/leveldb_wrapper_unittest.cc

@@ -31,7 +31,7 @@ class LevelDBWrapperTest : public testing::Test {
 
   void SetUp() override {
     ASSERT_TRUE(database_dir_.CreateUniqueTempDir());
-    in_memory_env_.reset(leveldb_chrome::NewMemEnv(leveldb::Env::Default()));
+    in_memory_env_ = leveldb_chrome::NewMemEnv("LevelDBWrapperTest");
     InitializeLevelDB();
   }
 

chrome/browser/sync_file_system/drive_backend/list_changes_task_unittest.cc

@@ -44,7 +44,7 @@ class ListChangesTaskTest : public testing::Test {
 
   void SetUp() override {
     ASSERT_TRUE(database_dir_.CreateUniqueTempDir());
-    in_memory_env_.reset(leveldb_chrome::NewMemEnv(leveldb::Env::Default()));
+    in_memory_env_ = leveldb_chrome::NewMemEnv("ListChangesTaskTest");
 
     std::unique_ptr<drive::FakeDriveService> fake_drive_service(
         new drive::FakeDriveService);

chrome/browser/sync_file_system/drive_backend/local_to_remote_syncer_unittest.cc

@@ -59,7 +59,7 @@ class LocalToRemoteSyncerTest : public testing::Test {
 
   void SetUp() override {
     ASSERT_TRUE(database_dir_.CreateUniqueTempDir());
-    in_memory_env_.reset(leveldb_chrome::NewMemEnv(leveldb::Env::Default()));
+    in_memory_env_ = leveldb_chrome::NewMemEnv("LocalToRemoteSyncerTest");
 
     std::unique_ptr<FakeDriveServiceWrapper> fake_drive_service(
         new FakeDriveServiceWrapper);

chrome/browser/sync_file_system/drive_backend/metadata_database_index_on_disk_unittest.cc

@@ -40,7 +40,8 @@ class MetadataDatabaseIndexOnDiskTest : public testing::Test {
 
   void SetUp() override {
     ASSERT_TRUE(database_dir_.CreateUniqueTempDir());
-    in_memory_env_.reset(leveldb_chrome::NewMemEnv(leveldb::Env::Default()));
+    in_memory_env_ =
+        leveldb_chrome::NewMemEnv("MetadataDatabaseIndexOnDiskTest");
     db_ = InitializeLevelDB();
     index_ = MetadataDatabaseIndexOnDisk::Create(db_.get());
   }

chrome/browser/sync_file_system/drive_backend/metadata_database_index_unittest.cc

@@ -68,7 +68,7 @@ std::unique_ptr<DatabaseContents> CreateTestDatabaseContents() {
 class MetadataDatabaseIndexTest : public testing::Test {
  public:
   void SetUp() override {
-    in_memory_env_.reset(leveldb_chrome::NewMemEnv(leveldb::Env::Default()));
+    in_memory_env_ = leveldb_chrome::NewMemEnv("MetadataDatabaseIndexTest");
     InitializeLevelDB();
 
     contents_ = CreateTestDatabaseContents();

chrome/browser/sync_file_system/drive_backend/metadata_database_unittest.cc

@@ -198,7 +198,7 @@ class MetadataDatabaseTest : public testing::TestWithParam<bool> {
 
   void SetUp() override {
     ASSERT_TRUE(database_dir_.CreateUniqueTempDir());
-    in_memory_env_.reset(leveldb_chrome::NewMemEnv(leveldb::Env::Default()));
+    in_memory_env_ = leveldb_chrome::NewMemEnv("MetadataDatabaseTest");
   }
 
   void TearDown() override { DropDatabase(); }

chrome/browser/sync_file_system/drive_backend/register_app_task_unittest.cc

@@ -50,7 +50,7 @@ class RegisterAppTaskTest : public testing::Test {
 
   void SetUp() override {
     ASSERT_TRUE(database_dir_.CreateUniqueTempDir());
-    in_memory_env_.reset(leveldb_chrome::NewMemEnv(leveldb::Env::Default()));
+    in_memory_env_ = leveldb_chrome::NewMemEnv("RegisterAppTaskTest");
 
     std::unique_ptr<drive::FakeDriveService> fake_drive_service(
         new drive::FakeDriveService);

chrome/browser/sync_file_system/drive_backend/remote_to_local_syncer_unittest.cc

@@ -55,7 +55,7 @@ class RemoteToLocalSyncerTest : public testing::Test {
 
   void SetUp() override {
     ASSERT_TRUE(database_dir_.CreateUniqueTempDir());
-    in_memory_env_.reset(leveldb_chrome::NewMemEnv(leveldb::Env::Default()));
+    in_memory_env_ = leveldb_chrome::NewMemEnv("RemoteToLocalSyncerTest");
 
     std::unique_ptr<drive::FakeDriveService> fake_drive_service(
         new drive::FakeDriveService);

chrome/browser/sync_file_system/drive_backend/sync_engine_initializer_unittest.cc

@@ -51,7 +51,7 @@ class SyncEngineInitializerTest : public testing::Test {
 
   void SetUp() override {
     ASSERT_TRUE(database_dir_.CreateUniqueTempDir());
-    in_memory_env_.reset(leveldb_chrome::NewMemEnv(leveldb::Env::Default()));
+    in_memory_env_ = leveldb_chrome::NewMemEnv("SyncEngineInitializerTest");
 
     std::unique_ptr<drive::FakeDriveService> fake_drive_service(
         new drive::FakeDriveService);

chrome/browser/sync_file_system/drive_backend/sync_worker_unittest.cc

@@ -105,7 +105,7 @@ class SyncWorkerTest : public testing::Test,
 
   void SetUp() override {
     ASSERT_TRUE(profile_dir_.CreateUniqueTempDir());
-    in_memory_env_.reset(leveldb_chrome::NewMemEnv(leveldb::Env::Default()));
+    in_memory_env_ = leveldb_chrome::NewMemEnv("SyncWorkerTest");
 
     extension_service_.reset(new MockExtensionService);
     std::unique_ptr<drive::DriveServiceInterface> fake_drive_service(

chrome/browser/sync_file_system/local/local_file_change_tracker_unittest.cc

@@ -40,7 +40,7 @@ class LocalFileChangeTrackerTest : public testing::Test {
  public:
   LocalFileChangeTrackerTest()
       : thread_bundle_(content::TestBrowserThreadBundle::IO_MAINLOOP),
-        in_memory_env_(leveldb_chrome::NewMemEnv(leveldb::Env::Default())),
+        in_memory_env_(leveldb_chrome::NewMemEnv("LocalFileChangeTrackerTest")),
         file_system_(GURL("http://example.com"),
                      in_memory_env_.get(),
                      base::ThreadTaskRunnerHandle::Get().get(),

chrome/browser/sync_file_system/local/local_file_sync_context_unittest.cc

@@ -67,7 +67,7 @@ class LocalFileSyncContextTest : public testing::Test {
   void SetUp() override {
     RegisterSyncableFileSystem();
     ASSERT_TRUE(dir_.CreateUniqueTempDir());
-    in_memory_env_.reset(leveldb_chrome::NewMemEnv(leveldb::Env::Default()));
+    in_memory_env_ = leveldb_chrome::NewMemEnv("LocalFileSyncContextTest");
 
     ui_task_runner_ = base::ThreadTaskRunnerHandle::Get();
     io_task_runner_ = BrowserThread::GetTaskRunnerForThread(BrowserThread::IO);

chrome/browser/sync_file_system/local/local_file_sync_service_unittest.cc

@@ -113,7 +113,7 @@ class LocalFileSyncServiceTest
 
   void SetUp() override {
     ASSERT_TRUE(temp_dir_.CreateUniqueTempDir());
-    in_memory_env_.reset(leveldb_chrome::NewMemEnv(leveldb::Env::Default()));
+    in_memory_env_ = leveldb_chrome::NewMemEnv("LocalFileSyncServiceTest");
 
     file_system_.reset(new CannedSyncableFileSystem(
         GURL(kOrigin), in_memory_env_.get(),

chrome/browser/sync_file_system/local/syncable_file_operation_runner_unittest.cc

@@ -55,7 +55,8 @@ class SyncableFileOperationRunnerTest : public testing::Test {
   // operations in the tests.
   SyncableFileOperationRunnerTest()
       : thread_bundle_(content::TestBrowserThreadBundle::IO_MAINLOOP),
-        in_memory_env_(leveldb_chrome::NewMemEnv(leveldb::Env::Default())),
+        in_memory_env_(
+            leveldb_chrome::NewMemEnv("SyncableFileOperationRunnerTest")),
         file_system_(GURL("http://example.com"),
                      in_memory_env_.get(),
                      base::ThreadTaskRunnerHandle::Get().get(),

chrome/browser/sync_file_system/local/syncable_file_system_unittest.cc

@@ -35,7 +35,7 @@ namespace sync_file_system {
 class SyncableFileSystemTest : public testing::Test {
  public:
   SyncableFileSystemTest()
-      : in_memory_env_(leveldb_chrome::NewMemEnv(leveldb::Env::Default())),
+      : in_memory_env_(leveldb_chrome::NewMemEnv("SyncableFileSystemTest")),
         file_system_(GURL("http://example.com/"),
                      in_memory_env_.get(),
                      base::ThreadTaskRunnerHandle::Get().get(),

chrome/browser/sync_file_system/sync_file_system_service_unittest.cc

@@ -131,7 +131,7 @@ class SyncFileSystemServiceTest : public testing::Test {
       : thread_bundle_(content::TestBrowserThreadBundle::REAL_IO_THREAD) {}
 
   void SetUp() override {
-    in_memory_env_.reset(leveldb_chrome::NewMemEnv(leveldb::Env::Default()));
+    in_memory_env_ = leveldb_chrome::NewMemEnv("SyncFileSystemServiceTest");
     file_system_.reset(new CannedSyncableFileSystem(
         GURL(kOrigin), in_memory_env_.get(),
         BrowserThread::GetTaskRunnerForThread(BrowserThread::IO),

components/leveldb_proto/leveldb_database.cc

@@ -49,7 +49,7 @@ bool LevelDB::Init(const base::FilePath& database_dir,
   open_options_ = options;
 
   if (database_dir.empty()) {
-    env_.reset(leveldb_chrome::NewMemEnv(leveldb::Env::Default()));
+    env_ = leveldb_chrome::NewMemEnv("LevelDB");
     open_options_.env = env_.get();
   }
 

components/leveldb_proto/proto_database_impl_unittest.cc

@@ -346,9 +346,20 @@ TEST_F(ProtoDatabaseImplTest, TestDBGetSuccess) {
   base::RunLoop().RunUntilIdle();
 }
 
-TEST(ProtoDatabaseImplLevelDBTest, TestDBSaveAndLoadKeys) {
-  base::MessageLoop main_loop;
+class ProtoDatabaseImplLevelDBTest : public testing::Test {
+ public:
+  void SetUp() override { main_loop_.reset(new MessageLoop()); }
+
+  void TearDown() override {
+    base::RunLoop().RunUntilIdle();
+    main_loop_.reset();
+  }
+
+ private:
+  std::unique_ptr<MessageLoop> main_loop_;
+};
 
+TEST_F(ProtoDatabaseImplLevelDBTest, TestDBSaveAndLoadKeys) {
   ScopedTempDir temp_dir;
   ASSERT_TRUE(temp_dir.CreateUniqueTempDir());
   base::Thread db_thread("dbthread");
@@ -677,15 +688,15 @@ void TestLevelDBSaveAndLoad(bool close_after_save) {
   ExpectEntryPointersEquals(model, loaded_protos);
 }
 
-TEST(ProtoDatabaseImplLevelDBTest, TestDBSaveAndLoad) {
+TEST_F(ProtoDatabaseImplLevelDBTest, TestDBSaveAndLoad) {
   TestLevelDBSaveAndLoad(false);
 }
 
-TEST(ProtoDatabaseImplLevelDBTest, TestDBCloseAndReopen) {
+TEST_F(ProtoDatabaseImplLevelDBTest, TestDBCloseAndReopen) {
   TestLevelDBSaveAndLoad(true);
 }
 
-TEST(ProtoDatabaseImplLevelDBTest, TestDBInitFail) {
+TEST_F(ProtoDatabaseImplLevelDBTest, TestDBInitFail) {
   ScopedTempDir temp_dir;
   ASSERT_TRUE(temp_dir.CreateUniqueTempDir());
 
@@ -702,7 +713,7 @@ TEST(ProtoDatabaseImplLevelDBTest, TestDBInitFail) {
   EXPECT_FALSE(db->Save(save_entries, remove_keys));
 }
 
-TEST(ProtoDatabaseImplLevelDBTest, TestMemoryDatabase) {
+TEST_F(ProtoDatabaseImplLevelDBTest, TestMemoryDatabase) {
   std::unique_ptr<LevelDB> db(new LevelDB(kTestLevelDBClientName));
 
   std::vector<std::string> load_entries;
@@ -723,7 +734,7 @@ TEST(ProtoDatabaseImplLevelDBTest, TestMemoryDatabase) {
   EXPECT_EQ(1u, second_load_entries.size());
 }
 
-TEST(ProtoDatabaseImplLevelDBTest, TestCorruptDBReset) {
+TEST_F(ProtoDatabaseImplLevelDBTest, TestCorruptDBReset) {
   ScopedTempDir temp_dir;
   ASSERT_TRUE(temp_dir.CreateUniqueTempDir());
 

components/services/leveldb/leveldb_service_impl.cc

@@ -74,14 +74,14 @@ void LevelDBServiceImpl::OpenWithOptions(
 void LevelDBServiceImpl::OpenInMemory(
     const base::Optional<base::trace_event::MemoryAllocatorDumpGuid>&
         memory_dump_id,
+    const std::string& tracking_name,
     leveldb::mojom::LevelDBDatabaseAssociatedRequest database,
     OpenCallback callback) {
   leveldb_env::Options options;
   options.create_if_missing = true;
   options.max_open_files = 0;  // Use minimum.
 
-  std::unique_ptr<leveldb::Env> env(
-      leveldb_chrome::NewMemEnv(leveldb::Env::Default()));
+  auto env = leveldb_chrome::NewMemEnv(tracking_name);
   options.env = env.get();
 
   std::unique_ptr<leveldb::DB> db;

components/services/leveldb/leveldb_service_impl.h

@@ -43,6 +43,7 @@ class LevelDBServiceImpl : public mojom::LevelDBService {
   void OpenInMemory(
       const base::Optional<base::trace_event::MemoryAllocatorDumpGuid>&
           memory_dump_id,
+      const std::string& tracking_name,
       leveldb::mojom::LevelDBDatabaseAssociatedRequest database,
       OpenInMemoryCallback callback) override;
   void Destroy(filesystem::mojom::DirectoryPtr directory,

components/services/leveldb/leveldb_service_unittest.cc

@@ -108,7 +108,7 @@ void LevelDBSyncOpenInMemory(mojom::LevelDBService* leveldb,
                              mojom::LevelDBDatabaseAssociatedRequest database,
                              mojom::DatabaseError* out_error) {
   base::RunLoop run_loop;
-  leveldb->OpenInMemory(base::nullopt, std::move(database),
+  leveldb->OpenInMemory(base::nullopt, "LevelDBSync", std::move(database),
                         Capture(out_error, run_loop.QuitClosure()));
   run_loop.Run();
 }

components/services/leveldb/public/interfaces/leveldb.mojom

@@ -91,8 +91,10 @@ interface LevelDBService {
                   associated LevelDBDatabase& database) => (DatabaseError status);
 
   // Opens a database stored purely in memory.
+  // "tracking_name" will be used for memory-infra reporting to associate memory
+  // use with its origin.
   OpenInMemory(mojo_base.mojom.MemoryAllocatorDumpCrossProcessUid?
-               memory_dump_id,
+               memory_dump_id, string tracking_name,
                associated LevelDBDatabase& database) => (DatabaseError status);
 
   // Destroys the contents of the specified database. Returns OK if the database

components/services/leveldb/remote_iterator_unittest.cc

@@ -52,7 +52,8 @@ class RemoteIteratorTest : public service_manager::test::ServiceTest {
 
     mojom::DatabaseError error;
     base::RunLoop run_loop;
-    leveldb()->OpenInMemory(base::nullopt, MakeRequest(&database_),
+    leveldb()->OpenInMemory(base::nullopt, "RemoteIteratorTest",
+                            MakeRequest(&database_),
                             Capture(&error, run_loop.QuitClosure()));
     run_loop.Run();
     EXPECT_EQ(mojom::DatabaseError::OK, error);

components/sync/model_impl/model_type_store_backend.cc

@@ -106,7 +106,7 @@ ModelTypeStoreBackend::~ModelTypeStoreBackend() {
 }
 
 std::unique_ptr<leveldb::Env> ModelTypeStoreBackend::CreateInMemoryEnv() {
-  return base::WrapUnique(leveldb_chrome::NewMemEnv(leveldb::Env::Default()));
+  return leveldb_chrome::NewMemEnv("ModelTypeStore");
 }
 
 // static

content/browser/dom_storage/local_storage_context_mojo.cc

@@ -679,7 +679,7 @@ void LocalStorageContextMojo::InitiateConnection(bool in_memory_only) {
     // We were not given a subdirectory. Use a memory backed database.
     connector_->BindInterface(file::mojom::kServiceName, &leveldb_service_);
     leveldb_service_->OpenInMemory(
-        memory_dump_id_, MakeRequest(&database_),
+        memory_dump_id_, "local-storage", MakeRequest(&database_),
         base::BindOnce(&LocalStorageContextMojo::OnDatabaseOpened,
                        weak_ptr_factory_.GetWeakPtr(), true));
   }

content/browser/dom_storage/local_storage_context_mojo_unittest.cc

@@ -1126,7 +1126,7 @@ class FakeLevelDBService : public leveldb::mojom::LevelDBService {
             leveldb::mojom::LevelDBDatabaseAssociatedRequest request,
             OpenCallback callback) override {
     open_requests_.push_back(
-        {false, dbname, std::move(request), std::move(callback)});
+        {false, dbname, "", std::move(request), std::move(callback)});
     if (on_open_callback_)
       on_open_callback_.Run();
   }
@@ -1140,18 +1140,19 @@ class FakeLevelDBService : public leveldb::mojom::LevelDBService {
       leveldb::mojom::LevelDBDatabaseAssociatedRequest request,
       OpenCallback callback) override {
     open_requests_.push_back(
-        {false, dbname, std::move(request), std::move(callback)});
+        {false, dbname, "", std::move(request), std::move(callback)});
     if (on_open_callback_)
       on_open_callback_.Run();
   }
 
   void OpenInMemory(
       const base::Optional<base::trace_event::MemoryAllocatorDumpGuid>&
           memory_dump_id,
+      const std::string& tracking_name,
       leveldb::mojom::LevelDBDatabaseAssociatedRequest request,
       OpenCallback callback) override {
     open_requests_.push_back(
-        {true, "", std::move(request), std::move(callback)});
+        {true, "", tracking_name, std::move(request), std::move(callback)});
     if (on_open_callback_)
       on_open_callback_.Run();
   }
@@ -1166,6 +1167,7 @@ class FakeLevelDBService : public leveldb::mojom::LevelDBService {
   struct OpenRequest {
     bool in_memory;
     std::string dbname;
+    std::string memenv_tracking_name;
     leveldb::mojom::LevelDBDatabaseAssociatedRequest request;
     OpenCallback callback;
   };

content/browser/fileapi/browser_file_system_helper.cc

@@ -62,11 +62,12 @@ FileSystemOptions CreateBrowserFileSystemOptions(bool is_incognito) {
           switches::kAllowFileAccessFromFiles)) {
     additional_allowed_schemes.push_back(url::kFileScheme);
   }
-  leveldb::Env* env_override = nullptr;
+  std::unique_ptr<leveldb::Env> env_override;
   if (is_incognito)
-    env_override = leveldb_chrome::NewMemEnv(leveldb::Env::Default());
+    env_override = leveldb_chrome::NewMemEnv("filesystem");
+  // TODO(crbug.com/823854) Fix leaking override MemEnv.
   return FileSystemOptions(profile_mode, additional_allowed_schemes,
-                           env_override);
+                           env_override.release());
 }
 
 }  // namespace