forked from chromium/chromium
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[PartitionAlloc] Replace SpinLock with a spinning lock on Linux kernels.
PartitionAlloc's main lock is the per-PartitionRoot lock. It has to be fast on the common uncontended path, and better than a simple spinlock when it's contended. Past attempts to replace it with base::Lock regressed low and high-level benchmarks. This commit introduces SpinningFutex, a futex()-based lock which spins in userspace like SpinLock before calling into the kernel, like base::Lock does. It is intended to be as fast as SpinLock, but better for contended cases. Since it relies on the linux-specific futex() system call, this is only available on Linux-based kernels, for now. Bug: 1125866, 1125999, 1061437 Change-Id: I1e0cf23beb48c32518d0ffc293d1a9326bc2d7bc Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2409902 Commit-Queue: Benoit L <lizeb@chromium.org> Reviewed-by: Kentaro Hara <haraken@chromium.org> Cr-Commit-Position: refs/heads/master@{#808343}
- Loading branch information
Benoit Lize
authored and
Commit Bot
committed
Sep 18, 2020
1 parent
d1c780a
commit d6ce9d7
Showing
7 changed files
with
355 additions
and
50 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
117 changes: 117 additions & 0 deletions
117
base/allocator/partition_allocator/partition_lock_unittest.cc
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,117 @@ | ||
| // Copyright 2020 The Chromium Authors. All rights reserved. | ||
| // Use of this source code is governed by a BSD-style license that can be | ||
| // found in the LICENSE file. | ||
|
|
||
| #include "base/allocator/partition_allocator/partition_lock.h" | ||
|
|
||
| #include "base/bind.h" | ||
| #include "base/callback.h" | ||
| #include "base/macros.h" | ||
| #include "base/test/bind_test_util.h" | ||
| #include "base/test/gtest_util.h" | ||
| #include "base/threading/platform_thread.h" | ||
| #include "base/time/time.h" | ||
| #include "testing/gtest/include/gtest/gtest.h" | ||
|
|
||
| namespace base { | ||
| namespace internal { | ||
| namespace { | ||
|
|
||
| class LambdaThreadDelegate : public PlatformThread::Delegate { | ||
| public: | ||
| explicit LambdaThreadDelegate(RepeatingClosure f) : f_(f) {} | ||
| void ThreadMain() override { f_.Run(); } | ||
|
|
||
| private: | ||
| RepeatingClosure f_; | ||
| }; | ||
|
|
||
| TEST(SpinLockTest, Simple) { | ||
| MaybeSpinLock<true> lock; | ||
| lock.Lock(); | ||
| lock.Unlock(); | ||
| } | ||
|
|
||
| MaybeSpinLock<true> g_lock; | ||
| TEST(SpinLockTest, StaticLockStartsUnlocked) { | ||
| g_lock.Lock(); | ||
| g_lock.Unlock(); | ||
| } | ||
|
|
||
| TEST(SpinLockTest, Contended) { | ||
| int counter = 0; // *Not* atomic. | ||
| std::vector<PlatformThreadHandle> thread_handles; | ||
| constexpr int iterations_per_thread = 1000000; | ||
| constexpr int num_threads = 4; | ||
|
|
||
| MaybeSpinLock<true> lock; | ||
| MaybeSpinLock<true> start_lock; | ||
|
|
||
| LambdaThreadDelegate delegate{BindLambdaForTesting([&]() { | ||
| start_lock.Lock(); | ||
| start_lock.Unlock(); | ||
|
|
||
| for (int i = 0; i < iterations_per_thread; i++) { | ||
| lock.Lock(); | ||
| counter++; | ||
| lock.Unlock(); | ||
| } | ||
| })}; | ||
|
|
||
| start_lock.Lock(); // Make sure that the threads compete, by waiting until | ||
| // all of them have at least been created. | ||
| for (int i = 0; i < num_threads; i++) { | ||
| PlatformThreadHandle handle; | ||
| PlatformThread::Create(0, &delegate, &handle); | ||
| thread_handles.push_back(handle); | ||
| } | ||
|
|
||
| start_lock.Unlock(); | ||
|
|
||
| for (int i = 0; i < num_threads; i++) { | ||
| PlatformThread::Join(thread_handles[i]); | ||
| } | ||
| EXPECT_EQ(iterations_per_thread * num_threads, counter); | ||
| } | ||
|
|
||
| TEST(SpinLockTest, SlowThreads) { | ||
| int counter = 0; // *Not* atomic. | ||
| std::vector<PlatformThreadHandle> thread_handles; | ||
| constexpr int iterations_per_thread = 100; | ||
| constexpr int num_threads = 4; | ||
|
|
||
| MaybeSpinLock<true> lock; | ||
| MaybeSpinLock<true> start_lock; | ||
|
|
||
| LambdaThreadDelegate delegate{BindLambdaForTesting([&]() { | ||
| start_lock.Lock(); | ||
| start_lock.Unlock(); | ||
|
|
||
| for (int i = 0; i < iterations_per_thread; i++) { | ||
| lock.Lock(); | ||
| counter++; | ||
| // Hold the lock for a while, to force futex()-based locks to sleep. | ||
| PlatformThread::Sleep(TimeDelta::FromMilliseconds(1)); | ||
| lock.Unlock(); | ||
| } | ||
| })}; | ||
|
|
||
| start_lock.Lock(); // Make sure that the threads compete, by waiting until | ||
| // all of them have at least been created. | ||
| for (int i = 0; i < num_threads; i++) { | ||
| PlatformThreadHandle handle; | ||
| PlatformThread::Create(0, &delegate, &handle); | ||
| thread_handles.push_back(handle); | ||
| } | ||
|
|
||
| start_lock.Unlock(); | ||
|
|
||
| for (int i = 0; i < num_threads; i++) { | ||
| PlatformThread::Join(thread_handles[i]); | ||
| } | ||
| EXPECT_EQ(iterations_per_thread * num_threads, counter); | ||
| } | ||
|
|
||
| } // namespace | ||
| } // namespace internal | ||
| } // namespace base |
73 changes: 73 additions & 0 deletions
73
base/allocator/partition_allocator/spinning_futex_linux.cc
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,73 @@ | ||
| // Copyright 2020 The Chromium Authors. All rights reserved. | ||
| // Use of this source code is governed by a BSD-style license that can be | ||
| // found in the LICENSE file. | ||
|
|
||
| #include "base/allocator/partition_allocator/spinning_futex_linux.h" | ||
|
|
||
| #include "base/allocator/partition_allocator/partition_alloc_check.h" | ||
| #include "build/build_config.h" | ||
|
|
||
| #if defined(OS_LINUX) || defined(OS_ANDROID) | ||
|
|
||
| #include <errno.h> | ||
| #include <linux/futex.h> | ||
| #include <sys/syscall.h> | ||
| #include <unistd.h> | ||
|
|
||
| namespace base { | ||
| namespace internal { | ||
|
|
||
| void SpinningFutex::FutexWait() { | ||
| // Save and restore errno. | ||
| int saved_errno = errno; | ||
| // Don't check the return value, as we will not be awaken by a timeout, since | ||
| // none is specified. | ||
| // | ||
| // Ignoring the return value doesn't impact correctness, as this acts as an | ||
| // immediate wakeup. For completeness, the possible errors for FUTEX_WAIT are: | ||
| // - EACCES: state_ is not readable. Should not happen. | ||
| // - EAGAIN: the value is not as expected, that is not |kLockedContended|, in | ||
| // which case retrying the loop is the right behavior. | ||
| // - EINTR: signal, looping is the right behavior. | ||
| // - EINVAL: invalid argument. | ||
| // | ||
| // Note: not checking the return value is the approach used in bionic and | ||
| // glibc as well. | ||
| // | ||
| // Will return immediately if |state_| is no longer equal to | ||
| // |kLockedContended|. Otherwise, sleeps and wakes up when |state_| may not be | ||
| // |kLockedContended| anymore. Note that even without spurious wakeups, the | ||
| // value of |state_| is not guaranteed when this returns, as another thread | ||
| // may get the lock before we get to run. | ||
| int err = syscall(SYS_futex, &state_, FUTEX_WAIT | FUTEX_PRIVATE_FLAG, | ||
| kLockedContended, nullptr, nullptr, 0); | ||
|
|
||
| if (err) { | ||
| // These are programming error, check them. | ||
| PA_DCHECK(errno != EACCES); | ||
| PA_DCHECK(errno != EINVAL); | ||
| } | ||
| errno = saved_errno; | ||
| } | ||
|
|
||
| void SpinningFutex::FutexWake() { | ||
| int saved_errno = errno; | ||
| long retval = syscall(SYS_futex, &state_, FUTEX_WAKE | FUTEX_PRIVATE_FLAG, | ||
| 1 /* wake up a single waiter */, nullptr, nullptr, 0); | ||
| PA_CHECK(retval != -1); | ||
| errno = saved_errno; | ||
| } | ||
|
|
||
| void SpinningFutex::LockSlow() { | ||
| // If this thread gets awaken but another one got the lock first, then go back | ||
| // to sleeping. See comments in |FutexWait()| to see why a loop is required. | ||
| while (state_.exchange(kLockedContended, std::memory_order_acquire) != | ||
| kUnlocked) { | ||
| FutexWait(); | ||
| } | ||
| } | ||
|
|
||
| } // namespace internal | ||
| } // namespace base | ||
|
|
||
| #endif // defined(OS_LINUX) || defined(OS_ANDROID) |
Oops, something went wrong.