Skip to content

Commit

Permalink
rgw/amqp: lock erase and create connection before emplace
Browse files Browse the repository at this point in the history 
When sanitizer is enabled, unittest_rgw_amqp shows,

```
=================================================================
==1429129==ERROR: LeakSanitizer: detected memory leaks

Direct leak of 416 byte(s) in 1 object(s) allocated from:
    #0 0xaaaab56a0008 in operator new(unsigned long) (/root/ceph/build/bin/unittest_rgw_amqp+0x1c0008) (BuildId: a20c317434e8d5f2ec33bbb71a69d81eb751c494)
    #1 0xaaaab57eecfc in amqp_new_connection /root/ceph/src/test/rgw/amqp_mock.cc:110:12
    #2 0xaaaab58095d8 in rgw::amqp::new_state(rgw::amqp::connection_t*, rgw::amqp::connection_id_t const&) /root/ceph/src/rgw/rgw_amqp.cc:373:16
    #3 0xaaaab5813c70 in rgw::amqp::Manager::run() /root/ceph/src/rgw/rgw_amqp.cc:684:18
    #4 0xaaaab5849e50 in void std::__invoke_impl<void, void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>(std::__invoke_memfun_deref, void (rgw::amqp::Manager::*&&)() noexcept, rgw::amqp::Manager*&&) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:74:14
    #5 0xaaaab5849b48 in std::__invoke_result<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>::type std::__invoke<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>(void (rgw::amqp::Manager::*&&)() noexcept, rgw::amqp::Manager*&&) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:96:14
    #6 0xaaaab5849978 in void std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> >::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:259:13
    #7 0xaaaab584979c in std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> >::operator()() /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:266:11
    ceph#8 0xaaaab5849420 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> > >::_M_run() /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:211:13
    ceph#9 0xffffb0fb31f8  (/lib/aarch64-linux-gnu/libstdc++.so.6+0xd31f8) (BuildId: a012b2bb77110e84b266cd7425b50e57427abb02)
    ceph#10 0xffffb0d7d5c4 in start_thread nptl/./nptl/pthread_create.c:442:8
    ceph#11 0xffffb0de5ed8  misc/../sysdeps/unix/sysv/linux/aarch64/clone.S:79

Direct leak of 64 byte(s) in 1 object(s) allocated from:
    #0 0xaaaab5669bb8 in posix_memalign (/root/ceph/build/bin/unittest_rgw_amqp+0x189bb8) (BuildId: a20c317434e8d5f2ec33bbb71a69d81eb751c494)
    #1 0xaaaab57f5294 in boost::alignment::aligned_alloc(unsigned long, unsigned long) /root/ceph/build/boost/include/boost/align/detail/aligned_alloc_posix.hpp:26:9
    #2 0xaaaab57f4d88 in boost::alignment::aligned_allocator<boost::lockfree::queue<amqp_basic_ack_t_>::node, 64ul>::allocate(unsigned long, void const*) /root/ceph/build/boost/include/boost/align/aligned_allocator.hpp:70:19
    #3 0xaaaab57f4204 in boost::lockfree::detail::freelist_stack<boost::lockfree::queue<amqp_basic_ack_t_>::node, boost::alignment::aligned_allocator<boost::lockfree::queue<amqp_basic_ack_t_>::node, 64ul> >::freelist_stack<boost::alignment::aligned_allocator<boost::lockfree::queue<amqp_basic_ack_t_>::node, 64ul> >(boost::alignment::aligned_allocator<boost::lockfree::queue<amqp_basic_ack_t_>::node, 64ul> const&, unsigned long) /root/ceph/build/boost/include/boost/lockfree/detail/freelist.hpp:62:31
    #4 0xaaaab57f3728 in boost::lockfree::queue<amqp_basic_ack_t_>::queue(unsigned long) /root/ceph/build/boost/include/boost/lockfree/queue.hpp:234:9
    #5 0xaaaab57f2ea8 in amqp_connection_state_t_::amqp_connection_state_t_() /root/ceph/src/test/rgw/amqp_mock.cc:90:5
    #6 0xaaaab57eed04 in amqp_new_connection /root/ceph/src/test/rgw/amqp_mock.cc:110:16
    #7 0xaaaab58095d8 in rgw::amqp::new_state(rgw::amqp::connection_t*, rgw::amqp::connection_id_t const&) /root/ceph/src/rgw/rgw_amqp.cc:373:16
    ceph#8 0xaaaab5813c70 in rgw::amqp::Manager::run() /root/ceph/src/rgw/rgw_amqp.cc:684:18
    ceph#9 0xaaaab5849e50 in void std::__invoke_impl<void, void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>(std::__invoke_memfun_deref, void (rgw::amqp::Manager::*&&)() noexcept, rgw::amqp::Manager*&&) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:74:14
    ceph#10 0xaaaab5849b48 in std::__invoke_result<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>::type std::__invoke<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>(void (rgw::amqp::Manager::*&&)() noexcept, rgw::amqp::Manager*&&) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:96:14
    ceph#11 0xaaaab5849978 in void std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> >::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:259:13
    ceph#12 0xaaaab584979c in std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> >::operator()() /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:266:11
    ceph#13 0xaaaab5849420 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> > >::_M_run() /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:211:13
    ceph#14 0xffffb0fb31f8  (/lib/aarch64-linux-gnu/libstdc++.so.6+0xd31f8) (BuildId: a012b2bb77110e84b266cd7425b50e57427abb02)
    ceph#15 0xffffb0d7d5c4 in start_thread nptl/./nptl/pthread_create.c:442:8
    ceph#16 0xffffb0de5ed8  misc/../sysdeps/unix/sysv/linux/aarch64/clone.S:79

Direct leak of 64 byte(s) in 1 object(s) allocated from:
    #0 0xaaaab5669bb8 in posix_memalign (/root/ceph/build/bin/unittest_rgw_amqp+0x189bb8) (BuildId: a20c317434e8d5f2ec33bbb71a69d81eb751c494)
    #1 0xaaaab57f5294 in boost::alignment::aligned_alloc(unsigned long, unsigned long) /root/ceph/build/boost/include/boost/align/detail/aligned_alloc_posix.hpp:26:9
    #2 0xaaaab57f90bc in boost::alignment::aligned_allocator<boost::lockfree::queue<amqp_basic_nack_t_>::node, 64ul>::allocate(unsigned long, void const*) /root/ceph/build/boost/include/boost/align/aligned_allocator.hpp:70:19
    #3 0xaaaab57f8538 in boost::lockfree::detail::freelist_stack<boost::lockfree::queue<amqp_basic_nack_t_>::node, boost::alignment::aligned_allocator<boost::lockfree::queue<amqp_basic_nack_t_>::node, 64ul> >::freelist_stack<boost::alignment::aligned_allocator<boost::lockfree::queue<amqp_basic_nack_t_>::node, 64ul> >(boost::alignment::aligned_allocator<boost::lockfree::queue<amqp_basic_nack_t_>::node, 64ul> const&, unsigned long) /root/ceph/build/boost/include/boost/lockfree/detail/freelist.hpp:62:31
    #4 0xaaaab57f3a6c in boost::lockfree::queue<amqp_basic_nack_t_>::queue(unsigned long) /root/ceph/build/boost/include/boost/lockfree/queue.hpp:234:9
    #5 0xaaaab57f2eb8 in amqp_connection_state_t_::amqp_connection_state_t_() /root/ceph/src/test/rgw/amqp_mock.cc:91:5
    #6 0xaaaab57eed04 in amqp_new_connection /root/ceph/src/test/rgw/amqp_mock.cc:110:16
    #7 0xaaaab58095d8 in rgw::amqp::new_state(rgw::amqp::connection_t*, rgw::amqp::connection_id_t const&) /root/ceph/src/rgw/rgw_amqp.cc:373:16
    ceph#8 0xaaaab5813c70 in rgw::amqp::Manager::run() /root/ceph/src/rgw/rgw_amqp.cc:684:18
    ceph#9 0xaaaab5849e50 in void std::__invoke_impl<void, void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>(std::__invoke_memfun_deref, void (rgw::amqp::Manager::*&&)() noexcept, rgw::amqp::Manager*&&) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:74:14
    ceph#10 0xaaaab5849b48 in std::__invoke_result<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>::type std::__invoke<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>(void (rgw::amqp::Manager::*&&)() noexcept, rgw::amqp::Manager*&&) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:96:14
    ceph#11 0xaaaab5849978 in void std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> >::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:259:13
    ceph#12 0xaaaab584979c in std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> >::operator()() /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:266:11
    ceph#13 0xaaaab5849420 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> > >::_M_run() /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:211:13
    ceph#14 0xffffb0fb31f8  (/lib/aarch64-linux-gnu/libstdc++.so.6+0xd31f8) (BuildId: a012b2bb77110e84b266cd7425b50e57427abb02)
    ceph#15 0xffffb0d7d5c4 in start_thread nptl/./nptl/pthread_create.c:442:8
    ceph#16 0xffffb0de5ed8  misc/../sysdeps/unix/sysv/linux/aarch64/clone.S:79

Direct leak of 9 byte(s) in 1 object(s) allocated from:
    #0 0xaaaab56690a0 in malloc (/root/ceph/build/bin/unittest_rgw_amqp+0x1890a0) (BuildId: a20c317434e8d5f2ec33bbb71a69d81eb751c494)
    #1 0xaaaab57f2754 in amqp_bytes_malloc_dup /root/ceph/src/test/rgw/amqp_mock.cc:384:18
    #2 0xaaaab580b4b4 in rgw::amqp::new_state(rgw::amqp::connection_t*, rgw::amqp::connection_id_t const&) /root/ceph/src/rgw/rgw_amqp.cc:509:28
    #3 0xaaaab5813c70 in rgw::amqp::Manager::run() /root/ceph/src/rgw/rgw_amqp.cc:684:18
    #4 0xaaaab5849e50 in void std::__invoke_impl<void, void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>(std::__invoke_memfun_deref, void (rgw::amqp::Manager::*&&)() noexcept, rgw::amqp::Manager*&&) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:74:14
    #5 0xaaaab5849b48 in std::__invoke_result<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>::type std::__invoke<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>(void (rgw::amqp::Manager::*&&)() noexcept, rgw::amqp::Manager*&&) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:96:14
    #6 0xaaaab5849978 in void std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> >::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:259:13
    #7 0xaaaab584979c in std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> >::operator()() /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:266:11
    ceph#8 0xaaaab5849420 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> > >::_M_run() /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:211:13
    ceph#9 0xffffb0fb31f8  (/lib/aarch64-linux-gnu/libstdc++.so.6+0xd31f8) (BuildId: a012b2bb77110e84b266cd7425b50e57427abb02)
    ceph#10 0xffffb0d7d5c4 in start_thread nptl/./nptl/pthread_create.c:442:8
    ceph#11 0xffffb0de5ed8  misc/../sysdeps/unix/sysv/linux/aarch64/clone.S:79

Indirect leak of 65536 byte(s) in 1024 object(s) allocated from:
    #0 0xaaaab5669bb8 in posix_memalign (/root/ceph/build/bin/unittest_rgw_amqp+0x189bb8) (BuildId: a20c317434e8d5f2ec33bbb71a69d81eb751c494)
    #1 0xaaaab57f5294 in boost::alignment::aligned_alloc(unsigned long, unsigned long) /root/ceph/build/boost/include/boost/align/detail/aligned_alloc_posix.hpp:26:9
    #2 0xaaaab57f4d88 in boost::alignment::aligned_allocator<boost::lockfree::queue<amqp_basic_ack_t_>::node, 64ul>::allocate(unsigned long, void const*) /root/ceph/build/boost/include/boost/align/aligned_allocator.hpp:70:19
    #3 0xaaaab57f4204 in boost::lockfree::detail::freelist_stack<boost::lockfree::queue<amqp_basic_ack_t_>::node, boost::alignment::aligned_allocator<boost::lockfree::queue<amqp_basic_ack_t_>::node, 64ul> >::freelist_stack<boost::alignment::aligned_allocator<boost::lockfree::queue<amqp_basic_ack_t_>::node, 64ul> >(boost::alignment::aligned_allocator<boost::lockfree::queue<amqp_basic_ack_t_>::node, 64ul> const&, unsigned long) /root/ceph/build/boost/include/boost/lockfree/detail/freelist.hpp:62:31
    #4 0xaaaab57f3728 in boost::lockfree::queue<amqp_basic_ack_t_>::queue(unsigned long) /root/ceph/build/boost/include/boost/lockfree/queue.hpp:234:9
    #5 0xaaaab57f2ea8 in amqp_connection_state_t_::amqp_connection_state_t_() /root/ceph/src/test/rgw/amqp_mock.cc:90:5
    #6 0xaaaab57eed04 in amqp_new_connection /root/ceph/src/test/rgw/amqp_mock.cc:110:16
    #7 0xaaaab58095d8 in rgw::amqp::new_state(rgw::amqp::connection_t*, rgw::amqp::connection_id_t const&) /root/ceph/src/rgw/rgw_amqp.cc:373:16
    ceph#8 0xaaaab5813c70 in rgw::amqp::Manager::run() /root/ceph/src/rgw/rgw_amqp.cc:684:18
    ceph#9 0xaaaab5849e50 in void std::__invoke_impl<void, void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>(std::__invoke_memfun_deref, void (rgw::amqp::Manager::*&&)() noexcept, rgw::amqp::Manager*&&) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:74:14
    ceph#10 0xaaaab5849b48 in std::__invoke_result<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>::type std::__invoke<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>(void (rgw::amqp::Manager::*&&)() noexcept, rgw::amqp::Manager*&&) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:96:14
    ceph#11 0xaaaab5849978 in void std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> >::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:259:13
    ceph#12 0xaaaab584979c in std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> >::operator()() /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:266:11
    ceph#13 0xaaaab5849420 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> > >::_M_run() /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:211:13
    ceph#14 0xffffb0fb31f8  (/lib/aarch64-linux-gnu/libstdc++.so.6+0xd31f8) (BuildId: a012b2bb77110e84b266cd7425b50e57427abb02)
    ceph#15 0xffffb0d7d5c4 in start_thread nptl/./nptl/pthread_create.c:442:8
    ceph#16 0xffffb0de5ed8  misc/../sysdeps/unix/sysv/linux/aarch64/clone.S:79

Indirect leak of 65536 byte(s) in 1024 object(s) allocated from:
    #0 0xaaaab5669bb8 in posix_memalign (/root/ceph/build/bin/unittest_rgw_amqp+0x189bb8) (BuildId: a20c317434e8d5f2ec33bbb71a69d81eb751c494)
    #1 0xaaaab57f5294 in boost::alignment::aligned_alloc(unsigned long, unsigned long) /root/ceph/build/boost/include/boost/align/detail/aligned_alloc_posix.hpp:26:9
    #2 0xaaaab57f90bc in boost::alignment::aligned_allocator<boost::lockfree::queue<amqp_basic_nack_t_>::node, 64ul>::allocate(unsigned long, void const*) /root/ceph/build/boost/include/boost/align/aligned_allocator.hpp:70:19
    #3 0xaaaab57f8538 in boost::lockfree::detail::freelist_stack<boost::lockfree::queue<amqp_basic_nack_t_>::node, boost::alignment::aligned_allocator<boost::lockfree::queue<amqp_basic_nack_t_>::node, 64ul> >::freelist_stack<boost::alignment::aligned_allocator<boost::lockfree::queue<amqp_basic_nack_t_>::node, 64ul> >(boost::alignment::aligned_allocator<boost::lockfree::queue<amqp_basic_nack_t_>::node, 64ul> const&, unsigned long) /root/ceph/build/boost/include/boost/lockfree/detail/freelist.hpp:62:31
    #4 0xaaaab57f3a6c in boost::lockfree::queue<amqp_basic_nack_t_>::queue(unsigned long) /root/ceph/build/boost/include/boost/lockfree/queue.hpp:234:9
    #5 0xaaaab57f2eb8 in amqp_connection_state_t_::amqp_connection_state_t_() /root/ceph/src/test/rgw/amqp_mock.cc:91:5
    #6 0xaaaab57eed04 in amqp_new_connection /root/ceph/src/test/rgw/amqp_mock.cc:110:16
    #7 0xaaaab58095d8 in rgw::amqp::new_state(rgw::amqp::connection_t*, rgw::amqp::connection_id_t const&) /root/ceph/src/rgw/rgw_amqp.cc:373:16
    ceph#8 0xaaaab5813c70 in rgw::amqp::Manager::run() /root/ceph/src/rgw/rgw_amqp.cc:684:18
    ceph#9 0xaaaab5849e50 in void std::__invoke_impl<void, void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>(std::__invoke_memfun_deref, void (rgw::amqp::Manager::*&&)() noexcept, rgw::amqp::Manager*&&) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:74:14
    ceph#10 0xaaaab5849b48 in std::__invoke_result<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>::type std::__invoke<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>(void (rgw::amqp::Manager::*&&)() noexcept, rgw::amqp::Manager*&&) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:96:14
    ceph#11 0xaaaab5849978 in void std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> >::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:259:13
    ceph#12 0xaaaab584979c in std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> >::operator()() /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:266:11
    ceph#13 0xaaaab5849420 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> > >::_M_run() /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:211:13
    ceph#14 0xffffb0fb31f8  (/lib/aarch64-linux-gnu/libstdc++.so.6+0xd31f8) (BuildId: a012b2bb77110e84b266cd7425b50e57427abb02)
    ceph#15 0xffffb0d7d5c4 in start_thread nptl/./nptl/pthread_create.c:442:8
    ceph#16 0xffffb0de5ed8  misc/../sysdeps/unix/sysv/linux/aarch64/clone.S:79

Indirect leak of 24 byte(s) in 1 object(s) allocated from:
    #0 0xaaaab56a0008 in operator new(unsigned long) (/root/ceph/build/bin/unittest_rgw_amqp+0x1c0008) (BuildId: a20c317434e8d5f2ec33bbb71a69d81eb751c494)
    #1 0xaaaab57eefb0 in amqp_tcp_socket_new /root/ceph/src/test/rgw/amqp_mock.cc:127:19
    #2 0xaaaab5809740 in rgw::amqp::new_state(rgw::amqp::connection_t*, rgw::amqp::connection_id_t const&) /root/ceph/src/rgw/rgw_amqp.cc:401:14
    #3 0xaaaab5813c70 in rgw::amqp::Manager::run() /root/ceph/src/rgw/rgw_amqp.cc:684:18
    #4 0xaaaab5849e50 in void std::__invoke_impl<void, void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>(std::__invoke_memfun_deref, void (rgw::amqp::Manager::*&&)() noexcept, rgw::amqp::Manager*&&) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:74:14
    #5 0xaaaab5849b48 in std::__invoke_result<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>::type std::__invoke<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>(void (rgw::amqp::Manager::*&&)() noexcept, rgw::amqp::Manager*&&) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:96:14
    #6 0xaaaab5849978 in void std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> >::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:259:13
    #7 0xaaaab584979c in std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> >::operator()() /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:266:11
    ceph#8 0xaaaab5849420 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> > >::_M_run() /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:211:13
    ceph#9 0xffffb0fb31f8  (/lib/aarch64-linux-gnu/libstdc++.so.6+0xd31f8) (BuildId: a012b2bb77110e84b266cd7425b50e57427abb02)
    ceph#10 0xffffb0d7d5c4 in start_thread nptl/./nptl/pthread_create.c:442:8
    ceph#11 0xffffb0de5ed8  misc/../sysdeps/unix/sysv/linux/aarch64/clone.S:79

Indirect leak of 24 byte(s) in 1 object(s) allocated from:
    #0 0xaaaab56a0008 in operator new(unsigned long) (/root/ceph/build/bin/unittest_rgw_amqp+0x1c0008) (BuildId: a20c317434e8d5f2ec33bbb71a69d81eb751c494)
    #1 0xaaaab57f102c in amqp_queue_declare /root/ceph/src/test/rgw/amqp_mock.cc:283:18
    #2 0xaaaab580ad14 in rgw::amqp::new_state(rgw::amqp::connection_t*, rgw::amqp::connection_id_t const&) /root/ceph/src/rgw/rgw_amqp.cc:480:27
    #3 0xaaaab5813c70 in rgw::amqp::Manager::run() /root/ceph/src/rgw/rgw_amqp.cc:684:18
    #4 0xaaaab5849e50 in void std::__invoke_impl<void, void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>(std::__invoke_memfun_deref, void (rgw::amqp::Manager::*&&)() noexcept, rgw::amqp::Manager*&&) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:74:14
    #5 0xaaaab5849b48 in std::__invoke_result<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>::type std::__invoke<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>(void (rgw::amqp::Manager::*&&)() noexcept, rgw::amqp::Manager*&&) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:96:14
    #6 0xaaaab5849978 in void std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> >::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:259:13
    #7 0xaaaab584979c in std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> >::operator()() /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:266:11
    ceph#8 0xaaaab5849420 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> > >::_M_run() /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:211:13
    ceph#9 0xffffb0fb31f8  (/lib/aarch64-linux-gnu/libstdc++.so.6+0xd31f8) (BuildId: a012b2bb77110e84b266cd7425b50e57427abb02)
    ceph#10 0xffffb0d7d5c4 in start_thread nptl/./nptl/pthread_create.c:442:8
    ceph#11 0xffffb0de5ed8  misc/../sysdeps/unix/sysv/linux/aarch64/clone.S:79

Indirect leak of 16 byte(s) in 1 object(s) allocated from:
    #0 0xaaaab56a0008 in operator new(unsigned long) (/root/ceph/build/bin/unittest_rgw_amqp+0x1c0008) (BuildId: a20c317434e8d5f2ec33bbb71a69d81eb751c494)
    #1 0xaaaab57f2280 in amqp_basic_consume /root/ceph/src/test/rgw/amqp_mock.cc:359:20
    #2 0xaaaab580b124 in rgw::amqp::new_state(rgw::amqp::connection_t*, rgw::amqp::connection_id_t const&) /root/ceph/src/rgw/rgw_amqp.cc:493:29
    #3 0xaaaab5813c70 in rgw::amqp::Manager::run() /root/ceph/src/rgw/rgw_amqp.cc:684:18
    #4 0xaaaab5849e50 in void std::__invoke_impl<void, void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>(std::__invoke_memfun_deref, void (rgw::amqp::Manager::*&&)() noexcept, rgw::amqp::Manager*&&) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:74:14
    #5 0xaaaab5849b48 in std::__invoke_result<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>::type std::__invoke<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>(void (rgw::amqp::Manager::*&&)() noexcept, rgw::amqp::Manager*&&) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:96:14
    #6 0xaaaab5849978 in void std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> >::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:259:13
    #7 0xaaaab584979c in std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> >::operator()() /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:266:11
    ceph#8 0xaaaab5849420 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> > >::_M_run() /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:211:13
    ceph#9 0xffffb0fb31f8  (/lib/aarch64-linux-gnu/libstdc++.so.6+0xd31f8) (BuildId: a012b2bb77110e84b266cd7425b50e57427abb02)
    ceph#10 0xffffb0d7d5c4 in start_thread nptl/./nptl/pthread_create.c:442:8
    ceph#11 0xffffb0de5ed8  misc/../sysdeps/unix/sysv/linux/aarch64/clone.S:79

Indirect leak of 16 byte(s) in 1 object(s) allocated from:
    #0 0xaaaab56a0008 in operator new(unsigned long) (/root/ceph/build/bin/unittest_rgw_amqp+0x1c0008) (BuildId: a20c317434e8d5f2ec33bbb71a69d81eb751c494)
    #1 0xaaaab57f0214 in amqp_channel_open /root/ceph/src/test/rgw/amqp_mock.cc:213:23
    #2 0xaaaab5809e78 in rgw::amqp::new_state(rgw::amqp::connection_t*, rgw::amqp::connection_id_t const&) /root/ceph/src/rgw/rgw_amqp.cc:448:21
    #3 0xaaaab5813c70 in rgw::amqp::Manager::run() /root/ceph/src/rgw/rgw_amqp.cc:684:18
    #4 0xaaaab5849e50 in void std::__invoke_impl<void, void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>(std::__invoke_memfun_deref, void (rgw::amqp::Manager::*&&)() noexcept, rgw::amqp::Manager*&&) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:74:14
    #5 0xaaaab5849b48 in std::__invoke_result<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>::type std::__invoke<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>(void (rgw::amqp::Manager::*&&)() noexcept, rgw::amqp::Manager*&&) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:96:14
    #6 0xaaaab5849978 in void std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> >::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:259:13
    #7 0xaaaab584979c in std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> >::operator()() /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:266:11
    ceph#8 0xaaaab5849420 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> > >::_M_run() /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:211:13
    ceph#9 0xffffb0fb31f8  (/lib/aarch64-linux-gnu/libstdc++.so.6+0xd31f8) (BuildId: a012b2bb77110e84b266cd7425b50e57427abb02)
    ceph#10 0xffffb0d7d5c4 in start_thread nptl/./nptl/pthread_create.c:442:8
    ceph#11 0xffffb0de5ed8  misc/../sysdeps/unix/sysv/linux/aarch64/clone.S:79

Indirect leak of 16 byte(s) in 1 object(s) allocated from:
    #0 0xaaaab56a0008 in operator new(unsigned long) (/root/ceph/build/bin/unittest_rgw_amqp+0x1c0008) (BuildId: a20c317434e8d5f2ec33bbb71a69d81eb751c494)
    #1 0xaaaab57f0294 in amqp_channel_open /root/ceph/src/test/rgw/amqp_mock.cc:217:21
    #2 0xaaaab580a188 in rgw::amqp::new_state(rgw::amqp::connection_t*, rgw::amqp::connection_id_t const&) /root/ceph/src/rgw/rgw_amqp.cc:453:21
    #3 0xaaaab5813c70 in rgw::amqp::Manager::run() /root/ceph/src/rgw/rgw_amqp.cc:684:18
    #4 0xaaaab5849e50 in void std::__invoke_impl<void, void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>(std::__invoke_memfun_deref, void (rgw::amqp::Manager::*&&)() noexcept, rgw::amqp::Manager*&&) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:74:14
    #5 0xaaaab5849b48 in std::__invoke_result<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>::type std::__invoke<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>(void (rgw::amqp::Manager::*&&)() noexcept, rgw::amqp::Manager*&&) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:96:14
    #6 0xaaaab5849978 in void std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> >::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:259:13
    #7 0xaaaab584979c in std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> >::operator()() /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:266:11
    ceph#8 0xaaaab5849420 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> > >::_M_run() /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:211:13
    ceph#9 0xffffb0fb31f8  (/lib/aarch64-linux-gnu/libstdc++.so.6+0xd31f8) (BuildId: a012b2bb77110e84b266cd7425b50e57427abb02)
    ceph#10 0xffffb0d7d5c4 in start_thread nptl/./nptl/pthread_create.c:442:8
    ceph#11 0xffffb0de5ed8  misc/../sysdeps/unix/sysv/linux/aarch64/clone.S:79

Indirect leak of 1 byte(s) in 1 object(s) allocated from:
    #0 0xaaaab56a0008 in operator new(unsigned long) (/root/ceph/build/bin/unittest_rgw_amqp+0x1c0008) (BuildId: a20c317434e8d5f2ec33bbb71a69d81eb751c494)
    #1 0xaaaab57f1454 in amqp_confirm_select /root/ceph/src/test/rgw/amqp_mock.cc:291:20
    #2 0xaaaab580a49c in rgw::amqp::new_state(rgw::amqp::connection_t*, rgw::amqp::connection_id_t const&) /root/ceph/src/rgw/rgw_amqp.cc:458:21
    #3 0xaaaab5813c70 in rgw::amqp::Manager::run() /root/ceph/src/rgw/rgw_amqp.cc:684:18
    #4 0xaaaab5849e50 in void std::__invoke_impl<void, void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>(std::__invoke_memfun_deref, void (rgw::amqp::Manager::*&&)() noexcept, rgw::amqp::Manager*&&) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:74:14
    #5 0xaaaab5849b48 in std::__invoke_result<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>::type std::__invoke<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>(void (rgw::amqp::Manager::*&&)() noexcept, rgw::amqp::Manager*&&) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:96:14
    #6 0xaaaab5849978 in void std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> >::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:259:13
    #7 0xaaaab584979c in std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> >::operator()() /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:266:11
    ceph#8 0xaaaab5849420 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> > >::_M_run() /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:211:13
    ceph#9 0xffffb0fb31f8  (/lib/aarch64-linux-gnu/libstdc++.so.6+0xd31f8) (BuildId: a012b2bb77110e84b266cd7425b50e57427abb02)
    ceph#10 0xffffb0d7d5c4 in start_thread nptl/./nptl/pthread_create.c:442:8
    ceph#11 0xffffb0de5ed8  misc/../sysdeps/unix/sysv/linux/aarch64/clone.S:79

Indirect leak of 1 byte(s) in 1 object(s) allocated from:
    #0 0xaaaab56a0008 in operator new(unsigned long) (/root/ceph/build/bin/unittest_rgw_amqp+0x1c0008) (BuildId: a20c317434e8d5f2ec33bbb71a69d81eb751c494)
    #1 0xaaaab57f0548 in amqp_exchange_declare /root/ceph/src/test/rgw/amqp_mock.cc:231:21
    #2 0xaaaab580a93c in rgw::amqp::new_state(rgw::amqp::connection_t*, rgw::amqp::connection_id_t const&) /root/ceph/src/rgw/rgw_amqp.cc:466:21
    #3 0xaaaab5813c70 in rgw::amqp::Manager::run() /root/ceph/src/rgw/rgw_amqp.cc:684:18
    #4 0xaaaab5849e50 in void std::__invoke_impl<void, void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>(std::__invoke_memfun_deref, void (rgw::amqp::Manager::*&&)() noexcept, rgw::amqp::Manager*&&) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:74:14
    #5 0xaaaab5849b48 in std::__invoke_result<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>::type std::__invoke<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*>(void (rgw::amqp::Manager::*&&)() noexcept, rgw::amqp::Manager*&&) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:96:14
    #6 0xaaaab5849978 in void std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> >::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>) /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:259:13
    #7 0xaaaab584979c in std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> >::operator()() /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:266:11
    ceph#8 0xaaaab5849420 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (rgw::amqp::Manager::*)() noexcept, rgw::amqp::Manager*> > >::_M_run() /usr/bin/../lib/gcc/aarch64-linux-gnu/11/../../../../include/c++/11/bits/std_thread.h:211:13
    ceph#9 0xffffb0fb31f8  (/lib/aarch64-linux-gnu/libstdc++.so.6+0xd31f8) (BuildId: a012b2bb77110e84b266cd7425b50e57427abb02)
    ceph#10 0xffffb0d7d5c4 in start_thread nptl/./nptl/pthread_create.c:442:8
    ceph#11 0xffffb0de5ed8  misc/../sysdeps/unix/sysv/linux/aarch64/clone.S:79

SUMMARY: AddressSanitizer: 131723 byte(s) leaked in 2059 allocation(s).
```

So to prevent multiple threads from operating the same element at the same time, add the lock only to erase and ensure fully create connection before emplace.

Fixes: https://tracker.ceph.com/issues/66266

Signed-off-by: Rongqi Sun <sunrongqi@huawei.com>
(cherry picked from commit 958ecba)
  • Loading branch information
@Svelar
Svelar committed Aug 5, 2024
1 parent c58591a commit ea9e779
Showing 1 changed file with 13 additions and 8 deletions.
21 changes: 13 additions & 8 deletions src/rgw/rgw_amqp.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -535,9 +535,12 @@ typedef boost::lockfree::queue<message_wrapper_t*, boost::lockfree::fixed_sized<
continue;

#define ERASE_AND_CONTINUE(IT,CONTAINER) \
IT=CONTAINER.erase(IT); \
--connection_count; \
continue;
{ \
std::lock_guard lock(connections_lock); \
IT=CONTAINER.erase(IT); \
--connection_count; \
continue; \
}

class Manager {
public:
Expand Down Expand Up @@ -883,12 +886,14 @@ class Manager {
}
// if error occurred during creation the creation will be retried in the main thread
++connection_count;
auto conn = connections.emplace(tmp_id, std::make_unique<connection_t>(cct, info, verify_ssl, ca_location)).first->second.get();
ldout(cct, 10) << "AMQP connect: new connection is created. Total connections: " << connection_count << dendl;
if (!new_state(conn, tmp_id)) {
ldout(cct, 1) << "AMQP connect: new connection '" << to_string(tmp_id) << "' is created. but state creation failed (will retry). error: " <<
status_to_string(conn->status) << " (" << conn->reply_code << ")" << dendl;
auto conn = std::make_unique<connection_t>(cct, info, verify_ssl, ca_location);
if (new_state(conn.get(), tmp_id)) {
ldout(cct, 10) << "AMQP connect: new connection is created. Total connections: " << connection_count << dendl;
} else {
ldout(cct, 1) << "AMQP connect: new connection '" << to_string(tmp_id) << "' is created. but state creation failed (will retry). error: " <<
status_to_string(conn->status) << " (" << conn->reply_code << ")" << dendl;
}
connections.emplace(tmp_id, std::move(conn));
id = std::move(tmp_id);
return true;
}
Expand Down

0 comments on commit ea9e779

Please sign in to comment.