malloc_hook.cc 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705
  1. // -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*-
  2. // Copyright (c) 2005, Google Inc.
  3. // All rights reserved.
  4. //
  5. // Redistribution and use in source and binary forms, with or without
  6. // modification, are permitted provided that the following conditions are
  7. // met:
  8. //
  9. // * Redistributions of source code must retain the above copyright
  10. // notice, this list of conditions and the following disclaimer.
  11. // * Redistributions in binary form must reproduce the above
  12. // copyright notice, this list of conditions and the following disclaimer
  13. // in the documentation and/or other materials provided with the
  14. // distribution.
  15. // * Neither the name of Google Inc. nor the names of its
  16. // contributors may be used to endorse or promote products derived from
  17. // this software without specific prior written permission.
  18. //
  19. // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20. // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21. // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22. // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23. // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24. // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25. // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26. // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27. // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28. // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29. // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30. // ---
  31. // Author: Sanjay Ghemawat <opensource@google.com>
  32. #include <config.h>
  33. // Disable the glibc prototype of mremap(), as older versions of the
  34. // system headers define this function with only four arguments,
  35. // whereas newer versions allow an optional fifth argument:
  36. #ifdef HAVE_MMAP
  37. # define mremap glibc_mremap
  38. # include <sys/mman.h>
  39. # undef mremap
  40. #endif
  41. #include <stddef.h>
  42. #ifdef HAVE_STDINT_H
  43. #include <stdint.h>
  44. #endif
  45. #include <algorithm>
  46. #include "base/logging.h"
  47. #include "base/spinlock.h"
  48. #include "maybe_emergency_malloc.h"
  49. #include "maybe_threads.h"
  50. #include "malloc_hook-inl.h"
  51. #include <gperftools/malloc_hook.h>
  52. // This #ifdef should almost never be set. Set NO_TCMALLOC_SAMPLES if
  53. // you're porting to a system where you really can't get a stacktrace.
  54. #ifdef NO_TCMALLOC_SAMPLES
  55. // We use #define so code compiles even if you #include stacktrace.h somehow.
  56. # define GetStackTrace(stack, depth, skip) (0)
  57. #else
  58. # include <gperftools/stacktrace.h>
  59. #endif
  60. // __THROW is defined in glibc systems. It means, counter-intuitively,
  61. // "This function will never throw an exception." It's an optional
  62. // optimization tool, but we may need to use it to match glibc prototypes.
  63. #ifndef __THROW // I guess we're not on a glibc system
  64. # define __THROW // __THROW is just an optimization, so ok to make it ""
  65. #endif
  66. using std::copy;
  67. // Declaration of default weak initialization function, that can be overridden
  68. // by linking-in a strong definition (as heap-checker.cc does). This is
  69. // extern "C" so that it doesn't trigger gold's --detect-odr-violations warning,
  70. // which only looks at C++ symbols.
  71. //
  72. // This function is declared here as weak, and defined later, rather than a more
  73. // straightforward simple weak definition, as a workround for an icc compiler
  74. // issue ((Intel reference 290819). This issue causes icc to resolve weak
  75. // symbols too early, at compile rather than link time. By declaring it (weak)
  76. // here, then defining it below after its use, we can avoid the problem.
  77. extern "C" {
  78. ATTRIBUTE_WEAK void MallocHook_InitAtFirstAllocation_HeapLeakChecker();
  79. }
  80. namespace {
  81. void RemoveInitialHooksAndCallInitializers(); // below.
  82. pthread_once_t once = PTHREAD_ONCE_INIT;
  83. // These hooks are installed in MallocHook as the only initial hooks. The first
  84. // hook that is called will run RemoveInitialHooksAndCallInitializers (see the
  85. // definition below) and then redispatch to any malloc hooks installed by
  86. // RemoveInitialHooksAndCallInitializers.
  87. //
  88. // Note(llib): there is a possibility of a race in the event that there are
  89. // multiple threads running before the first allocation. This is pretty
  90. // difficult to achieve, but if it is then multiple threads may concurrently do
  91. // allocations. The first caller will call
  92. // RemoveInitialHooksAndCallInitializers via one of the initial hooks. A
  93. // concurrent allocation may, depending on timing either:
  94. // * still have its initial malloc hook installed, run that and block on waiting
  95. // for the first caller to finish its call to
  96. // RemoveInitialHooksAndCallInitializers, and proceed normally.
  97. // * occur some time during the RemoveInitialHooksAndCallInitializers call, at
  98. // which point there could be no initial hooks and the subsequent hooks that
  99. // are about to be set up by RemoveInitialHooksAndCallInitializers haven't
  100. // been installed yet. I think the worst we can get is that some allocations
  101. // will not get reported to some hooks set by the initializers called from
  102. // RemoveInitialHooksAndCallInitializers.
  103. void InitialNewHook(const void* ptr, size_t size) {
  104. perftools_pthread_once(&once, &RemoveInitialHooksAndCallInitializers);
  105. MallocHook::InvokeNewHook(ptr, size);
  106. }
  107. void InitialPreMMapHook(const void* start,
  108. size_t size,
  109. int protection,
  110. int flags,
  111. int fd,
  112. off_t offset) {
  113. perftools_pthread_once(&once, &RemoveInitialHooksAndCallInitializers);
  114. MallocHook::InvokePreMmapHook(start, size, protection, flags, fd, offset);
  115. }
  116. void InitialPreSbrkHook(ptrdiff_t increment) {
  117. perftools_pthread_once(&once, &RemoveInitialHooksAndCallInitializers);
  118. MallocHook::InvokePreSbrkHook(increment);
  119. }
  120. // This function is called at most once by one of the above initial malloc
  121. // hooks. It removes all initial hooks and initializes all other clients that
  122. // want to get control at the very first memory allocation. The initializers
  123. // may assume that the initial malloc hooks have been removed. The initializers
  124. // may set up malloc hooks and allocate memory.
  125. void RemoveInitialHooksAndCallInitializers() {
  126. RAW_CHECK(MallocHook::RemoveNewHook(&InitialNewHook), "");
  127. RAW_CHECK(MallocHook::RemovePreMmapHook(&InitialPreMMapHook), "");
  128. RAW_CHECK(MallocHook::RemovePreSbrkHook(&InitialPreSbrkHook), "");
  129. // HeapLeakChecker is currently the only module that needs to get control on
  130. // the first memory allocation, but one can add other modules by following the
  131. // same weak/strong function pattern.
  132. MallocHook_InitAtFirstAllocation_HeapLeakChecker();
  133. }
  134. } // namespace
  135. // Weak default initialization function that must go after its use.
  136. extern "C" void MallocHook_InitAtFirstAllocation_HeapLeakChecker() {
  137. // Do nothing.
  138. }
  139. namespace base { namespace internal {
  140. // This lock is shared between all implementations of HookList::Add & Remove.
  141. // The potential for contention is very small. This needs to be a SpinLock and
  142. // not a Mutex since it's possible for Mutex locking to allocate memory (e.g.,
  143. // per-thread allocation in debug builds), which could cause infinite recursion.
  144. static SpinLock hooklist_spinlock(base::LINKER_INITIALIZED);
  145. template <typename T>
  146. bool HookList<T>::Add(T value_as_t) {
  147. AtomicWord value = bit_cast<AtomicWord>(value_as_t);
  148. if (value == 0) {
  149. return false;
  150. }
  151. SpinLockHolder l(&hooklist_spinlock);
  152. // Find the first slot in data that is 0.
  153. int index = 0;
  154. while ((index < kHookListMaxValues) &&
  155. (base::subtle::NoBarrier_Load(&priv_data[index]) != 0)) {
  156. ++index;
  157. }
  158. if (index == kHookListMaxValues) {
  159. return false;
  160. }
  161. AtomicWord prev_num_hooks = base::subtle::Acquire_Load(&priv_end);
  162. base::subtle::NoBarrier_Store(&priv_data[index], value);
  163. if (prev_num_hooks <= index) {
  164. base::subtle::NoBarrier_Store(&priv_end, index + 1);
  165. }
  166. return true;
  167. }
  168. template <typename T>
  169. void HookList<T>::FixupPrivEndLocked() {
  170. AtomicWord hooks_end = base::subtle::NoBarrier_Load(&priv_end);
  171. while ((hooks_end > 0) &&
  172. (base::subtle::NoBarrier_Load(&priv_data[hooks_end - 1]) == 0)) {
  173. --hooks_end;
  174. }
  175. base::subtle::NoBarrier_Store(&priv_end, hooks_end);
  176. }
  177. template <typename T>
  178. bool HookList<T>::Remove(T value_as_t) {
  179. if (value_as_t == 0) {
  180. return false;
  181. }
  182. SpinLockHolder l(&hooklist_spinlock);
  183. AtomicWord hooks_end = base::subtle::NoBarrier_Load(&priv_end);
  184. int index = 0;
  185. while (index < hooks_end && value_as_t != bit_cast<T>(
  186. base::subtle::NoBarrier_Load(&priv_data[index]))) {
  187. ++index;
  188. }
  189. if (index == hooks_end) {
  190. return false;
  191. }
  192. base::subtle::NoBarrier_Store(&priv_data[index], 0);
  193. FixupPrivEndLocked();
  194. return true;
  195. }
  196. template <typename T>
  197. int HookList<T>::Traverse(T* output_array, int n) const {
  198. AtomicWord hooks_end = base::subtle::Acquire_Load(&priv_end);
  199. int actual_hooks_end = 0;
  200. for (int i = 0; i < hooks_end && n > 0; ++i) {
  201. AtomicWord data = base::subtle::Acquire_Load(&priv_data[i]);
  202. if (data != 0) {
  203. *output_array++ = bit_cast<T>(data);
  204. ++actual_hooks_end;
  205. --n;
  206. }
  207. }
  208. return actual_hooks_end;
  209. }
  210. template <typename T>
  211. T HookList<T>::ExchangeSingular(T value_as_t) {
  212. AtomicWord value = bit_cast<AtomicWord>(value_as_t);
  213. AtomicWord old_value;
  214. SpinLockHolder l(&hooklist_spinlock);
  215. old_value = base::subtle::NoBarrier_Load(&priv_data[kHookListSingularIdx]);
  216. base::subtle::NoBarrier_Store(&priv_data[kHookListSingularIdx], value);
  217. if (value != 0) {
  218. base::subtle::NoBarrier_Store(&priv_end, kHookListSingularIdx + 1);
  219. } else {
  220. FixupPrivEndLocked();
  221. }
  222. return bit_cast<T>(old_value);
  223. }
  224. // Initialize a HookList (optionally with the given initial_value in index 0).
  225. #define INIT_HOOK_LIST { 0 }
  226. #define INIT_HOOK_LIST_WITH_VALUE(initial_value) \
  227. { 1, { reinterpret_cast<AtomicWord>(initial_value) } }
  228. // Explicit instantiation for malloc_hook_test.cc. This ensures all the methods
  229. // are instantiated.
  230. template struct HookList<MallocHook::NewHook>;
  231. HookList<MallocHook::NewHook> new_hooks_ =
  232. INIT_HOOK_LIST_WITH_VALUE(&InitialNewHook);
  233. HookList<MallocHook::DeleteHook> delete_hooks_ = INIT_HOOK_LIST;
  234. HookList<MallocHook::PreMmapHook> premmap_hooks_ =
  235. INIT_HOOK_LIST_WITH_VALUE(&InitialPreMMapHook);
  236. HookList<MallocHook::MmapHook> mmap_hooks_ = INIT_HOOK_LIST;
  237. HookList<MallocHook::MunmapHook> munmap_hooks_ = INIT_HOOK_LIST;
  238. HookList<MallocHook::MremapHook> mremap_hooks_ = INIT_HOOK_LIST;
  239. HookList<MallocHook::PreSbrkHook> presbrk_hooks_ =
  240. INIT_HOOK_LIST_WITH_VALUE(InitialPreSbrkHook);
  241. HookList<MallocHook::SbrkHook> sbrk_hooks_ = INIT_HOOK_LIST;
  242. // These lists contain either 0 or 1 hooks.
  243. HookList<MallocHook::MmapReplacement> mmap_replacement_ = { 0 };
  244. HookList<MallocHook::MunmapReplacement> munmap_replacement_ = { 0 };
  245. #undef INIT_HOOK_LIST_WITH_VALUE
  246. #undef INIT_HOOK_LIST
  247. } } // namespace base::internal
  248. using base::internal::kHookListMaxValues;
  249. using base::internal::new_hooks_;
  250. using base::internal::delete_hooks_;
  251. using base::internal::premmap_hooks_;
  252. using base::internal::mmap_hooks_;
  253. using base::internal::mmap_replacement_;
  254. using base::internal::munmap_hooks_;
  255. using base::internal::munmap_replacement_;
  256. using base::internal::mremap_hooks_;
  257. using base::internal::presbrk_hooks_;
  258. using base::internal::sbrk_hooks_;
  259. // These are available as C bindings as well as C++, hence their
  260. // definition outside the MallocHook class.
  261. extern "C"
  262. int MallocHook_AddNewHook(MallocHook_NewHook hook) {
  263. RAW_VLOG(10, "AddNewHook(%p)", hook);
  264. return new_hooks_.Add(hook);
  265. }
  266. extern "C"
  267. int MallocHook_RemoveNewHook(MallocHook_NewHook hook) {
  268. RAW_VLOG(10, "RemoveNewHook(%p)", hook);
  269. return new_hooks_.Remove(hook);
  270. }
  271. extern "C"
  272. int MallocHook_AddDeleteHook(MallocHook_DeleteHook hook) {
  273. RAW_VLOG(10, "AddDeleteHook(%p)", hook);
  274. return delete_hooks_.Add(hook);
  275. }
  276. extern "C"
  277. int MallocHook_RemoveDeleteHook(MallocHook_DeleteHook hook) {
  278. RAW_VLOG(10, "RemoveDeleteHook(%p)", hook);
  279. return delete_hooks_.Remove(hook);
  280. }
  281. extern "C"
  282. int MallocHook_AddPreMmapHook(MallocHook_PreMmapHook hook) {
  283. RAW_VLOG(10, "AddPreMmapHook(%p)", hook);
  284. return premmap_hooks_.Add(hook);
  285. }
  286. extern "C"
  287. int MallocHook_RemovePreMmapHook(MallocHook_PreMmapHook hook) {
  288. RAW_VLOG(10, "RemovePreMmapHook(%p)", hook);
  289. return premmap_hooks_.Remove(hook);
  290. }
  291. extern "C"
  292. int MallocHook_SetMmapReplacement(MallocHook_MmapReplacement hook) {
  293. RAW_VLOG(10, "SetMmapReplacement(%p)", hook);
  294. // NOTE this is a best effort CHECK. Concurrent sets could succeed since
  295. // this test is outside of the Add spin lock.
  296. RAW_CHECK(mmap_replacement_.empty(), "Only one MMapReplacement is allowed.");
  297. return mmap_replacement_.Add(hook);
  298. }
  299. extern "C"
  300. int MallocHook_RemoveMmapReplacement(MallocHook_MmapReplacement hook) {
  301. RAW_VLOG(10, "RemoveMmapReplacement(%p)", hook);
  302. return mmap_replacement_.Remove(hook);
  303. }
  304. extern "C"
  305. int MallocHook_AddMmapHook(MallocHook_MmapHook hook) {
  306. RAW_VLOG(10, "AddMmapHook(%p)", hook);
  307. return mmap_hooks_.Add(hook);
  308. }
  309. extern "C"
  310. int MallocHook_RemoveMmapHook(MallocHook_MmapHook hook) {
  311. RAW_VLOG(10, "RemoveMmapHook(%p)", hook);
  312. return mmap_hooks_.Remove(hook);
  313. }
  314. extern "C"
  315. int MallocHook_AddMunmapHook(MallocHook_MunmapHook hook) {
  316. RAW_VLOG(10, "AddMunmapHook(%p)", hook);
  317. return munmap_hooks_.Add(hook);
  318. }
  319. extern "C"
  320. int MallocHook_RemoveMunmapHook(MallocHook_MunmapHook hook) {
  321. RAW_VLOG(10, "RemoveMunmapHook(%p)", hook);
  322. return munmap_hooks_.Remove(hook);
  323. }
  324. extern "C"
  325. int MallocHook_SetMunmapReplacement(MallocHook_MunmapReplacement hook) {
  326. RAW_VLOG(10, "SetMunmapReplacement(%p)", hook);
  327. // NOTE this is a best effort CHECK. Concurrent sets could succeed since
  328. // this test is outside of the Add spin lock.
  329. RAW_CHECK(munmap_replacement_.empty(),
  330. "Only one MunmapReplacement is allowed.");
  331. return munmap_replacement_.Add(hook);
  332. }
  333. extern "C"
  334. int MallocHook_RemoveMunmapReplacement(MallocHook_MunmapReplacement hook) {
  335. RAW_VLOG(10, "RemoveMunmapReplacement(%p)", hook);
  336. return munmap_replacement_.Remove(hook);
  337. }
  338. extern "C"
  339. int MallocHook_AddMremapHook(MallocHook_MremapHook hook) {
  340. RAW_VLOG(10, "AddMremapHook(%p)", hook);
  341. return mremap_hooks_.Add(hook);
  342. }
  343. extern "C"
  344. int MallocHook_RemoveMremapHook(MallocHook_MremapHook hook) {
  345. RAW_VLOG(10, "RemoveMremapHook(%p)", hook);
  346. return mremap_hooks_.Remove(hook);
  347. }
  348. extern "C"
  349. int MallocHook_AddPreSbrkHook(MallocHook_PreSbrkHook hook) {
  350. RAW_VLOG(10, "AddPreSbrkHook(%p)", hook);
  351. return presbrk_hooks_.Add(hook);
  352. }
  353. extern "C"
  354. int MallocHook_RemovePreSbrkHook(MallocHook_PreSbrkHook hook) {
  355. RAW_VLOG(10, "RemovePreSbrkHook(%p)", hook);
  356. return presbrk_hooks_.Remove(hook);
  357. }
  358. extern "C"
  359. int MallocHook_AddSbrkHook(MallocHook_SbrkHook hook) {
  360. RAW_VLOG(10, "AddSbrkHook(%p)", hook);
  361. return sbrk_hooks_.Add(hook);
  362. }
  363. extern "C"
  364. int MallocHook_RemoveSbrkHook(MallocHook_SbrkHook hook) {
  365. RAW_VLOG(10, "RemoveSbrkHook(%p)", hook);
  366. return sbrk_hooks_.Remove(hook);
  367. }
  368. // The code below is DEPRECATED.
  369. extern "C"
  370. MallocHook_NewHook MallocHook_SetNewHook(MallocHook_NewHook hook) {
  371. RAW_VLOG(10, "SetNewHook(%p)", hook);
  372. return new_hooks_.ExchangeSingular(hook);
  373. }
  374. extern "C"
  375. MallocHook_DeleteHook MallocHook_SetDeleteHook(MallocHook_DeleteHook hook) {
  376. RAW_VLOG(10, "SetDeleteHook(%p)", hook);
  377. return delete_hooks_.ExchangeSingular(hook);
  378. }
  379. extern "C"
  380. MallocHook_PreMmapHook MallocHook_SetPreMmapHook(MallocHook_PreMmapHook hook) {
  381. RAW_VLOG(10, "SetPreMmapHook(%p)", hook);
  382. return premmap_hooks_.ExchangeSingular(hook);
  383. }
  384. extern "C"
  385. MallocHook_MmapHook MallocHook_SetMmapHook(MallocHook_MmapHook hook) {
  386. RAW_VLOG(10, "SetMmapHook(%p)", hook);
  387. return mmap_hooks_.ExchangeSingular(hook);
  388. }
  389. extern "C"
  390. MallocHook_MunmapHook MallocHook_SetMunmapHook(MallocHook_MunmapHook hook) {
  391. RAW_VLOG(10, "SetMunmapHook(%p)", hook);
  392. return munmap_hooks_.ExchangeSingular(hook);
  393. }
  394. extern "C"
  395. MallocHook_MremapHook MallocHook_SetMremapHook(MallocHook_MremapHook hook) {
  396. RAW_VLOG(10, "SetMremapHook(%p)", hook);
  397. return mremap_hooks_.ExchangeSingular(hook);
  398. }
  399. extern "C"
  400. MallocHook_PreSbrkHook MallocHook_SetPreSbrkHook(MallocHook_PreSbrkHook hook) {
  401. RAW_VLOG(10, "SetPreSbrkHook(%p)", hook);
  402. return presbrk_hooks_.ExchangeSingular(hook);
  403. }
  404. extern "C"
  405. MallocHook_SbrkHook MallocHook_SetSbrkHook(MallocHook_SbrkHook hook) {
  406. RAW_VLOG(10, "SetSbrkHook(%p)", hook);
  407. return sbrk_hooks_.ExchangeSingular(hook);
  408. }
  409. // End of DEPRECATED code section.
  410. // Note: embedding the function calls inside the traversal of HookList would be
  411. // very confusing, as it is legal for a hook to remove itself and add other
  412. // hooks. Doing traversal first, and then calling the hooks ensures we only
  413. // call the hooks registered at the start.
  414. #define INVOKE_HOOKS(HookType, hook_list, args) do { \
  415. HookType hooks[kHookListMaxValues]; \
  416. int num_hooks = hook_list.Traverse(hooks, kHookListMaxValues); \
  417. for (int i = 0; i < num_hooks; ++i) { \
  418. (*hooks[i])args; \
  419. } \
  420. } while (0)
  421. // There should only be one replacement. Return the result of the first
  422. // one, or false if there is none.
  423. #define INVOKE_REPLACEMENT(HookType, hook_list, args) do { \
  424. HookType hooks[kHookListMaxValues]; \
  425. int num_hooks = hook_list.Traverse(hooks, kHookListMaxValues); \
  426. return (num_hooks > 0 && (*hooks[0])args); \
  427. } while (0)
  428. void MallocHook::InvokeNewHookSlow(const void* p, size_t s) {
  429. if (tcmalloc::IsEmergencyPtr(p)) {
  430. return;
  431. }
  432. INVOKE_HOOKS(NewHook, new_hooks_, (p, s));
  433. }
  434. void MallocHook::InvokeDeleteHookSlow(const void* p) {
  435. if (tcmalloc::IsEmergencyPtr(p)) {
  436. return;
  437. }
  438. INVOKE_HOOKS(DeleteHook, delete_hooks_, (p));
  439. }
  440. void MallocHook::InvokePreMmapHookSlow(const void* start,
  441. size_t size,
  442. int protection,
  443. int flags,
  444. int fd,
  445. off_t offset) {
  446. INVOKE_HOOKS(PreMmapHook, premmap_hooks_, (start, size, protection, flags, fd,
  447. offset));
  448. }
  449. void MallocHook::InvokeMmapHookSlow(const void* result,
  450. const void* start,
  451. size_t size,
  452. int protection,
  453. int flags,
  454. int fd,
  455. off_t offset) {
  456. INVOKE_HOOKS(MmapHook, mmap_hooks_, (result, start, size, protection, flags,
  457. fd, offset));
  458. }
  459. bool MallocHook::InvokeMmapReplacementSlow(const void* start,
  460. size_t size,
  461. int protection,
  462. int flags,
  463. int fd,
  464. off_t offset,
  465. void** result) {
  466. INVOKE_REPLACEMENT(MmapReplacement, mmap_replacement_,
  467. (start, size, protection, flags, fd, offset, result));
  468. }
  469. void MallocHook::InvokeMunmapHookSlow(const void* p, size_t s) {
  470. INVOKE_HOOKS(MunmapHook, munmap_hooks_, (p, s));
  471. }
  472. bool MallocHook::InvokeMunmapReplacementSlow(const void* p,
  473. size_t s,
  474. int* result) {
  475. INVOKE_REPLACEMENT(MunmapReplacement, munmap_replacement_, (p, s, result));
  476. }
  477. void MallocHook::InvokeMremapHookSlow(const void* result,
  478. const void* old_addr,
  479. size_t old_size,
  480. size_t new_size,
  481. int flags,
  482. const void* new_addr) {
  483. INVOKE_HOOKS(MremapHook, mremap_hooks_, (result, old_addr, old_size, new_size,
  484. flags, new_addr));
  485. }
  486. void MallocHook::InvokePreSbrkHookSlow(ptrdiff_t increment) {
  487. INVOKE_HOOKS(PreSbrkHook, presbrk_hooks_, (increment));
  488. }
  489. void MallocHook::InvokeSbrkHookSlow(const void* result, ptrdiff_t increment) {
  490. INVOKE_HOOKS(SbrkHook, sbrk_hooks_, (result, increment));
  491. }
  492. #undef INVOKE_HOOKS
  493. #ifndef NO_TCMALLOC_SAMPLES
  494. DEFINE_ATTRIBUTE_SECTION_VARS(google_malloc);
  495. DECLARE_ATTRIBUTE_SECTION_VARS(google_malloc);
  496. // actual functions are in debugallocation.cc or tcmalloc.cc
  497. DEFINE_ATTRIBUTE_SECTION_VARS(malloc_hook);
  498. DECLARE_ATTRIBUTE_SECTION_VARS(malloc_hook);
  499. // actual functions are in this file, malloc_hook.cc, and low_level_alloc.cc
  500. #define ADDR_IN_ATTRIBUTE_SECTION(addr, name) \
  501. (reinterpret_cast<uintptr_t>(ATTRIBUTE_SECTION_START(name)) <= \
  502. reinterpret_cast<uintptr_t>(addr) && \
  503. reinterpret_cast<uintptr_t>(addr) < \
  504. reinterpret_cast<uintptr_t>(ATTRIBUTE_SECTION_STOP(name)))
  505. // Return true iff 'caller' is a return address within a function
  506. // that calls one of our hooks via MallocHook:Invoke*.
  507. // A helper for GetCallerStackTrace.
  508. static inline bool InHookCaller(const void* caller) {
  509. return ADDR_IN_ATTRIBUTE_SECTION(caller, google_malloc) ||
  510. ADDR_IN_ATTRIBUTE_SECTION(caller, malloc_hook);
  511. // We can use one section for everything except tcmalloc_or_debug
  512. // due to its special linkage mode, which prevents merging of the sections.
  513. }
  514. #undef ADDR_IN_ATTRIBUTE_SECTION
  515. static bool checked_sections = false;
  516. static inline void CheckInHookCaller() {
  517. if (!checked_sections) {
  518. INIT_ATTRIBUTE_SECTION_VARS(google_malloc);
  519. if (ATTRIBUTE_SECTION_START(google_malloc) ==
  520. ATTRIBUTE_SECTION_STOP(google_malloc)) {
  521. RAW_LOG(ERROR, "google_malloc section is missing, "
  522. "thus InHookCaller is broken!");
  523. }
  524. INIT_ATTRIBUTE_SECTION_VARS(malloc_hook);
  525. if (ATTRIBUTE_SECTION_START(malloc_hook) ==
  526. ATTRIBUTE_SECTION_STOP(malloc_hook)) {
  527. RAW_LOG(ERROR, "malloc_hook section is missing, "
  528. "thus InHookCaller is broken!");
  529. }
  530. checked_sections = true;
  531. }
  532. }
  533. #endif // !NO_TCMALLOC_SAMPLES
  534. // We can improve behavior/compactness of this function
  535. // if we pass a generic test function (with a generic arg)
  536. // into the implementations for GetStackTrace instead of the skip_count.
  537. extern "C" int MallocHook_GetCallerStackTrace(void** result, int max_depth,
  538. int skip_count) {
  539. #if defined(NO_TCMALLOC_SAMPLES)
  540. return 0;
  541. #elif !defined(HAVE_ATTRIBUTE_SECTION_START)
  542. // Fall back to GetStackTrace and good old but fragile frame skip counts.
  543. // Note: this path is inaccurate when a hook is not called directly by an
  544. // allocation function but is daisy-chained through another hook,
  545. // search for MallocHook::(Get|Set|Invoke)* to find such cases.
  546. return GetStackTrace(result, max_depth, skip_count + int(DEBUG_MODE));
  547. // due to -foptimize-sibling-calls in opt mode
  548. // there's no need for extra frame skip here then
  549. #else
  550. CheckInHookCaller();
  551. // MallocHook caller determination via InHookCaller works, use it:
  552. static const int kMaxSkip = 32 + 6 + 3;
  553. // Constant tuned to do just one GetStackTrace call below in practice
  554. // and not get many frames that we don't actually need:
  555. // currently max passsed max_depth is 32,
  556. // max passed/needed skip_count is 6
  557. // and 3 is to account for some hook daisy chaining.
  558. static const int kStackSize = kMaxSkip + 1;
  559. void* stack[kStackSize];
  560. int depth = GetStackTrace(stack, kStackSize, 1); // skip this function frame
  561. if (depth == 0) // silenty propagate cases when GetStackTrace does not work
  562. return 0;
  563. for (int i = 0; i < depth; ++i) { // stack[0] is our immediate caller
  564. if (InHookCaller(stack[i])) {
  565. RAW_VLOG(10, "Found hooked allocator at %d: %p <- %p",
  566. i, stack[i], stack[i+1]);
  567. i += 1; // skip hook caller frame
  568. depth -= i; // correct depth
  569. if (depth > max_depth) depth = max_depth;
  570. copy(stack + i, stack + i + depth, result);
  571. if (depth < max_depth && depth + i == kStackSize) {
  572. // get frames for the missing depth
  573. depth +=
  574. GetStackTrace(result + depth, max_depth - depth, 1 + kStackSize);
  575. }
  576. return depth;
  577. }
  578. }
  579. RAW_LOG(WARNING, "Hooked allocator frame not found, returning empty trace");
  580. // If this happens try increasing kMaxSkip
  581. // or else something must be wrong with InHookCaller,
  582. // e.g. for every section used in InHookCaller
  583. // all functions in that section must be inside the same library.
  584. return 0;
  585. #endif
  586. }
  587. // On systems where we know how, we override mmap/munmap/mremap/sbrk
  588. // to provide support for calling the related hooks (in addition,
  589. // of course, to doing what these functions normally do).
  590. #if defined(__linux)
  591. #ifndef TCMALLOC_SGX /*SGX don't support mmap*/
  592. # include "malloc_hook_mmap_linux.h"
  593. #endif
  594. #elif defined(__FreeBSD__)
  595. # include "malloc_hook_mmap_freebsd.h"
  596. #else
  597. /*static*/void* MallocHook::UnhookedMMap(void *start, size_t length, int prot,
  598. int flags, int fd, off_t offset) {
  599. void* result;
  600. if (!MallocHook::InvokeMmapReplacement(
  601. start, length, prot, flags, fd, offset, &result)) {
  602. result = mmap(start, length, prot, flags, fd, offset);
  603. }
  604. return result;
  605. }
  606. /*static*/int MallocHook::UnhookedMUnmap(void *start, size_t length) {
  607. int result;
  608. if (!MallocHook::InvokeMunmapReplacement(start, length, &result)) {
  609. result = munmap(start, length);
  610. }
  611. return result;
  612. }
  613. #endif