Track total memory allocations

Summary:
Using instrumentation in jemalloc, keep track of the total
memory allocations done in a particular script/request.  This allows
better understanding of what functions may be memory hungry, but not
peak memory hungry.

Test Plan: prod run logging total allocation

Reviewers: je

Reviewed By: je

CC: hphp-diffs@lists, ps, mwilliams, je

Differential Revision: 340437

src/idl/options.idl.php

@@ -654,6 +654,18 @@
     'taint_observer' => false,
   ));
 
+DefineFunction(
+  array(
+    'name'   => "memory_get_allocation",
+    'desc'   => "Returns the total memory, in bytes, that your PHP script has allocated.",
+    'flags'  =>  HasDocComment | NoProfile,
+    'return' => array(
+      'type'   => Int64,
+      'desc'   => "Returns the total memory allocated in bytes.",
+    ),
+    'taint_observer' => false,
+  ));
+
 DefineFunction(
   array(
     'name'   => "memory_get_peak_usage",

src/runtime/base/memory/memory_manager.cpp

@@ -138,9 +138,11 @@ void MemoryManager::resetStats() {
   m_stats.alloc = 0;
   m_stats.peakUsage = 0;
   m_stats.peakAlloc = 0;
+  m_stats.totalAlloc = 0;
 #ifdef USE_JEMALLOC
   if (s_statsEnabled) {
-    m_delta = int64(*m_allocated) - int64(*m_deallocated);
+    m_prevAllocated = int64(*m_allocated);
+    m_delta = m_prevAllocated - int64(*m_deallocated);
   }
 #endif
 }

src/runtime/base/memory/memory_manager.h

@@ -145,8 +145,11 @@ class MemoryManager {
     // substantially exceed m_stats.usage.
     if (s_statsEnabled) {
       int64 delta = int64(*m_allocated) - int64(*m_deallocated);
+      int64 deltaAllocated = int64(*m_allocated) - m_prevAllocated;
       m_stats.usage += delta - m_delta;
+      m_stats.totalAlloc += deltaAllocated;
       m_delta = delta;
+      m_prevAllocated = int64(*m_allocated);
     }
 #endif
     if (m_stats.usage > m_stats.peakUsage) {
@@ -214,6 +217,7 @@ class MemoryManager {
   uint64* m_allocated;
   uint64* m_deallocated;
   int64  m_delta;
+  int64  m_prevAllocated;
   size_t* m_cactive;
   size_t m_cactiveLimit;
   bool m_stopped; // Set to true if m_cactive exceeded limit.

src/runtime/base/memory/smart_allocator.h

@@ -130,11 +130,12 @@ void InitAllocatorThreadLocal() ATTRIBUTE_COLD;
  * Usage stats, all in bytes.
  */
 struct MemoryUsageStats {
-  int64 maxBytes;  // what's request's max bytes allowed
-  int64 usage;     // how many bytes are currently being used
-  int64 alloc;     // how many bytes are currently malloc-ed
-  int64 peakUsage; // how many bytes have been dispensed at maximum
-  int64 peakAlloc; // how many bytes malloc-ed at maximum
+  int64 maxBytes;   // what's request's max bytes allowed
+  int64 usage;      // how many bytes are currently being used
+  int64 alloc;      // how many bytes are currently malloc-ed
+  int64 peakUsage;  // how many bytes have been dispensed at maximum
+  int64 peakAlloc;  // how many bytes malloc-ed at maximum
+  int64 totalAlloc; // how many bytes allocated, in total.
 };
 
 ///////////////////////////////////////////////////////////////////////////////

src/runtime/ext/ext_options.cpp

@@ -656,6 +656,15 @@ String f_ini_set(CStrRef varname, CStrRef newvalue) {
   return oldvalue;
 }
 
+int64 f_memory_get_allocation() {
+  if (RuntimeOption::EnableMemoryManager) {
+    MemoryManager *mm = MemoryManager::TheMemoryManager().getNoCheck();
+    const MemoryUsageStats &stats = mm->getStats(true);
+    return stats.totalAlloc;
+  }
+  return 0;
+}
+
 int64 f_memory_get_peak_usage(bool real_usage /* = false */) {
   if (RuntimeOption::EnableMemoryManager) {
     MemoryManager *mm = MemoryManager::TheMemoryManager().getNoCheck();

src/runtime/ext/ext_options.h

@@ -59,6 +59,7 @@ Array f_ini_get_all(CStrRef extension = null_string);
 String f_ini_get(CStrRef varname);
 void f_ini_restore(CStrRef varname);
 String f_ini_set(CStrRef varname, CStrRef newvalue);
+int64 f_memory_get_allocation();
 int64 f_memory_get_peak_usage(bool real_usage = false);
 int64 f_memory_get_usage(bool real_usage = false);
 String f_php_ini_scanned_files();

src/runtime/ext/profile/extprofile_options.h

@@ -198,6 +198,11 @@ inline String x_ini_set(CStrRef varname, CStrRef newvalue) {
   return f_ini_set(varname, newvalue);
 }
 
+inline int64 x_memory_get_allocation() {
+  FUNCTION_NOPROFILE_BUILTIN(memory_get_allocation);
+  return f_memory_get_allocation();
+}
+
 inline int64 x_memory_get_peak_usage(bool real_usage = false) {
   FUNCTION_INJECTION_BUILTIN(memory_get_peak_usage);
   return f_memory_get_peak_usage(real_usage);

src/system/gen/php/classes/arrayaccess.cpp

@@ -18,8 +18,8 @@
 #include <runtime/base/hphp_system.h>
 #include <system/gen/sys/literal_strings_remap.h>
 #include <system/gen/sys/scalar_arrays_remap.h>
-#include <php/classes/arrayaccess.h>
 #include <php/classes/arrayaccess.fws.h>
+#include <php/classes/arrayaccess.h>
 
 // Dependencies
 #include <runtime/ext/ext.h>

src/system/gen/php/classes/debugger.cpp

@@ -18,8 +18,8 @@
 #include <runtime/base/hphp_system.h>
 #include <system/gen/sys/literal_strings_remap.h>
 #include <system/gen/sys/scalar_arrays_remap.h>
-#include <php/classes/debugger.h>
 #include <php/classes/debugger.fws.h>
+#include <php/classes/debugger.h>
 
 // Dependencies
 #include <runtime/ext/ext.h>

src/system/gen/php/classes/directory.cpp

@@ -18,8 +18,8 @@
 #include <runtime/base/hphp_system.h>
 #include <system/gen/sys/literal_strings_remap.h>
 #include <system/gen/sys/scalar_arrays_remap.h>
-#include <php/classes/directory.h>
 #include <php/classes/directory.fws.h>
+#include <php/classes/directory.h>
 
 // Dependencies
 #include <runtime/ext/ext.h>

src/system/gen/php/classes/directoryiterator.cpp

@@ -18,8 +18,8 @@
 #include <runtime/base/hphp_system.h>
 #include <system/gen/sys/literal_strings_remap.h>
 #include <system/gen/sys/scalar_arrays_remap.h>
-#include <php/classes/directoryiterator.h>
 #include <php/classes/directoryiterator.fws.h>
+#include <php/classes/directoryiterator.h>
 
 // Dependencies
 #include <php/classes/exception.h>

src/system/gen/php/classes/exception.cpp

@@ -18,8 +18,8 @@
 #include <runtime/base/hphp_system.h>
 #include <system/gen/sys/literal_strings_remap.h>
 #include <system/gen/sys/scalar_arrays_remap.h>
-#include <php/classes/exception.h>
 #include <php/classes/exception.fws.h>
+#include <php/classes/exception.h>
 
 // Dependencies
 #include <runtime/ext/ext.h>

src/system/gen/php/classes/fbmysqllexer.cpp

@@ -18,8 +18,8 @@
 #include <runtime/base/hphp_system.h>
 #include <system/gen/sys/literal_strings_remap.h>
 #include <system/gen/sys/scalar_arrays_remap.h>
-#include <php/classes/fbmysqllexer.h>
 #include <php/classes/fbmysqllexer.fws.h>
+#include <php/classes/fbmysqllexer.h>
 
 // Dependencies
 #include <runtime/ext/ext.h>

src/system/gen/php/classes/iterator.cpp

@@ -18,8 +18,8 @@
 #include <runtime/base/hphp_system.h>
 #include <system/gen/sys/literal_strings_remap.h>
 #include <system/gen/sys/scalar_arrays_remap.h>
-#include <php/classes/iterator.h>
 #include <php/classes/iterator.fws.h>
+#include <php/classes/iterator.h>
 
 // Dependencies
 #include <php/classes/arrayaccess.h>

src/system/gen/php/classes/reflection.cpp

@@ -18,8 +18,8 @@
 #include <runtime/base/hphp_system.h>
 #include <system/gen/sys/literal_strings_remap.h>
 #include <system/gen/sys/scalar_arrays_remap.h>
-#include <php/classes/reflection.h>
 #include <php/classes/reflection.fws.h>
+#include <php/classes/reflection.h>
 
 // Dependencies
 #include <php/classes/exception.h>

src/system/gen/php/classes/soapfault.cpp

@@ -18,8 +18,8 @@
 #include <runtime/base/hphp_system.h>
 #include <system/gen/sys/literal_strings_remap.h>
 #include <system/gen/sys/scalar_arrays_remap.h>
-#include <php/classes/soapfault.h>
 #include <php/classes/soapfault.fws.h>
+#include <php/classes/soapfault.h>
 
 // Dependencies
 #include <php/classes/exception.h>

src/system/gen/php/classes/splfile.cpp

@@ -18,8 +18,8 @@
 #include <runtime/base/hphp_system.h>
 #include <system/gen/sys/literal_strings_remap.h>
 #include <system/gen/sys/scalar_arrays_remap.h>
-#include <php/classes/splfile.h>
 #include <php/classes/splfile.fws.h>
+#include <php/classes/splfile.h>
 
 // Dependencies
 #include <php/classes/iterator.h>

src/system/gen/php/classes/splobjectstorage.cpp

@@ -18,8 +18,8 @@
 #include <runtime/base/hphp_system.h>
 #include <system/gen/sys/literal_strings_remap.h>
 #include <system/gen/sys/scalar_arrays_remap.h>
-#include <php/classes/splobjectstorage.h>
 #include <php/classes/splobjectstorage.fws.h>
+#include <php/classes/splobjectstorage.h>
 
 // Dependencies
 #include <php/classes/iterator.h>

src/system/gen/php/classes/stdclass.cpp

@@ -18,8 +18,8 @@
 #include <runtime/base/hphp_system.h>
 #include <system/gen/sys/literal_strings_remap.h>
 #include <system/gen/sys/scalar_arrays_remap.h>
-#include <php/classes/stdclass.h>
 #include <php/classes/stdclass.fws.h>
+#include <php/classes/stdclass.h>
 
 // Dependencies
 #include <runtime/ext/ext.h>

src/system/gen/php/classes/xhprof.cpp

@@ -18,8 +18,8 @@
 #include <runtime/base/hphp_system.h>
 #include <system/gen/sys/literal_strings_remap.h>
 #include <system/gen/sys/scalar_arrays_remap.h>
-#include <php/classes/xhprof.h>
 #include <php/classes/xhprof.fws.h>
+#include <php/classes/xhprof.h>
 
 // Dependencies
 #include <runtime/ext/ext.h>

src/system/gen/php/globals/constants.cpp

@@ -18,8 +18,8 @@
 #include <runtime/base/hphp_system.h>
 #include <system/gen/sys/literal_strings_remap.h>
 #include <system/gen/sys/scalar_arrays_remap.h>
-#include <php/globals/constants.h>
 #include <php/globals/constants.fws.h>
+#include <php/globals/constants.h>
 
 // Dependencies
 #include <runtime/ext/ext.h>

src/system/gen/php/globals/symbols.cpp

@@ -18,8 +18,8 @@
 #include <runtime/base/hphp_system.h>
 #include <system/gen/sys/literal_strings_remap.h>
 #include <system/gen/sys/scalar_arrays_remap.h>
-#include <php/globals/symbols.h>
 #include <php/globals/symbols.fws.h>
+#include <php/globals/symbols.h>
 
 // Dependencies
 #include <runtime/ext/ext.h>

src/system/gen/sys/dynamic_table_func.no.cpp

@@ -1337,6 +1337,13 @@ Variant ifa_imap_utf7_decode(void *extra, int count, INVOKE_FEW_ARGS_IMPL_ARGS)
 Variant i_imap_utf7_decode(void *extra, CArrRef params) {
   return invoke_func_few_handler(extra, params, &ifa_imap_utf7_decode);
 }
+Variant ifa_memory_get_allocation(void *extra, int count, INVOKE_FEW_ARGS_IMPL_ARGS) {
+  if (UNLIKELY(count > 0)) return throw_toomany_arguments("memory_get_allocation", 0, 1);
+  return (x_memory_get_allocation());
+}
+Variant i_memory_get_allocation(void *extra, CArrRef params) {
+  return invoke_func_few_handler(extra, params, &ifa_memory_get_allocation);
+}
 Variant ifa_magicknextimage(void *extra, int count, INVOKE_FEW_ARGS_IMPL_ARGS) {
   if (UNLIKELY(count != 1)) return throw_wrong_arguments("magicknextimage", count, 1, 1, 1);
   CVarRef arg0(a0);
@@ -21502,6 +21509,7 @@ CallInfo ci_drawpathcurvetosmoothabsolute((void*)&i_drawpathcurvetosmoothabsolut
 CallInfo ci_wandgetexception((void*)&i_wandgetexception, (void*)&ifa_wandgetexception, 1, 0, 0x0000000000000000LL);
 CallInfo ci_crc32((void*)&i_crc32, (void*)&ifa_crc32, 1, 0, 0x0000000000000000LL);
 CallInfo ci_imap_utf7_decode((void*)&i_imap_utf7_decode, (void*)&ifa_imap_utf7_decode, 1, 0, 0x0000000000000000LL);
+CallInfo ci_memory_get_allocation((void*)&i_memory_get_allocation, (void*)&ifa_memory_get_allocation, 0, 0, 0x0000000000000000LL);
 CallInfo ci_magicknextimage((void*)&i_magicknextimage, (void*)&ifa_magicknextimage, 1, 0, 0x0000000000000000LL);
 CallInfo ci_imagecreatefromwbmp((void*)&i_imagecreatefromwbmp, (void*)&ifa_imagecreatefromwbmp, 1, 0, 0x0000000000000000LL);
 CallInfo ci_libxml_get_last_error((void*)&i_libxml_get_last_error, (void*)&ifa_libxml_get_last_error, 0, 0, 0x0000000000000000LL);
@@ -27550,6 +27558,10 @@ bool get_call_info_builtin(const CallInfo *&ci, void *&extra, const char *s, int
       }
       break;
     case 2723:
+      HASH_GUARD(0x57B7D47A9D1E6AA3LL, memory_get_allocation) {
+        ci = &ci_memory_get_allocation;
+        return true;
+      }
       HASH_GUARD(0x2B75B48A53AACAA3LL, imagestring) {
         ci = &ci_imagestring;
         return true;

src/system/options.inc

@@ -36,6 +36,7 @@
 "ini_get", T(String), S(0), "varname", T(String), NULL, NULL, S(0), NULL, S(16384), "/**\n * ( excerpt from http://php.net/manual/en/function.ini-get.php )\n *\n * Returns the value of the configuration option on success.\n *\n * @varname    string  The configuration option name.\n *\n * @return     string  Returns the value of the configuration option as a\n *                     string on success, or an empty string on failure or\n *                     for null values.\n */", 
 "ini_restore", T(Void), S(0), "varname", T(String), NULL, NULL, S(0), NULL, S(16384), "/**\n * ( excerpt from http://php.net/manual/en/function.ini-restore.php )\n *\n * Restores a given configuration option to its original value.\n *\n * @varname    string  The configuration option name.\n *\n * @return     mixed   No value is returned.\n */", 
 "ini_set", T(String), S(0), "varname", T(String), NULL, NULL, S(0), "newvalue", T(String), NULL, NULL, S(0), NULL, S(16384), "/**\n * ( excerpt from http://php.net/manual/en/function.ini-set.php )\n *\n * Sets the value of the given configuration option. The configuration\n * option will keep this new value during the script's execution, and will\n * be restored at the script's ending.\n *\n * @varname    string  Not all the available options can be changed using\n *                     ini_set(). There is a list of all available options\n *                     in the appendix.\n * @newvalue   string  The new value for the option.\n *\n * @return     string  Returns the old value on success, FALSE on failure.\n */", 
+"memory_get_allocation", T(Int64), S(0), NULL, S(33570816), "/**\n * ( excerpt from\n * http://php.net/manual/en/function.memory-get-allocation.php )\n *\n * Returns the total memory, in bytes, that your PHP script has allocated.\n *\n * @return     int     Returns the total memory allocated in bytes.\n */", 
 "memory_get_peak_usage", T(Int64), S(0), "real_usage", T(Boolean), "b:0;", "false", S(0), NULL, S(16384), "/**\n * ( excerpt from\n * http://php.net/manual/en/function.memory-get-peak-usage.php )\n *\n * Returns the peak of memory, in bytes, that's been allocated to your PHP\n * script.\n *\n * @real_usage bool    Set this to TRUE to get the real size of memory\n *                     allocated from system. If not set or FALSE only the\n *                     memory used by emalloc() is reported.\n *\n * @return     int     Returns the memory peak in bytes.\n */", 
 "memory_get_usage", T(Int64), S(0), "real_usage", T(Boolean), "b:0;", "false", S(0), NULL, S(16384), "/**\n * ( excerpt from http://php.net/manual/en/function.memory-get-usage.php )\n *\n * Returns the amount of memory, in bytes, that's currently being\n * allocated to your PHP script.\n *\n * @real_usage bool    Set this to TRUE to get the real size of memory\n *                     allocated from system. If not set or FALSE only the\n *                     memory used by emalloc() is reported.\n *\n * @return     int     Returns the memory amount in bytes.\n */", 
 "php_ini_scanned_files", T(String), S(0), NULL, S(16384), "/**\n * ( excerpt from\n * http://php.net/manual/en/function.php-ini-scanned-files.php )\n *\n * php_ini_scanned_files() returns a comma-separated list of configuration\n * files parsed after php.ini. These files are found in a directory defined\n * by the --with-config-file-scan-dir option which is set during\n * compilation.\n *\n * The returned configuration files also include the path as declared in\n * the --with-config-file-scan-dir option.\n *\n * @return     string  Returns a comma-separated string of .ini files on\n *                     success. Each comma is followed by a newline. If the\n *                     directive --with-config-file-scan-dir wasn't set,\n *                     FALSE is returned. If it was set and the directory\n *                     was empty, an empty string is returned. If a file is\n *                     unrecognizable, the file will still make it into the\n *                     returned string but a PHP error will also result.\n *                     This PHP error will be seen both at compile time and\n *                     while using php_ini_scanned_files().\n */",