Merge pull request #27232 from artfulrobot/artfulrobot-add-smart-grou…

…p-profiler Add simple smart group profiler
civicrm · Sep 16, 2023 · 554ec0e · 554ec0e
2 parents 0f1d2b7 + c8a1963
commit 554ec0e
Show file tree

Hide file tree

Showing 4 changed files with 72 additions and 9 deletions.
diff --git a/CRM/Contact/BAO/GroupContactCache.php b/CRM/Contact/BAO/GroupContactCache.php
@@ -162,21 +162,35 @@ public static function add($groupIDs) {
    * @param array $groupID
    * @param bool $processed
    *   Whether the cache data was recently modified.
+   * @param ?float $startTime
+   *   A float from microtime() for when we began work updating this group.
    */
-  protected static function updateCacheTime($groupID, $processed) {
+  protected static function updateCacheTime($groupID, $processed, ?float $startTime = NULL) {
     // only update cache entry if we had any values
+    $now = 'null';
+    $setCacheFillTook = '';
     if ($processed) {
       // also update the group with cache date information
       $now = date('YmdHis');
-    }
-    else {
-      $now = 'null';
+      if ($startTime) {
+        // If we can calculate how long this took, we update the cache_fill_took column.
+        $took = microtime(TRUE) - $startTime;
+        $setCacheFillTook = ", cache_fill_took = $took";
+        $maxTime = CRM_Utils_Constant::value('CIVICRM_SLOW_SMART_GROUP_SECONDS');
+        if ($maxTime && $took > $maxTime) {
+          Civi::log()->warning(
+            "Updating smart group $groupID took over "
+            . $maxTime
+            . "s defined by CIVICRM_SLOW_SMART_GROUP_SECONDS (took " . number_format($took, 3) . "s)"
+          );
+        }
+      }
     }
 
     $groupIDs = implode(',', $groupID);
     $sql = "
 UPDATE civicrm_group
-SET    cache_date = $now
+SET    cache_date = $now$setCacheFillTook
 WHERE  id IN ( $groupIDs )
 ";
     CRM_Core_DAO::executeQuery($sql);
@@ -353,14 +367,16 @@ public static function load($group, $force = FALSE) {
 
     $lockedGroups = self::getLocksForRefreshableGroupsTo([$groupID]);
     foreach ($lockedGroups as $groupID) {
+      $startTime = microtime(TRUE);
       $groupContactsTempTable = CRM_Utils_SQL_TempTable::build()
         ->setCategory('gccache')
         ->setMemory();
       self::buildGroupContactTempTable([$groupID], $groupContactsTempTable);
       self::clearGroupContactCache([$groupID]);
-      self::updateCacheFromTempTable($groupContactsTempTable, [$groupID]);
+      self::updateCacheFromTempTable($groupContactsTempTable, [$groupID], $startTime);
       self::releaseGroupLocks([$groupID]);
       $groupContactsTempTable->drop();
+
     }
     return in_array($groupID, $lockedGroups);
   }
@@ -723,8 +739,10 @@ protected static function getGroupsNeedingRefreshing(?array $groupIDs, int $limi
    *
    * @param \CRM_Utils_SQL_TempTable $groupContactsTempTable
    * @param array $groupIDs
+   * @param ?float $startTime
+   *   A float from microtime() for when we began work updating this group.
    */
-  private static function updateCacheFromTempTable(CRM_Utils_SQL_TempTable $groupContactsTempTable, array $groupIDs): void {
+  private static function updateCacheFromTempTable(CRM_Utils_SQL_TempTable $groupContactsTempTable, array $groupIDs, ?float $startTime = NULL): void {
     $tempTable = $groupContactsTempTable->getName();
 
     // @fixme: GROUP BY is here to guard against having duplicate contacts in the temptable.
@@ -736,7 +754,7 @@ private static function updateCacheFromTempTable(CRM_Utils_SQL_TempTable $groupC
         GROUP BY contact_id,group_id
       ");
     foreach ($groupIDs as $groupID) {
-      self::updateCacheTime([$groupID], TRUE);
+      self::updateCacheTime([$groupID], TRUE, $startTime);
     }
   }
 

diff --git a/CRM/Contact/DAO/Group.php b/CRM/Contact/DAO/Group.php
@@ -6,7 +6,7 @@
  *
  * Generated from xml/schema/CRM/Contact/Group.xml
  * DO NOT EDIT.  Generated by CRM_Core_CodeGen
- * (GenCodeChecksum:81bebe4ba76713e7fb6a39e7710634a4)
+ * (GenCodeChecksum:b2a4c3f0d055dd4cb49c4bb756f57f59)
  */
 
 /**
@@ -177,6 +177,15 @@ class CRM_Contact_DAO_Group extends CRM_Core_DAO {
    */
   public $cache_date;
 
+  /**
+   * Seconds taken to fill smart group cache
+   *
+   * @var float|string
+   *   (SQL type: decimal(20,2))
+   *   Note that values will be retrieved from the database as a string.
+   */
+  public $cache_fill_took;
+
   /**
    * Unused deprecated column.
    *
@@ -594,6 +603,30 @@ public static function &fields() {
           'readonly' => TRUE,
           'add' => '2.1',
         ],
+        'cache_fill_took' => [
+          'name' => 'cache_fill_took',
+          'type' => CRM_Utils_Type::T_MONEY,
+          'title' => ts('Seconds taken by last cache fill'),
+          'description' => ts('Seconds taken to fill smart group cache'),
+          'required' => FALSE,
+          'precision' => [
+            20,
+            2,
+          ],
+          'usage' => [
+            'import' => FALSE,
+            'export' => FALSE,
+            'duplicate_matching' => FALSE,
+            'token' => FALSE,
+          ],
+          'where' => 'civicrm_group.cache_fill_took',
+          'table_name' => 'civicrm_group',
+          'entity' => 'Group',
+          'bao' => 'CRM_Contact_BAO_Group',
+          'localizable' => 0,
+          'readonly' => TRUE,
+          'add' => '5.65',
+        ],
         'refresh_date' => [
           'name' => 'refresh_date',
           'type' => CRM_Utils_Type::T_TIMESTAMP,

diff --git a/CRM/Upgrade/Incremental/php/FiveSixtySeven.php b/CRM/Upgrade/Incremental/php/FiveSixtySeven.php
@@ -33,6 +33,9 @@ public function upgrade_5_67_alpha1($rev): void {
     $this->addExtensionTask('Enable Authx extension', ['authx'], 1101);
     $this->addExtensionTask('Enable Afform extension', ['org.civicrm.afform'], 1102);
     $this->addTask('Add "civicrm_note" to "note_used_for" option group', 'addNoteNote');
+    $this->addTask('Add cache_fill_took column to Group table', 'addColumn', 'civicrm_group', 'cache_fill_took',
+      'DOUBLE DEFAULT NULL COMMENT "Seconds taken to fill smart group cache, not always related to cache_date"',
+      FALSE);
   }
 
   public static function addNoteNote(CRM_Queue_TaskContext $ctx): bool {

diff --git a/xml/schema/Contact/Group.xml b/xml/schema/Contact/Group.xml
@@ -176,6 +176,15 @@
     <fieldName>cache_date</fieldName>
     <add>5.34</add>
   </index>
+  <field>
+    <name>cache_fill_took</name>
+    <type>float</type>
+    <title>Seconds taken by last cache fill</title>
+    <comment>Seconds taken to fill smart group cache</comment>
+    <required>false</required>
+    <readonly>true</readonly>
+    <add>5.67</add>
+  </field>
   <field>
     <name>refresh_date</name>
     <type>timestamp</type>