Provide bin/garbage for interacting with garbage collection

Summary:
Fixes T9494. This:

  - Removes all the random GC.x.y.z config.
  - Puts it all in one place that's locked and which you use `bin/garbage set-policy ...` to adjust.
  - Makes every TTL-based GC configurable.
  - Simplifies the code in the actual GCs.

Test Plan:
  - Ran `bin/garbage collect` to collect some garbage, until it stopped collecting.
  - Ran `bin/garbage set-policy ...` to shorten policy. Saw change in web UI. Ran `bin/garbage collect` again and saw it collect more garbage.
  - Set policy to indefinite and saw it not collect garabge.
  - Set policy to default and saw it reflected in web UI / `collect`.
  - Ran `bin/phd debug trigger` and saw all GCs fire with reasonable looking queries.
  - Read new docs.

{F857928}

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T9494

Differential Revision: https://secure.phabricator.com/D14219

Author: epriestley

bin/garbage

@@ -0,0 +1 @@
+../scripts/setup/manage_garbage.php

scripts/setup/manage_garbage.php

@@ -0,0 +1,21 @@
+#!/usr/bin/env php
+<?php
+
+$root = dirname(dirname(dirname(__FILE__)));
+require_once $root.'/scripts/__init_script__.php';
+
+$args = new PhutilArgumentParser($argv);
+$args->setTagline(pht('manage garbage colletors'));
+$args->setSynopsis(<<<EOSYNOPSIS
+**garbage** __command__ [__options__]
+    Manage garbage collectors.
+
+EOSYNOPSIS
+  );
+$args->parseStandardArguments();
+
+$workflows = id(new PhutilClassMapQuery())
+  ->setAncestorClass('PhabricatorGarbageCollectorManagementWorkflow')
+  ->execute();
+$workflows[] = new PhutilHelpArgumentWorkflow();
+$args->parseWorkflows($workflows);

src/__phutil_library_map__.php

@@ -2197,7 +2197,9 @@
     'PhabricatorFundApplication' => 'applications/fund/application/PhabricatorFundApplication.php',
     'PhabricatorGDSetupCheck' => 'applications/config/check/PhabricatorGDSetupCheck.php',
     'PhabricatorGarbageCollector' => 'infrastructure/daemon/garbagecollector/PhabricatorGarbageCollector.php',
-    'PhabricatorGarbageCollectorConfigOptions' => 'applications/config/option/PhabricatorGarbageCollectorConfigOptions.php',
+    'PhabricatorGarbageCollectorManagementCollectWorkflow' => 'infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementCollectWorkflow.php',
+    'PhabricatorGarbageCollectorManagementSetPolicyWorkflow' => 'infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementSetPolicyWorkflow.php',
+    'PhabricatorGarbageCollectorManagementWorkflow' => 'infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementWorkflow.php',
     'PhabricatorGestureUIExample' => 'applications/uiexample/examples/PhabricatorGestureUIExample.php',
     'PhabricatorGitGraphStream' => 'applications/repository/daemon/PhabricatorGitGraphStream.php',
     'PhabricatorGitHubAuthProvider' => 'applications/auth/provider/PhabricatorGitHubAuthProvider.php',
@@ -6197,7 +6199,9 @@
     'PhabricatorFundApplication' => 'PhabricatorApplication',
     'PhabricatorGDSetupCheck' => 'PhabricatorSetupCheck',
     'PhabricatorGarbageCollector' => 'Phobject',
-    'PhabricatorGarbageCollectorConfigOptions' => 'PhabricatorApplicationConfigOptions',
+    'PhabricatorGarbageCollectorManagementCollectWorkflow' => 'PhabricatorGarbageCollectorManagementWorkflow',
+    'PhabricatorGarbageCollectorManagementSetPolicyWorkflow' => 'PhabricatorGarbageCollectorManagementWorkflow',
+    'PhabricatorGarbageCollectorManagementWorkflow' => 'PhabricatorManagementWorkflow',
     'PhabricatorGestureUIExample' => 'PhabricatorUIExample',
     'PhabricatorGitGraphStream' => 'PhabricatorRepositoryGraphStream',
     'PhabricatorGitHubAuthProvider' => 'PhabricatorOAuth2AuthProvider',

src/applications/auth/garbagecollector/PhabricatorAuthSessionGarbageCollector.php

@@ -6,14 +6,14 @@ final class PhabricatorAuthSessionGarbageCollector
   const COLLECTORCONST = 'auth.sessions';
 
   public function getCollectorName() {
-    return pht('Auth Sessions');
+    return pht('Authentication Sessions');
   }
 
   public function hasAutomaticPolicy() {
     return true;
   }
 
-  public function collectGarbage() {
+  protected function collectGarbage() {
     $session_table = new PhabricatorAuthSession();
     $conn_w = $session_table->establishConnection('w');
 

src/applications/auth/garbagecollector/PhabricatorAuthTemporaryTokenGarbageCollector.php

@@ -6,14 +6,14 @@ final class PhabricatorAuthTemporaryTokenGarbageCollector
   const COLLECTORCONST = 'auth.tokens';
 
   public function getCollectorName() {
-    return pht('Auth Tokens');
+    return pht('Authentication Tokens');
   }
 
   public function hasAutomaticPolicy() {
     return true;
   }
 
-  public function collectGarbage() {
+  protected function collectGarbage() {
     $session_table = new PhabricatorAuthTemporaryToken();
     $conn_w = $session_table->establishConnection('w');
 

src/applications/cache/garbagecollector/PhabricatorCacheGeneralGarbageCollector.php

@@ -13,13 +13,7 @@ public function getDefaultRetentionPolicy() {
     return phutil_units('30 days in seconds');
   }
 
-  public function collectGarbage() {
-    $key = 'gcdaemon.ttl.general-cache';
-    $ttl = PhabricatorEnv::getEnvConfig($key);
-    if ($ttl <= 0) {
-      return false;
-    }
-
+  protected function collectGarbage() {
     $cache = new PhabricatorKeyValueDatabaseCache();
     $conn_w = $cache->establishConnection('w');
 
@@ -28,7 +22,7 @@ public function collectGarbage() {
       'DELETE FROM %T WHERE cacheCreated < %d
         ORDER BY cacheCreated ASC LIMIT 100',
       $cache->getTableName(),
-      time() - $ttl);
+      $this->getGarbageEpoch());
 
     return ($conn_w->getAffectedRows() == 100);
   }

src/applications/cache/garbagecollector/PhabricatorCacheMarkupGarbageCollector.php

@@ -13,21 +13,15 @@ public function getDefaultRetentionPolicy() {
     return phutil_units('30 days in seconds');
   }
 
-  public function collectGarbage() {
-    $key = 'gcdaemon.ttl.markup-cache';
-    $ttl = PhabricatorEnv::getEnvConfig($key);
-    if ($ttl <= 0) {
-      return false;
-    }
-
+  protected function collectGarbage() {
     $table = new PhabricatorMarkupCache();
     $conn_w = $table->establishConnection('w');
 
     queryfx(
       $conn_w,
       'DELETE FROM %T WHERE dateCreated < %d LIMIT 100',
       $table->getTableName(),
-      time() - $ttl);
+      $this->getGarbageEpoch());
 
     return ($conn_w->getAffectedRows() == 100);
   }

src/applications/cache/garbagecollector/PhabricatorCacheTTLGarbageCollector.php

@@ -13,7 +13,7 @@ public function hasAutomaticPolicy() {
     return true;
   }
 
-  public function collectGarbage() {
+  protected function collectGarbage() {
     $cache = new PhabricatorKeyValueDatabaseCache();
     $conn_w = $cache->establishConnection('w');
 
@@ -22,7 +22,7 @@ public function collectGarbage() {
       'DELETE FROM %T WHERE cacheExpires < %d
         ORDER BY cacheExpires ASC LIMIT 100',
       $cache->getTableName(),
-      time());
+      PhabricatorTime::getNow());
 
     return ($conn_w->getAffectedRows() == 100);
   }

src/applications/conduit/garbagecollector/ConduitConnectionGarbageCollector.php

@@ -13,21 +13,16 @@ public function getDefaultRetentionPolicy() {
     return phutil_units('180 days in seconds');
   }
 
-  public function collectGarbage() {
-    $key = 'gcdaemon.ttl.conduit-logs';
-    $ttl = PhabricatorEnv::getEnvConfig($key);
-    if ($ttl <= 0) {
-      return false;
-    }
-
+  protected function collectGarbage() {
     $table = new PhabricatorConduitConnectionLog();
     $conn_w = $table->establishConnection('w');
+
     queryfx(
       $conn_w,
       'DELETE FROM %T WHERE dateCreated < %d
         ORDER BY dateCreated ASC LIMIT 100',
       $table->getTableName(),
-      time() - $ttl);
+      $this->getGarbageEpoch());
 
     return ($conn_w->getAffectedRows() == 100);
   }

src/applications/conduit/garbagecollector/ConduitLogGarbageCollector.php

@@ -13,21 +13,16 @@ public function getDefaultRetentionPolicy() {
     return phutil_units('180 days in seconds');
   }
 
-  public function collectGarbage() {
-    $key = 'gcdaemon.ttl.conduit-logs';
-    $ttl = PhabricatorEnv::getEnvConfig($key);
-    if ($ttl <= 0) {
-      return false;
-    }
-
+  protected function collectGarbage() {
     $table = new PhabricatorConduitMethodCallLog();
     $conn_w = $table->establishConnection('w');
+
     queryfx(
       $conn_w,
       'DELETE FROM %T WHERE dateCreated < %d
         ORDER BY dateCreated ASC LIMIT 100',
       $table->getTableName(),
-      time() - $ttl);
+      $this->getGarbageEpoch());
 
     return ($conn_w->getAffectedRows() == 100);
   }

src/applications/conduit/garbagecollector/ConduitTokenGarbageCollector.php

@@ -13,9 +13,10 @@ public function hasAutomaticPolicy() {
     return true;
   }
 
-  public function collectGarbage() {
+  protected function collectGarbage() {
     $table = new PhabricatorConduitToken();
     $conn_w = $table->establishConnection('w');
+
     queryfx(
       $conn_w,
       'DELETE FROM %T WHERE expires <= %d

src/applications/config/check/PhabricatorExtraConfigSetupCheck.php

@@ -176,6 +176,10 @@ public static function getAncientConfig() {
       'Inbound mail addresses are now configured for each application '.
       'in the Applications tool.');
 
+    $gc_reason = pht(
+      'Garbage collectors are now configured with "%s".',
+      'bin/garbage set-policy');
+
     $ancient_config += array(
       'phid.external-loaders' =>
         pht(
@@ -280,6 +284,14 @@ public static function getAncientConfig() {
       'auth.login-message' => pht(
         'This configuration option has been replaced with a modular '.
         'handler. See T9346.'),
+
+      'gcdaemon.ttl.herald-transcripts' => $gc_reason,
+      'gcdaemon.ttl.daemon-logs' => $gc_reason,
+      'gcdaemon.ttl.differential-parse-cache' => $gc_reason,
+      'gcdaemon.ttl.markup-cache' => $gc_reason,
+      'gcdaemon.ttl.task-archive' => $gc_reason,
+      'gcdaemon.ttl.general-cache' => $gc_reason,
+      'gcdaemon.ttl.conduit-logs' => $gc_reason,
     );
 
     return $ancient_config;

src/applications/config/module/PhabricatorConfigCollectorsModule.php

@@ -17,11 +17,13 @@ public function renderModuleStatus(AphrontRequest $request) {
     $collectors = msort($collectors, 'getCollectorConstant');
 
     $rows = array();
+    $rowc = array();
     foreach ($collectors as $key => $collector) {
+      $class = null;
       if ($collector->hasAutomaticPolicy()) {
         $policy_view = phutil_tag('em', array(), pht('Automatic'));
       } else {
-        $policy = $collector->getDefaultRetentionPolicy();
+        $policy = $collector->getRetentionPolicy();
         if ($policy === null) {
           $policy_view = pht('Indefinite');
         } else {
@@ -30,8 +32,15 @@ public function renderModuleStatus(AphrontRequest $request) {
             '%s Day(s)',
             new PhutilNumber($days));
         }
+
+        $default = $collector->getDefaultRetentionPolicy();
+        if ($policy !== $default) {
+          $class = 'highlighted';
+          $policy_view = phutil_tag('strong', array(), $policy_view);
+        }
       }
 
+      $rowc[] = $class;
       $rows[] = array(
         $collector->getCollectorConstant(),
         $collector->getCollectorName(),
@@ -40,6 +49,7 @@ public function renderModuleStatus(AphrontRequest $request) {
     }
 
     $table = id(new AphrontTableView($rows))
+      ->setRowClasses($rowc)
       ->setHeaders(
         array(
           pht('Constant'),
@@ -53,8 +63,16 @@ public function renderModuleStatus(AphrontRequest $request) {
           null,
         ));
 
+    $header = id(new PHUIHeaderView())
+      ->setHeader(pht('Garbage Collectors'))
+      ->setSubheader(
+        pht(
+          'Collectors with custom policies are highlighted. Use '.
+          '%s to change retention policies.',
+          phutil_tag('tt', array(), 'bin/garbage set-policy')));
+
     return id(new PHUIObjectBoxView())
-      ->setHeaderText(pht('Garbage Collectors'))
+      ->setHeader($header)
       ->setTable($table);
   }
 

src/applications/config/option/PhabricatorGarbageCollectorConfigOptions.php

@@ -80,6 +80,17 @@ public function getOptions() {
             'and the daemons. Primarily, this is a way to suppress the '.
             '"Daemons and Web Have Different Config" setup issue on a per '.
             'config key basis.')),
+      $this->newOption('phd.garbage-collection', 'wild', array())
+        ->setLocked(true)
+        ->setLockedMessage(
+          pht(
+            'This option can not be edited from the web UI. Use %s to adjust '.
+            'garbage collector policies.',
+            phutil_tag('tt', array(), 'bin/garbage set-policy')))
+        ->setSummary(pht('Retention policies for garbage collection.'))
+        ->setDescription(
+          pht(
+            'Customizes retention policies for garbage collectors.')),
     );
   }