Create and populate a stopwords table for InnoDB fulltext indexes to use in the future

Summary:
Ref T11741. InnoDB uses a stopwords table instead of a stopwords file.

During `storage upgrade`, synchronize the table from the stopwords file on disk.

Test Plan:
  - Ran `storage upgrade`.
  - Ran `select * from stopwords`, saw stopwords.
  - Added some garbage to the table.
  - Ran `storage upgrade`, saw it remove it.

Reviewers: chad

Reviewed By: chad

Maniphest Tasks: T11741

Differential Revision: https://secure.phabricator.com/D16940

Author: epriestley

resources/sql/autopatches/20161124.search.01.stopwords.sql

@@ -0,0 +1,3 @@
+CREATE TABLE {$NAMESPACE}_search.stopwords (
+  value VARCHAR(32) NOT NULL COLLATE {$COLLATE_SORT}
+) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};

src/applications/search/storage/PhabricatorSearchSchemaSpec.php

@@ -5,6 +5,14 @@ final class PhabricatorSearchSchemaSpec
 
   public function buildSchemata() {
     $this->buildEdgeSchemata(new PhabricatorProfilePanelConfiguration());
+
+    $this->buildRawSchema(
+      'search',
+      PhabricatorSearchDocument::STOPWORDS_TABLE,
+      array(
+        'value' => 'sort32',
+      ),
+      array());
   }
 
 }

src/applications/search/storage/document/PhabricatorSearchDocument.php

@@ -7,6 +7,8 @@ final class PhabricatorSearchDocument extends PhabricatorSearchDAO {
   protected $documentCreated;
   protected $documentModified;
 
+  const STOPWORDS_TABLE = 'stopwords';
+
   protected function getConfiguration() {
     return array(
       self::CONFIG_TIMESTAMPS => false,

src/infrastructure/storage/management/workflow/PhabricatorStorageManagementUpgradeWorkflow.php

@@ -77,6 +77,17 @@ public function didExecute(PhutilArgumentParser $args) {
 
     $this->upgradeSchemata($apis, $apply_only, $no_quickstart, $init_only);
 
+    if ($apply_only || $init_only) {
+      echo tsprintf(
+        "%s\n",
+        pht('Declining to synchronize static tables.'));
+    } else {
+      echo tsprintf(
+        "%s\n",
+        pht('Synchronizing static tables...'));
+      $this->synchronizeSchemata();
+    }
+
     if ($no_adjust || $init_only || $apply_only) {
       $console->writeOut(
         "%s\n",
@@ -93,4 +104,46 @@ public function didExecute(PhutilArgumentParser $args) {
     return 0;
   }
 
+  private function synchronizeSchemata() {
+    // Synchronize the InnoDB fulltext stopwords table from the stopwords file
+    // on disk.
+
+    $table = new PhabricatorSearchDocument();
+    $conn = $table->establishConnection('w');
+    $table_ref = PhabricatorSearchDocument::STOPWORDS_TABLE;
+
+    $stopwords_database = queryfx_all(
+      $conn,
+      'SELECT value FROM %T',
+      $table_ref);
+    $stopwords_database = ipull($stopwords_database, 'value', 'value');
+
+    $stopwords_path = phutil_get_library_root('phabricator');
+    $stopwords_path = $stopwords_path.'/../resources/sql/stopwords.txt';
+    $stopwords_file = Filesystem::readFile($stopwords_path);
+    $stopwords_file = phutil_split_lines($stopwords_file, false);
+    $stopwords_file = array_fuse($stopwords_file);
+
+    $rem_words = array_diff_key($stopwords_database, $stopwords_file);
+    if ($rem_words) {
+      queryfx(
+        $conn,
+        'DELETE FROM %T WHERE value IN (%Ls)',
+        $table_ref,
+        $rem_words);
+    }
+
+    $add_words = array_diff_key($stopwords_file, $stopwords_database);
+    if ($add_words) {
+      foreach ($add_words as $word) {
+        queryfx(
+          $conn,
+          'INSERT IGNORE INTO %T (value) VALUES (%s)',
+          $table_ref,
+          $word);
+      }
+    }
+  }
+
+
 }