From 3e589cdd73ba40c3e408dabd482ee7fc6165ace1 Mon Sep 17 00:00:00 2001
From: epriestley <git@epriestley.com>
Date: Mon, 2 Oct 2017 16:14:46 -0700
Subject: [PATCH] Add a workflow for populating (or depopulating) the common
 ngrams table

Summary:
Depends on D18672. Ref T13000. This does an on-demand build of the common ngrams table.

Plan here is:

  - Push to `secure`.
  - Build the common ngrams table here.
  - See if stuff breaks?

If it looks okay on this dataset, we can build out the GC support and try it in production.

Test Plan:
  - Locally, my dataset has a bunch of `bin/lipsum` tasks with similar, common words.
  - Verified that ipsum terms now skip ngrams. For "lorem ipsum" search performance actually IMPROVED by skipping the ngrams table (12s to 9s).
  - Queried for normal terms, got very fast results using the ngram table, as normal.

Reviewers: amckinley

Reviewed By: amckinley

Maniphest Tasks: T13000

Differential Revision: https://secure.phabricator.com/D18673
---
 src/__phutil_library_map__.php                |   2 +
 ...bricatorSearchManagementNgramsWorkflow.php | 106 ++++++++++++++++++
 2 files changed, 108 insertions(+)
 create mode 100644 src/applications/search/management/PhabricatorSearchManagementNgramsWorkflow.php

diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php
index c535d46322..dd2bcd8dfe 100644
--- a/src/__phutil_library_map__.php
+++ b/src/__phutil_library_map__.php
@@ -3948,6 +3948,7 @@ phutil_register_library_map(array(
     'PhabricatorSearchIndexVersionDestructionEngineExtension' => 'applications/search/engineextension/PhabricatorSearchIndexVersionDestructionEngineExtension.php',
     'PhabricatorSearchManagementIndexWorkflow' => 'applications/search/management/PhabricatorSearchManagementIndexWorkflow.php',
     'PhabricatorSearchManagementInitWorkflow' => 'applications/search/management/PhabricatorSearchManagementInitWorkflow.php',
+    'PhabricatorSearchManagementNgramsWorkflow' => 'applications/search/management/PhabricatorSearchManagementNgramsWorkflow.php',
     'PhabricatorSearchManagementWorkflow' => 'applications/search/management/PhabricatorSearchManagementWorkflow.php',
     'PhabricatorSearchNgrams' => 'applications/search/ngrams/PhabricatorSearchNgrams.php',
     'PhabricatorSearchNgramsDestructionEngineExtension' => 'applications/search/engineextension/PhabricatorSearchNgramsDestructionEngineExtension.php',
@@ -9528,6 +9529,7 @@ phutil_register_library_map(array(
     'PhabricatorSearchIndexVersionDestructionEngineExtension' => 'PhabricatorDestructionEngineExtension',
     'PhabricatorSearchManagementIndexWorkflow' => 'PhabricatorSearchManagementWorkflow',
     'PhabricatorSearchManagementInitWorkflow' => 'PhabricatorSearchManagementWorkflow',
+    'PhabricatorSearchManagementNgramsWorkflow' => 'PhabricatorSearchManagementWorkflow',
     'PhabricatorSearchManagementWorkflow' => 'PhabricatorManagementWorkflow',
     'PhabricatorSearchNgrams' => 'PhabricatorSearchDAO',
     'PhabricatorSearchNgramsDestructionEngineExtension' => 'PhabricatorDestructionEngineExtension',
diff --git a/src/applications/search/management/PhabricatorSearchManagementNgramsWorkflow.php b/src/applications/search/management/PhabricatorSearchManagementNgramsWorkflow.php
new file mode 100644
index 0000000000..d1a6e0bdff
--- /dev/null
+++ b/src/applications/search/management/PhabricatorSearchManagementNgramsWorkflow.php
@@ -0,0 +1,106 @@
+<?php
+
+final class PhabricatorSearchManagementNgramsWorkflow
+  extends PhabricatorSearchManagementWorkflow {
+
+  protected function didConstruct() {
+    $this
+      ->setName('ngrams')
+      ->setSynopsis(pht('Recompute common ngrams.'))
+      ->setArguments(
+        array(
+          array(
+            'name' => 'reset',
+            'help' => pht('Reset all common ngram records.'),
+          ),
+        ));
+  }
+
+  public function execute(PhutilArgumentParser $args) {
+    $is_reset = $args->getArg('reset');
+
+    $all_objects = id(new PhutilClassMapQuery())
+      ->setAncestorClass('PhabricatorFerretInterface')
+      ->execute();
+
+    $min_documents = 4096;
+    $threshold = 0.15;
+
+    foreach ($all_objects as $object) {
+      $engine = $object->newFerretEngine();
+      $conn = $object->establishConnection('w');
+      $display_name = get_class($object);
+
+      if ($is_reset) {
+        echo tsprintf(
+          "%s\n",
+          pht(
+            'Resetting common ngrams for "%s".',
+            $display_name));
+
+        queryfx(
+          $conn,
+          'DELETE FROM %T',
+          $engine->getCommonNgramsTableName());
+        continue;
+      }
+
+      $document_count = queryfx_one(
+        $conn,
+        'SELECT COUNT(*) N FROM %T',
+        $engine->getDocumentTableName());
+      $document_count = $document_count['N'];
+
+      if ($document_count < $min_documents) {
+        echo tsprintf(
+          "%s\n",
+          pht(
+            'Too few documents of type "%s" for any ngrams to be common.',
+            $display_name));
+        continue;
+      }
+
+      $min_frequency = (int)ceil($document_count * $threshold);
+      $common_ngrams = queryfx_all(
+        $conn,
+        'SELECT ngram, COUNT(*) N FROM %T
+          GROUP BY ngram
+          HAVING N >= %d',
+        $engine->getNgramsTableName(),
+        $min_frequency);
+
+      if (!$common_ngrams) {
+        echo tsprintf(
+          "%s\n",
+          pht(
+            'No new common ngrams exist for "%s".',
+            $display_name));
+        continue;
+      }
+
+      $sql = array();
+      foreach ($common_ngrams as $ngram) {
+        $sql[] = qsprintf(
+          $conn,
+          '(%s, 1)',
+          $ngram['ngram']);
+      }
+
+      foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) {
+        queryfx(
+          $conn,
+          'INSERT IGNORE INTO %T (ngram, needsCollection)
+            VALUES %Q',
+          $engine->getCommonNgramsTableName(),
+          $chunk);
+      }
+
+      echo tsprintf(
+        "%s\n",
+        pht(
+          'Updated common ngrams for "%s".',
+          $display_name));
+    }
+  }
+
+}
-- 
2.51.2