From 89d038f53e8e93d4069243e024dae9538662b04e Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 10 Apr 2019 11:05:49 -0700 Subject: [PATCH] Make Portals indexable with Ferret Summary: Ref T13275. Add portals to the search index so that: - they show up in fulltext global search; and - the typeahead actually uses an index. Also make them taggable with projects as an organizational aid. Test Plan: Indexed portals with `bin/serach index`, searched for a portal with "Query", with fulltext search in main menu, with typehead on "Install Dashboard...", changed the name of a portal and searched again to check that the index updates properly. Reviewers: amckinley Reviewed By: amckinley Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam Maniphest Tasks: T13275 Differential Revision: https://secure.phabricator.com/D20389 --- .../20190410.portals.01.ferret.doc.sql | 9 ++++ .../20190410.portals.02.ferret.field.sql | 8 ++++ .../20190410.portals.03.ferret.ngrams.sql | 5 +++ .../20190410.portals.04.ferret.cngrams.sql | 7 ++++ src/__phutil_library_map__.php | 7 ++++ .../PhabricatorDashboardPortalEditor.php | 4 ++ ...PhabricatorDashboardPortalFerretEngine.php | 18 ++++++++ ...abricatorDashboardPortalFulltextEngine.php | 23 ++++++++++ .../query/PhabricatorDashboardPortalQuery.php | 10 +++-- .../storage/PhabricatorDashboardPortal.php | 21 +++++++++- .../PhabricatorDashboardPortalDatasource.php | 6 ++- .../typeahead/PhrictionDocumentDatasource.php | 42 ++++--------------- .../PhabricatorTypeaheadDatasource.php | 34 +++++++++++++++ 13 files changed, 156 insertions(+), 38 deletions(-) create mode 100644 resources/sql/autopatches/20190410.portals.01.ferret.doc.sql create mode 100644 resources/sql/autopatches/20190410.portals.02.ferret.field.sql create mode 100644 resources/sql/autopatches/20190410.portals.03.ferret.ngrams.sql create mode 100644 resources/sql/autopatches/20190410.portals.04.ferret.cngrams.sql create mode 100644 src/applications/dashboard/engine/PhabricatorDashboardPortalFerretEngine.php create mode 100644 src/applications/dashboard/engine/PhabricatorDashboardPortalFulltextEngine.php diff --git a/resources/sql/autopatches/20190410.portals.01.ferret.doc.sql b/resources/sql/autopatches/20190410.portals.01.ferret.doc.sql new file mode 100644 index 0000000000..984f3196f9 --- /dev/null +++ b/resources/sql/autopatches/20190410.portals.01.ferret.doc.sql @@ -0,0 +1,9 @@ +CREATE TABLE {$NAMESPACE}_dashboard.dashboard_portal_fdocument ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + objectPHID VARBINARY(64) NOT NULL, + isClosed BOOL NOT NULL, + authorPHID VARBINARY(64), + ownerPHID VARBINARY(64), + epochCreated INT UNSIGNED NOT NULL, + epochModified INT UNSIGNED NOT NULL +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20190410.portals.02.ferret.field.sql b/resources/sql/autopatches/20190410.portals.02.ferret.field.sql new file mode 100644 index 0000000000..af02b8f0d6 --- /dev/null +++ b/resources/sql/autopatches/20190410.portals.02.ferret.field.sql @@ -0,0 +1,8 @@ +CREATE TABLE {$NAMESPACE}_dashboard.dashboard_portal_ffield ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + documentID INT UNSIGNED NOT NULL, + fieldKey VARCHAR(4) NOT NULL COLLATE {$COLLATE_TEXT}, + rawCorpus LONGTEXT NOT NULL COLLATE {$COLLATE_SORT}, + termCorpus LONGTEXT NOT NULL COLLATE {$COLLATE_SORT}, + normalCorpus LONGTEXT NOT NULL COLLATE {$COLLATE_SORT} +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20190410.portals.03.ferret.ngrams.sql b/resources/sql/autopatches/20190410.portals.03.ferret.ngrams.sql new file mode 100644 index 0000000000..37a5fc80ab --- /dev/null +++ b/resources/sql/autopatches/20190410.portals.03.ferret.ngrams.sql @@ -0,0 +1,5 @@ +CREATE TABLE {$NAMESPACE}_dashboard.dashboard_portal_fngrams ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + documentID INT UNSIGNED NOT NULL, + ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT} +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20190410.portals.04.ferret.cngrams.sql b/resources/sql/autopatches/20190410.portals.04.ferret.cngrams.sql new file mode 100644 index 0000000000..678af664bf --- /dev/null +++ b/resources/sql/autopatches/20190410.portals.04.ferret.cngrams.sql @@ -0,0 +1,7 @@ +CREATE TABLE {$NAMESPACE}_dashboard.dashboard_portal_fngrams_common ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + ngram CHAR(3) NOT NULL COLLATE {$COLLATE_TEXT}, + needsCollection BOOL NOT NULL, + UNIQUE KEY `key_ngram` (ngram), + KEY `key_collect` (needsCollection) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index adfe9e3a9f..037344c09f 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2957,6 +2957,8 @@ phutil_register_library_map(array( 'PhabricatorDashboardPortalEditController' => 'applications/dashboard/controller/portal/PhabricatorDashboardPortalEditController.php', 'PhabricatorDashboardPortalEditEngine' => 'applications/dashboard/editor/PhabricatorDashboardPortalEditEngine.php', 'PhabricatorDashboardPortalEditor' => 'applications/dashboard/editor/PhabricatorDashboardPortalEditor.php', + 'PhabricatorDashboardPortalFerretEngine' => 'applications/dashboard/engine/PhabricatorDashboardPortalFerretEngine.php', + 'PhabricatorDashboardPortalFulltextEngine' => 'applications/dashboard/engine/PhabricatorDashboardPortalFulltextEngine.php', 'PhabricatorDashboardPortalInstallWorkflow' => 'applications/dashboard/install/PhabricatorDashboardPortalInstallWorkflow.php', 'PhabricatorDashboardPortalListController' => 'applications/dashboard/controller/portal/PhabricatorDashboardPortalListController.php', 'PhabricatorDashboardPortalMenuItem' => 'applications/dashboard/menuitem/PhabricatorDashboardPortalMenuItem.php', @@ -8938,6 +8940,9 @@ phutil_register_library_map(array( 'PhabricatorApplicationTransactionInterface', 'PhabricatorPolicyInterface', 'PhabricatorDestructibleInterface', + 'PhabricatorProjectInterface', + 'PhabricatorFulltextInterface', + 'PhabricatorFerretInterface', ), 'PhabricatorDashboardPortalController' => 'PhabricatorDashboardController', 'PhabricatorDashboardPortalDatasource' => 'PhabricatorTypeaheadDatasource', @@ -8945,6 +8950,8 @@ phutil_register_library_map(array( 'PhabricatorDashboardPortalEditController' => 'PhabricatorDashboardPortalController', 'PhabricatorDashboardPortalEditEngine' => 'PhabricatorEditEngine', 'PhabricatorDashboardPortalEditor' => 'PhabricatorApplicationTransactionEditor', + 'PhabricatorDashboardPortalFerretEngine' => 'PhabricatorFerretEngine', + 'PhabricatorDashboardPortalFulltextEngine' => 'PhabricatorFulltextEngine', 'PhabricatorDashboardPortalInstallWorkflow' => 'PhabricatorDashboardObjectInstallWorkflow', 'PhabricatorDashboardPortalListController' => 'PhabricatorDashboardPortalController', 'PhabricatorDashboardPortalMenuItem' => 'PhabricatorProfileMenuItem', diff --git a/src/applications/dashboard/editor/PhabricatorDashboardPortalEditor.php b/src/applications/dashboard/editor/PhabricatorDashboardPortalEditor.php index 9989d1e7d5..2c8a3ccfac 100644 --- a/src/applications/dashboard/editor/PhabricatorDashboardPortalEditor.php +++ b/src/applications/dashboard/editor/PhabricatorDashboardPortalEditor.php @@ -28,4 +28,8 @@ final class PhabricatorDashboardPortalEditor return $types; } + protected function supportsSearch() { + return true; + } + } diff --git a/src/applications/dashboard/engine/PhabricatorDashboardPortalFerretEngine.php b/src/applications/dashboard/engine/PhabricatorDashboardPortalFerretEngine.php new file mode 100644 index 0000000000..b6cbe857fc --- /dev/null +++ b/src/applications/dashboard/engine/PhabricatorDashboardPortalFerretEngine.php @@ -0,0 +1,18 @@ +setDocumentTitle($portal->getName()); + + $document->addRelationship( + $portal->isArchived() + ? PhabricatorSearchRelationship::RELATIONSHIP_CLOSED + : PhabricatorSearchRelationship::RELATIONSHIP_OPEN, + $portal->getPHID(), + PhabricatorDashboardPortalPHIDType::TYPECONST, + PhabricatorTime::getNow()); + } + +} diff --git a/src/applications/dashboard/query/PhabricatorDashboardPortalQuery.php b/src/applications/dashboard/query/PhabricatorDashboardPortalQuery.php index d352b99c8f..857c4dc215 100644 --- a/src/applications/dashboard/query/PhabricatorDashboardPortalQuery.php +++ b/src/applications/dashboard/query/PhabricatorDashboardPortalQuery.php @@ -36,21 +36,21 @@ final class PhabricatorDashboardPortalQuery if ($this->ids !== null) { $where[] = qsprintf( $conn, - 'id IN (%Ld)', + 'portal.id IN (%Ld)', $this->ids); } if ($this->phids !== null) { $where[] = qsprintf( $conn, - 'phid IN (%Ls)', + 'portal.phid IN (%Ls)', $this->phids); } if ($this->statuses !== null) { $where[] = qsprintf( $conn, - 'status IN (%Ls)', + 'portal.status IN (%Ls)', $this->statuses); } @@ -61,4 +61,8 @@ final class PhabricatorDashboardPortalQuery return 'PhabricatorDashboardApplication'; } + protected function getPrimaryTableAlias() { + return 'portal'; + } + } diff --git a/src/applications/dashboard/storage/PhabricatorDashboardPortal.php b/src/applications/dashboard/storage/PhabricatorDashboardPortal.php index 1930a23bd4..d04f8282fb 100644 --- a/src/applications/dashboard/storage/PhabricatorDashboardPortal.php +++ b/src/applications/dashboard/storage/PhabricatorDashboardPortal.php @@ -5,7 +5,10 @@ final class PhabricatorDashboardPortal implements PhabricatorApplicationTransactionInterface, PhabricatorPolicyInterface, - PhabricatorDestructibleInterface { + PhabricatorDestructibleInterface, + PhabricatorProjectInterface, + PhabricatorFulltextInterface, + PhabricatorFerretInterface { protected $name; protected $viewPolicy; @@ -55,6 +58,11 @@ final class PhabricatorDashboardPortal return '/portal/view/'.$this->getID().'/'; } + public function isArchived() { + $status_archived = PhabricatorDashboardPortalStatus::STATUS_ARCHIVED; + return ($this->getStatus() === $status_archived); + } + /* -( PhabricatorApplicationTransactionInterface )------------------------- */ @@ -100,5 +108,16 @@ final class PhabricatorDashboardPortal $this->delete(); } +/* -( PhabricatorFulltextInterface )--------------------------------------- */ + + public function newFulltextEngine() { + return new PhabricatorDashboardPortalFulltextEngine(); + } + +/* -( PhabricatorFerretInterface )----------------------------------------- */ + + public function newFerretEngine() { + return new PhabricatorDashboardPortalFerretEngine(); + } } diff --git a/src/applications/dashboard/typeahead/PhabricatorDashboardPortalDatasource.php b/src/applications/dashboard/typeahead/PhabricatorDashboardPortalDatasource.php index 898f9efa2f..008bb542ab 100644 --- a/src/applications/dashboard/typeahead/PhabricatorDashboardPortalDatasource.php +++ b/src/applications/dashboard/typeahead/PhabricatorDashboardPortalDatasource.php @@ -27,7 +27,11 @@ final class PhabricatorDashboardPortalDatasource public function buildResults() { $query = new PhabricatorDashboardPortalQuery(); - // TODO: Actually query by name so this scales past 100 portals. + $this->applyFerretConstraints( + $query, + id(new PhabricatorDashboardPortal())->newFerretEngine(), + 'title', + $this->getRawQuery()); $portals = $this->executeQuery($query); diff --git a/src/applications/phriction/typeahead/PhrictionDocumentDatasource.php b/src/applications/phriction/typeahead/PhrictionDocumentDatasource.php index bb9608edb8..09056fe761 100644 --- a/src/applications/phriction/typeahead/PhrictionDocumentDatasource.php +++ b/src/applications/phriction/typeahead/PhrictionDocumentDatasource.php @@ -18,41 +18,17 @@ final class PhrictionDocumentDatasource public function loadResults() { $viewer = $this->getViewer(); - $raw_query = $this->getRawQuery(); - - $engine = id(new PhrictionDocument()) - ->newFerretEngine(); - - $compiler = id(new PhutilSearchQueryCompiler()) - ->setEnableFunctions(true); - - $raw_tokens = $compiler->newTokens($raw_query); - - $fulltext_tokens = array(); - foreach ($raw_tokens as $raw_token) { - - // This is a little hacky and could maybe be cleaner. We're treating - // every search term as though the user had entered "title:dog" insead - // of "dog". - - $alternate_token = PhutilSearchQueryToken::newFromDictionary( - array( - 'quoted' => $raw_token->isQuoted(), - 'value' => $raw_token->getValue(), - 'operator' => PhutilSearchQueryCompiler::OPERATOR_SUBSTRING, - 'function' => 'title', - )); + $query = id(new PhrictionDocumentQuery()) + ->setViewer($viewer) + ->needContent(true); - $fulltext_token = id(new PhabricatorFulltextToken()) - ->setToken($alternate_token); - $fulltext_tokens[] = $fulltext_token; - } + $this->applyFerretConstraints( + $query, + id(new PhrictionDocument())->newFerretEngine(), + 'title', + $this->getRawQuery()); - $documents = id(new PhrictionDocumentQuery()) - ->setViewer($viewer) - ->withFerretConstraint($engine, $fulltext_tokens) - ->needContent(true) - ->execute(); + $documents = $query->execute(); $results = array(); foreach ($documents as $document) { diff --git a/src/applications/typeahead/datasource/PhabricatorTypeaheadDatasource.php b/src/applications/typeahead/datasource/PhabricatorTypeaheadDatasource.php index e077d7a7ec..29a8d4b7ba 100644 --- a/src/applications/typeahead/datasource/PhabricatorTypeaheadDatasource.php +++ b/src/applications/typeahead/datasource/PhabricatorTypeaheadDatasource.php @@ -604,4 +604,38 @@ abstract class PhabricatorTypeaheadDatasource extends Phobject { return mpull($tokens, 'getWireFormat', 'getPHID'); } + final protected function applyFerretConstraints( + PhabricatorCursorPagedPolicyAwareQuery $query, + PhabricatorFerretEngine $engine, + $ferret_function, + $raw_query) { + + $compiler = id(new PhutilSearchQueryCompiler()) + ->setEnableFunctions(true); + + $raw_tokens = $compiler->newTokens($raw_query); + + $fulltext_tokens = array(); + foreach ($raw_tokens as $raw_token) { + // This is a little hacky and could maybe be cleaner. We're treating + // every search term as though the user had entered "title:dog" instead + // of "dog". + + $alternate_token = PhutilSearchQueryToken::newFromDictionary( + array( + 'quoted' => $raw_token->isQuoted(), + 'value' => $raw_token->getValue(), + 'operator' => PhutilSearchQueryCompiler::OPERATOR_SUBSTRING, + 'function' => $ferret_function, + )); + + $fulltext_token = id(new PhabricatorFulltextToken()) + ->setToken($alternate_token); + $fulltext_tokens[] = $fulltext_token; + } + + $query->withFerretConstraint($engine, $fulltext_tokens); + } + + } -- 2.51.2