From 3f4cc3ad6eb4628ad5bdf5505f04ef1134ef1089 Mon Sep 17 00:00:00 2001 From: epriestley Date: Tue, 08 Mar 2016 13:59:24 +0000 Subject: [PATCH] Allow Nuances sources to provide import cursors Summary: Ref T10537. Some sources (like the future "GitHub Repository" source) need to poll remotes. - Provide a mechanism for sources to emit import cursors. - Hook them into the trigger daemon so they'll fire periodically. - Provide some storage. This diff does nothing useful or interesting, and is pure infrastructure. Test Plan: - Ran `bin/storage upgrade -f`, no adjustment issues. - Poked around Nuance. - Ran the trigger daemon, verified it didn't crash and checked for Nuance stuff to do. Reviewers: chad Reviewed By: chad Maniphest Tasks: T10537 Differential Revision: https://secure.phabricator.com/D15435 --- src/__phutil_library_map__.php | 8 ++++++++ resources/sql/autopatches/20160308.nuance.01.disabled.sql | 2 ++ resources/sql/autopatches/20160308.nuance.02.cursordata.sql | 12 ++++++++++++ src/applications/nuance/cursor/NuanceImportCursor.php | 10 ++++++++++ src/applications/nuance/phid/NuanceImportCursorPHIDType.php | 38 ++++++++++++++++++++++++++++++++++++++ src/applications/nuance/query/NuanceImportCursorDataQuery.php | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/applications/nuance/query/NuanceSourceQuery.php | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/applications/nuance/source/NuanceSourceDefinition.php | 17 +++++++++++++++++ src/applications/nuance/storage/NuanceImportCursorData.php | 35 +++++++++++++++++++++++++++++++++++ src/applications/nuance/storage/NuanceSource.php | 5 ++++- src/infrastructure/daemon/workers/PhabricatorTriggerDaemon.php | 71 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 11 file(s) changed, 307 insertion(s)(+), 1 deletion(s)(-) diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -1421,6 +1421,10 @@ 'NuanceController' => 'applications/nuance/controller/NuanceController.php', 'NuanceCreateItemConduitAPIMethod' => 'applications/nuance/conduit/NuanceCreateItemConduitAPIMethod.php', 'NuanceDAO' => 'applications/nuance/storage/NuanceDAO.php', + 'NuanceImportCursor' => 'applications/nuance/cursor/NuanceImportCursor.php', + 'NuanceImportCursorData' => 'applications/nuance/storage/NuanceImportCursorData.php', + 'NuanceImportCursorDataQuery' => 'applications/nuance/query/NuanceImportCursorDataQuery.php', + 'NuanceImportCursorPHIDType' => 'applications/nuance/phid/NuanceImportCursorPHIDType.php', 'NuanceItem' => 'applications/nuance/storage/NuanceItem.php', 'NuanceItemEditController' => 'applications/nuance/controller/NuanceItemEditController.php', 'NuanceItemEditor' => 'applications/nuance/editor/NuanceItemEditor.php', @@ -5661,6 +5665,10 @@ 'NuanceController' => 'PhabricatorController', 'NuanceCreateItemConduitAPIMethod' => 'NuanceConduitAPIMethod', 'NuanceDAO' => 'PhabricatorLiskDAO', + 'NuanceImportCursor' => 'Phobject', + 'NuanceImportCursorData' => 'NuanceDAO', + 'NuanceImportCursorDataQuery' => 'NuanceQuery', + 'NuanceImportCursorPHIDType' => 'PhabricatorPHIDType', 'NuanceItem' => array( 'NuanceDAO', 'PhabricatorPolicyInterface', diff --git a/resources/sql/autopatches/20160308.nuance.01.disabled.sql b/resources/sql/autopatches/20160308.nuance.01.disabled.sql new file mode 100644 --- /dev/null +++ b/resources/sql/autopatches/20160308.nuance.01.disabled.sql @@ -0,0 +1,2 @@ +ALTER TABLE {$NAMESPACE}_nuance.nuance_source + ADD isDisabled BOOL NOT NULL; diff --git a/resources/sql/autopatches/20160308.nuance.02.cursordata.sql b/resources/sql/autopatches/20160308.nuance.02.cursordata.sql new file mode 100644 --- /dev/null +++ b/resources/sql/autopatches/20160308.nuance.02.cursordata.sql @@ -0,0 +1,12 @@ +CREATE TABLE {$NAMESPACE}_nuance.nuance_importcursordata ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + phid VARBINARY(64) NOT NULL, + sourcePHID VARBINARY(64) NOT NULL, + cursorKey VARCHAR(32) NOT NULL COLLATE {$COLLATE_TEXT}, + cursorType VARCHAR(32) NOT NULL COLLATE {$COLLATE_TEXT}, + properties LONGTEXT NOT NULL COLLATE {$COLLATE_TEXT}, + dateCreated INT UNSIGNED NOT NULL, + dateModified INT UNSIGNED NOT NULL, + UNIQUE KEY `key_phid` (phid), + UNIQUE KEY `key_source` (sourcePHID, cursorKey) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/src/applications/nuance/cursor/NuanceImportCursor.php b/src/applications/nuance/cursor/NuanceImportCursor.php new file mode 100644 --- /dev/null +++ b/src/applications/nuance/cursor/NuanceImportCursor.php @@ -0,0 +1,10 @@ +withPHIDs($phids); + } + + public function loadHandles( + PhabricatorHandleQuery $query, + array $handles, + array $objects) { + + $viewer = $query->getViewer(); + foreach ($handles as $phid => $handle) { + $item = $objects[$phid]; + } + } + +} diff --git a/src/applications/nuance/query/NuanceImportCursorDataQuery.php b/src/applications/nuance/query/NuanceImportCursorDataQuery.php new file mode 100644 --- /dev/null +++ b/src/applications/nuance/query/NuanceImportCursorDataQuery.php @@ -0,0 +1,60 @@ +ids = $ids; + return $this; + } + + public function withPHIDs(array $phids) { + $this->phids = $phids; + return $this; + } + + public function withSourcePHIDs(array $source_phids) { + $this->sourcePHIDs = $source_phids; + return $this; + } + + public function newResultObject() { + return new NuanceImportCursorData(); + } + + protected function loadPage() { + return $this->loadStandardPage($this->newResultObject()); + } + + protected function buildWhereClauseParts(AphrontDatabaseConnection $conn) { + $where = parent::buildWhereClauseParts($conn); + + if ($this->sourcePHIDs !== null) { + $where[] = qsprintf( + $conn, + 'sourcePHID IN (%Ls)', + $this->sourcePHIDs); + } + + if ($this->ids !== null) { + $where[] = qsprintf( + $conn, + 'id IN (%Ld)', + $this->ids); + } + + if ($this->phids !== null) { + $where[] = qsprintf( + $conn, + 'phid IN (%Ls)', + $this->phids); + } + + return $where; + } + +} diff --git a/src/applications/nuance/query/NuanceSourceQuery.php b/src/applications/nuance/query/NuanceSourceQuery.php --- a/src/applications/nuance/query/NuanceSourceQuery.php +++ b/src/applications/nuance/query/NuanceSourceQuery.php @@ -6,6 +6,8 @@ private $ids; private $phids; private $types; + private $isDisabled; + private $hasCursors; public function withIDs(array $ids) { $this->ids = $ids; @@ -19,6 +21,16 @@ public function withTypes($types) { $this->types = $types; + return $this; + } + + public function withIsDisabled($disabled) { + $this->isDisabled = $disabled; + return $this; + } + + public function withHasImportCursors($has_cursors) { + $this->hasCursors = $has_cursors; return $this; } @@ -69,6 +81,44 @@ $conn, 'phid IN (%Ls)', $this->phids); + } + + if ($this->isDisabled !== null) { + $where[] = qsprintf( + $conn, + 'isDisabled = %d', + (int)$this->isDisabled); + } + + if ($this->hasCursors !== null) { + $cursor_types = array(); + + $definitions = NuanceSourceDefinition::getAllDefinitions(); + foreach ($definitions as $key => $definition) { + if ($definition->hasImportCursors()) { + $cursor_types[] = $key; + } + } + + if ($this->hasCursors) { + if (!$cursor_types) { + throw new PhabricatorEmptyQueryException(); + } else { + $where[] = qsprintf( + $conn, + 'type IN (%Ls)', + $cursor_types); + } + } else { + if (!$cursor_types) { + // Apply no constraint. + } else { + $where[] = qsprintf( + $conn, + 'type NOT IN (%Ls)', + $cursor_types); + } + } } return $where; diff --git a/src/applications/nuance/source/NuanceSourceDefinition.php b/src/applications/nuance/source/NuanceSourceDefinition.php --- a/src/applications/nuance/source/NuanceSourceDefinition.php +++ b/src/applications/nuance/source/NuanceSourceDefinition.php @@ -43,6 +43,23 @@ ->execute(); } + public function hasImportCursors() { + return false; + } + + final public function getImportCursors() { + if (!$this->hasImportCursors()) { + throw new Exception( + pht('This source has no input cursors.')); + } + + return $this->newImportCursors(); + } + + protected function newImportCursors() { + throw new PhutilMethodNotImplementedException(); + } + /** * A human readable string like "Twitter" or "Phabricator Form". */ diff --git a/src/applications/nuance/storage/NuanceImportCursorData.php b/src/applications/nuance/storage/NuanceImportCursorData.php new file mode 100644 --- /dev/null +++ b/src/applications/nuance/storage/NuanceImportCursorData.php @@ -0,0 +1,35 @@ + true, + self::CONFIG_SERIALIZATION => array( + 'properties' => self::SERIALIZATION_JSON, + ), + self::CONFIG_COLUMN_SCHEMA => array( + 'cursorType' => 'text32', + 'cursorKey' => 'text32', + ), + self::CONFIG_KEY_SCHEMA => array( + 'key_source' => array( + 'columns' => array('sourcePHID', 'cursorKey'), + 'unique' => true, + ), + ), + ) + parent::getConfiguration(); + } + + public function generatePHID() { + return PhabricatorPHID::generateNewPHID( + NuanceImportCursorPHIDType::TYPECONST); + } + +} diff --git a/src/applications/nuance/storage/NuanceSource.php b/src/applications/nuance/storage/NuanceSource.php --- a/src/applications/nuance/storage/NuanceSource.php +++ b/src/applications/nuance/storage/NuanceSource.php @@ -12,6 +12,7 @@ protected $viewPolicy; protected $editPolicy; protected $defaultQueuePHID; + protected $isDisabled; private $definition = self::ATTACHABLE; @@ -25,6 +26,7 @@ 'name' => 'text255?', 'type' => 'text32', 'mailKey' => 'bytes20', + 'isDisabled' => 'bool', ), self::CONFIG_KEY_SCHEMA => array( 'key_type' => array( @@ -66,7 +68,8 @@ ->setViewPolicy($view_policy) ->setEditPolicy($edit_policy) ->setType($definition->getSourceTypeConstant()) - ->attachDefinition($definition); + ->attachDefinition($definition) + ->setIsDisabled(0); } public function getDefinition() { diff --git a/src/infrastructure/daemon/workers/PhabricatorTriggerDaemon.php b/src/infrastructure/daemon/workers/PhabricatorTriggerDaemon.php --- a/src/infrastructure/daemon/workers/PhabricatorTriggerDaemon.php +++ b/src/infrastructure/daemon/workers/PhabricatorTriggerDaemon.php @@ -16,6 +16,10 @@ private $garbageCollectors; private $nextCollection; + private $anyNuanceData; + private $nuanceSources; + private $nuanceCursors; + protected function run() { // The trigger daemon is a low-level infrastructure daemon which schedules @@ -99,6 +103,7 @@ $lock->unlock(); $sleep_duration = $this->getSleepDuration(); + $sleep_duration = $this->runNuanceImportCursors($sleep_duration); $sleep_duration = $this->runGarbageCollection($sleep_duration); $this->sleep($sleep_duration); } while (!$this->shouldExit()); @@ -377,6 +382,72 @@ } return false; + } + + +/* -( Nuance Importers )--------------------------------------------------- */ + + + private function runNuanceImportCursors($duration) { + $run_until = (PhabricatorTime::getNow() + $duration); + + do { + $more_data = $this->updateNuanceImportCursors(); + if (!$more_data) { + break; + } + } while (PhabricatorTime::getNow() <= $run_until); + + $remaining = max(0, $run_until - PhabricatorTime::getNow()); + + return $remaining; + } + + + private function updateNuanceImportCursors() { + $nuance_app = 'PhabricatorNuanceApplication'; + if (!PhabricatorApplication::isClassInstalled($nuance_app)) { + return false; + } + + // If we haven't loaded sources yet, load them first. + if (!$this->nuanceSources) { + $this->anyNuanceData = false; + + $sources = id(new NuanceSourceQuery()) + ->setViewer($this->getViewer()) + ->withIsDisabled(false) + ->withHasImportCursors(true) + ->execute(); + if (!$sources) { + return false; + } + + $this->nuanceSources = array_reverse($sources); + } + + // If we don't have any cursors, move to the next source and generate its + // cursors. + if (!$this->nuanceCursors) { + $source = array_pop($this->nuanceSources); + $cursors = $source->getImportCursors(); + $this->nuanceCursors = array_reverse($cursors); + } + + // Update the next cursor. + $cursor = array_pop($this->nuanceCursors); + if ($cursor) { + $more_data = $cursor->importFromSource(); + if ($more_data) { + $this->anyNuanceData = true; + } + } + + if (!$this->nuanceSources && !$this->nuanceCursors) { + return $this->anyNuanceData; + } + + return true; } } -- tangled.sh