diff --git a/resources/celerity/map.php b/resources/celerity/map.php index 6bc17f938e..ee119787c5 100644 --- a/resources/celerity/map.php +++ b/resources/celerity/map.php @@ -55,6 +55,7 @@ return array( 'rsrc/css/application/conpherence/widget-pane.css' => '2af42ebe', 'rsrc/css/application/contentsource/content-source-view.css' => '4b8b05d4', 'rsrc/css/application/countdown/timer.css' => '86b7b0a0', + 'rsrc/css/application/daemon/bulk-job.css' => 'df9c1d4a', 'rsrc/css/application/dashboard/dashboard.css' => '17937d22', 'rsrc/css/application/diff/inline-comment-summary.css' => '51efda3a', 'rsrc/css/application/differential/add-comment.css' => 'c47f8c40', @@ -343,6 +344,7 @@ return array( 'rsrc/js/application/conpherence/behavior-quicksand-blacklist.js' => '7927a7d3', 'rsrc/js/application/conpherence/behavior-widget-pane.js' => '93568464', 'rsrc/js/application/countdown/timer.js' => 'e4cc26b3', + 'rsrc/js/application/daemon/behavior-bulk-job-reload.js' => 'edf8a145', 'rsrc/js/application/dashboard/behavior-dashboard-async-panel.js' => '469c0d9e', 'rsrc/js/application/dashboard/behavior-dashboard-move-panels.js' => '82439934', 'rsrc/js/application/dashboard/behavior-dashboard-query-panel-select.js' => '453c5375', @@ -495,6 +497,7 @@ return array( 'aphront-two-column-view-css' => '16ab3ad2', 'aphront-typeahead-control-css' => '0e403212', 'auth-css' => '44975d4b', + 'bulk-job-css' => 'df9c1d4a', 'calendar-icon-css' => '98ce946d', 'changeset-view-manager' => '58562350', 'conduit-api-css' => '7bc725c4', @@ -541,6 +544,7 @@ return array( 'javelin-behavior-aphront-more' => 'a80d0378', 'javelin-behavior-audio-source' => '59b251eb', 'javelin-behavior-audit-preview' => 'd835b03a', + 'javelin-behavior-bulk-job-reload' => 'edf8a145', 'javelin-behavior-choose-control' => '6153c708', 'javelin-behavior-config-reorder-fields' => 'b6993408', 'javelin-behavior-conpherence-drag-and-drop-photo' => 'cf86d16a', @@ -1925,6 +1929,10 @@ return array( 'javelin-uri', 'phabricator-notification', ), + 'edf8a145' => array( + 'javelin-behavior', + 'javelin-uri', + ), 'eeaa9e5a' => array( 'javelin-behavior', 'javelin-stratcom', diff --git a/resources/sql/autopatches/20150622.bulk.1.job.sql b/resources/sql/autopatches/20150622.bulk.1.job.sql new file mode 100644 index 0000000000..a0cdb5d678 --- /dev/null +++ b/resources/sql/autopatches/20150622.bulk.1.job.sql @@ -0,0 +1,15 @@ +CREATE TABLE {$NAMESPACE}_worker.worker_bulkjob ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + phid VARBINARY(64) NOT NULL, + authorPHID VARBINARY(64) NOT NULL, + jobTypeKey VARCHAR(32) COLLATE {$COLLATE_TEXT} NOT NULL, + status VARCHAR(32) COLLATE {$COLLATE_TEXT} NOT NULL, + parameters LONGTEXT COLLATE {$COLLATE_TEXT} NOT NULL, + size INT UNSIGNED NOT NULL, + dateCreated INT UNSIGNED NOT NULL, + dateModified INT UNSIGNED NOT NULL, + UNIQUE KEY `key_phid` (phid), + KEY `key_type` (jobTypeKey), + KEY `key_author` (authorPHID), + KEY `key_status` (status) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20150622.bulk.2.task.sql b/resources/sql/autopatches/20150622.bulk.2.task.sql new file mode 100644 index 0000000000..f98c205180 --- /dev/null +++ b/resources/sql/autopatches/20150622.bulk.2.task.sql @@ -0,0 +1,9 @@ +CREATE TABLE {$NAMESPACE}_worker.worker_bulktask ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + bulkJobPHID VARBINARY(64) NOT NULL, + objectPHID VARBINARY(64) NOT NULL, + status VARCHAR(32) COLLATE {$COLLATE_TEXT} NOT NULL, + data LONGTEXT COLLATE {$COLLATE_TEXT} NOT NULL, + KEY `key_job` (bulkJobPHID, status), + KEY `key_object` (objectPHID) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20150622.bulk.3.xaction.sql b/resources/sql/autopatches/20150622.bulk.3.xaction.sql new file mode 100644 index 0000000000..27aa2d5caf --- /dev/null +++ b/resources/sql/autopatches/20150622.bulk.3.xaction.sql @@ -0,0 +1,19 @@ +CREATE TABLE {$NAMESPACE}_worker.worker_bulkjobtransaction ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + phid VARBINARY(64) NOT NULL, + authorPHID VARBINARY(64) NOT NULL, + objectPHID VARBINARY(64) NOT NULL, + viewPolicy VARBINARY(64) NOT NULL, + editPolicy VARBINARY(64) NOT NULL, + commentPHID VARBINARY(64) DEFAULT NULL, + commentVersion INT UNSIGNED NOT NULL, + transactionType VARCHAR(32) COLLATE {$COLLATE_TEXT} NOT NULL, + oldValue LONGTEXT COLLATE {$COLLATE_TEXT} NOT NULL, + newValue LONGTEXT COLLATE {$COLLATE_TEXT} NOT NULL, + contentSource LONGTEXT COLLATE {$COLLATE_TEXT} NOT NULL, + metadata LONGTEXT COLLATE {$COLLATE_TEXT} NOT NULL, + dateCreated INT UNSIGNED NOT NULL, + dateModified INT UNSIGNED NOT NULL, + UNIQUE KEY `key_phid` (`phid`), + KEY `key_object` (`objectPHID`) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20150622.bulk.4.edge.sql b/resources/sql/autopatches/20150622.bulk.4.edge.sql new file mode 100644 index 0000000000..3d81a1bcf6 --- /dev/null +++ b/resources/sql/autopatches/20150622.bulk.4.edge.sql @@ -0,0 +1,16 @@ +CREATE TABLE {$NAMESPACE}_worker.edge ( + src VARBINARY(64) NOT NULL, + type INT UNSIGNED NOT NULL, + dst VARBINARY(64) NOT NULL, + dateCreated INT UNSIGNED NOT NULL, + seq INT UNSIGNED NOT NULL, + dataID INT UNSIGNED, + PRIMARY KEY (src, type, dst), + KEY `src` (src, type, dateCreated, seq), + UNIQUE KEY `key_dst` (dst, type, src) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; + +CREATE TABLE {$NAMESPACE}_worker.edgedata ( + id INT UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT, + data LONGTEXT NOT NULL COLLATE {$COLLATE_TEXT} +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 2b06ac7d6e..72a99b56df 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -1082,6 +1082,7 @@ phutil_register_library_map(array( 'ManiphestTaskDependedOnByTaskEdgeType' => 'applications/maniphest/edge/ManiphestTaskDependedOnByTaskEdgeType.php', 'ManiphestTaskDependsOnTaskEdgeType' => 'applications/maniphest/edge/ManiphestTaskDependsOnTaskEdgeType.php', 'ManiphestTaskDetailController' => 'applications/maniphest/controller/ManiphestTaskDetailController.php', + 'ManiphestTaskEditBulkJobType' => 'applications/maniphest/bulk/ManiphestTaskEditBulkJobType.php', 'ManiphestTaskEditController' => 'applications/maniphest/controller/ManiphestTaskEditController.php', 'ManiphestTaskHasCommitEdgeType' => 'applications/maniphest/edge/ManiphestTaskHasCommitEdgeType.php', 'ManiphestTaskHasMockEdgeType' => 'applications/maniphest/edge/ManiphestTaskHasMockEdgeType.php', @@ -1708,6 +1709,9 @@ phutil_register_library_map(array( 'PhabricatorCustomFieldStringIndexStorage' => 'infrastructure/customfield/storage/PhabricatorCustomFieldStringIndexStorage.php', 'PhabricatorCustomHeaderConfigType' => 'applications/config/custom/PhabricatorCustomHeaderConfigType.php', 'PhabricatorDaemon' => 'infrastructure/daemon/PhabricatorDaemon.php', + 'PhabricatorDaemonBulkJobListController' => 'applications/daemon/controller/PhabricatorDaemonBulkJobListController.php', + 'PhabricatorDaemonBulkJobMonitorController' => 'applications/daemon/controller/PhabricatorDaemonBulkJobMonitorController.php', + 'PhabricatorDaemonBulkJobViewController' => 'applications/daemon/controller/PhabricatorDaemonBulkJobViewController.php', 'PhabricatorDaemonConsoleController' => 'applications/daemon/controller/PhabricatorDaemonConsoleController.php', 'PhabricatorDaemonController' => 'applications/daemon/controller/PhabricatorDaemonController.php', 'PhabricatorDaemonDAO' => 'applications/daemon/storage/PhabricatorDaemonDAO.php', @@ -2820,6 +2824,19 @@ phutil_register_library_map(array( 'PhabricatorWorkerActiveTask' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerActiveTask.php', 'PhabricatorWorkerArchiveTask' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerArchiveTask.php', 'PhabricatorWorkerArchiveTaskQuery' => 'infrastructure/daemon/workers/query/PhabricatorWorkerArchiveTaskQuery.php', + 'PhabricatorWorkerBulkJob' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerBulkJob.php', + 'PhabricatorWorkerBulkJobCreateWorker' => 'infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobCreateWorker.php', + 'PhabricatorWorkerBulkJobEditor' => 'infrastructure/daemon/workers/editor/PhabricatorWorkerBulkJobEditor.php', + 'PhabricatorWorkerBulkJobPHIDType' => 'infrastructure/daemon/workers/phid/PhabricatorWorkerBulkJobPHIDType.php', + 'PhabricatorWorkerBulkJobQuery' => 'infrastructure/daemon/workers/query/PhabricatorWorkerBulkJobQuery.php', + 'PhabricatorWorkerBulkJobSearchEngine' => 'infrastructure/daemon/workers/query/PhabricatorWorkerBulkJobSearchEngine.php', + 'PhabricatorWorkerBulkJobTaskWorker' => 'infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobTaskWorker.php', + 'PhabricatorWorkerBulkJobTestCase' => 'infrastructure/daemon/workers/__tests__/PhabricatorWorkerBulkJobTestCase.php', + 'PhabricatorWorkerBulkJobTransaction' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerBulkJobTransaction.php', + 'PhabricatorWorkerBulkJobTransactionQuery' => 'infrastructure/daemon/workers/query/PhabricatorWorkerBulkJobTransactionQuery.php', + 'PhabricatorWorkerBulkJobType' => 'infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobType.php', + 'PhabricatorWorkerBulkJobWorker' => 'infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobWorker.php', + 'PhabricatorWorkerBulkTask' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerBulkTask.php', 'PhabricatorWorkerDAO' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerDAO.php', 'PhabricatorWorkerLeaseQuery' => 'infrastructure/daemon/workers/query/PhabricatorWorkerLeaseQuery.php', 'PhabricatorWorkerManagementCancelWorkflow' => 'infrastructure/daemon/workers/management/PhabricatorWorkerManagementCancelWorkflow.php', @@ -2829,6 +2846,7 @@ phutil_register_library_map(array( 'PhabricatorWorkerManagementRetryWorkflow' => 'infrastructure/daemon/workers/management/PhabricatorWorkerManagementRetryWorkflow.php', 'PhabricatorWorkerManagementWorkflow' => 'infrastructure/daemon/workers/management/PhabricatorWorkerManagementWorkflow.php', 'PhabricatorWorkerPermanentFailureException' => 'infrastructure/daemon/workers/exception/PhabricatorWorkerPermanentFailureException.php', + 'PhabricatorWorkerSchemaSpec' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerSchemaSpec.php', 'PhabricatorWorkerTask' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerTask.php', 'PhabricatorWorkerTaskData' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerTaskData.php', 'PhabricatorWorkerTaskDetailController' => 'applications/daemon/controller/PhabricatorWorkerTaskDetailController.php', @@ -4589,6 +4607,7 @@ phutil_register_library_map(array( 'ManiphestTaskDependedOnByTaskEdgeType' => 'PhabricatorEdgeType', 'ManiphestTaskDependsOnTaskEdgeType' => 'PhabricatorEdgeType', 'ManiphestTaskDetailController' => 'ManiphestController', + 'ManiphestTaskEditBulkJobType' => 'PhabricatorWorkerBulkJobType', 'ManiphestTaskEditController' => 'ManiphestController', 'ManiphestTaskHasCommitEdgeType' => 'PhabricatorEdgeType', 'ManiphestTaskHasMockEdgeType' => 'PhabricatorEdgeType', @@ -5300,6 +5319,9 @@ phutil_register_library_map(array( 'PhabricatorCustomFieldStringIndexStorage' => 'PhabricatorCustomFieldIndexStorage', 'PhabricatorCustomHeaderConfigType' => 'PhabricatorConfigOptionType', 'PhabricatorDaemon' => 'PhutilDaemon', + 'PhabricatorDaemonBulkJobListController' => 'PhabricatorDaemonController', + 'PhabricatorDaemonBulkJobMonitorController' => 'PhabricatorDaemonController', + 'PhabricatorDaemonBulkJobViewController' => 'PhabricatorDaemonController', 'PhabricatorDaemonConsoleController' => 'PhabricatorDaemonController', 'PhabricatorDaemonController' => 'PhabricatorController', 'PhabricatorDaemonDAO' => 'PhabricatorLiskDAO', @@ -6603,6 +6625,25 @@ phutil_register_library_map(array( 'PhabricatorWorkerActiveTask' => 'PhabricatorWorkerTask', 'PhabricatorWorkerArchiveTask' => 'PhabricatorWorkerTask', 'PhabricatorWorkerArchiveTaskQuery' => 'PhabricatorQuery', + 'PhabricatorWorkerBulkJob' => array( + 'PhabricatorWorkerDAO', + 'PhabricatorPolicyInterface', + 'PhabricatorSubscribableInterface', + 'PhabricatorApplicationTransactionInterface', + 'PhabricatorDestructibleInterface', + ), + 'PhabricatorWorkerBulkJobCreateWorker' => 'PhabricatorWorkerBulkJobWorker', + 'PhabricatorWorkerBulkJobEditor' => 'PhabricatorApplicationTransactionEditor', + 'PhabricatorWorkerBulkJobPHIDType' => 'PhabricatorPHIDType', + 'PhabricatorWorkerBulkJobQuery' => 'PhabricatorCursorPagedPolicyAwareQuery', + 'PhabricatorWorkerBulkJobSearchEngine' => 'PhabricatorApplicationSearchEngine', + 'PhabricatorWorkerBulkJobTaskWorker' => 'PhabricatorWorkerBulkJobWorker', + 'PhabricatorWorkerBulkJobTestCase' => 'PhabricatorTestCase', + 'PhabricatorWorkerBulkJobTransaction' => 'PhabricatorApplicationTransaction', + 'PhabricatorWorkerBulkJobTransactionQuery' => 'PhabricatorApplicationTransactionQuery', + 'PhabricatorWorkerBulkJobType' => 'Phobject', + 'PhabricatorWorkerBulkJobWorker' => 'PhabricatorWorker', + 'PhabricatorWorkerBulkTask' => 'PhabricatorWorkerDAO', 'PhabricatorWorkerDAO' => 'PhabricatorLiskDAO', 'PhabricatorWorkerLeaseQuery' => 'PhabricatorQuery', 'PhabricatorWorkerManagementCancelWorkflow' => 'PhabricatorWorkerManagementWorkflow', @@ -6612,6 +6653,7 @@ phutil_register_library_map(array( 'PhabricatorWorkerManagementRetryWorkflow' => 'PhabricatorWorkerManagementWorkflow', 'PhabricatorWorkerManagementWorkflow' => 'PhabricatorManagementWorkflow', 'PhabricatorWorkerPermanentFailureException' => 'Exception', + 'PhabricatorWorkerSchemaSpec' => 'PhabricatorConfigSchemaSpec', 'PhabricatorWorkerTask' => 'PhabricatorWorkerDAO', 'PhabricatorWorkerTaskData' => 'PhabricatorWorkerDAO', 'PhabricatorWorkerTaskDetailController' => 'PhabricatorDaemonController', diff --git a/src/applications/daemon/application/PhabricatorDaemonsApplication.php b/src/applications/daemon/application/PhabricatorDaemonsApplication.php index f7f3cb77ee..c605be63b6 100644 --- a/src/applications/daemon/application/PhabricatorDaemonsApplication.php +++ b/src/applications/daemon/application/PhabricatorDaemonsApplication.php @@ -46,6 +46,15 @@ final class PhabricatorDaemonsApplication extends PhabricatorApplication { '(?P[1-9]\d*)/' => 'PhabricatorDaemonLogViewController', ), 'event/(?P[1-9]\d*)/' => 'PhabricatorDaemonLogEventViewController', + 'bulk/' => array( + '(?:query/(?P[^/]+)/)?' => + 'PhabricatorDaemonBulkJobListController', + 'monitor/(?P\d+)/' => + 'PhabricatorDaemonBulkJobMonitorController', + 'view/(?P\d+)/' => + 'PhabricatorDaemonBulkJobViewController', + + ), ), ); } diff --git a/src/applications/daemon/controller/PhabricatorDaemonBulkJobListController.php b/src/applications/daemon/controller/PhabricatorDaemonBulkJobListController.php new file mode 100644 index 0000000000..ee8d4f5bf4 --- /dev/null +++ b/src/applications/daemon/controller/PhabricatorDaemonBulkJobListController.php @@ -0,0 +1,31 @@ +setQueryKey($request->getURIData('queryKey')) + ->setSearchEngine(new PhabricatorWorkerBulkJobSearchEngine()) + ->setNavigation($this->buildSideNavView()); + return $this->delegateToController($controller); + } + + protected function buildSideNavView($for_app = false) { + $user = $this->getRequest()->getUser(); + + $nav = new AphrontSideNavFilterView(); + $nav->setBaseURI(new PhutilURI($this->getApplicationURI())); + + id(new PhabricatorWorkerBulkJobSearchEngine()) + ->setViewer($user) + ->addNavigationItems($nav->getMenu()); + $nav->selectFilter(null); + + return $nav; + } +} diff --git a/src/applications/daemon/controller/PhabricatorDaemonBulkJobMonitorController.php b/src/applications/daemon/controller/PhabricatorDaemonBulkJobMonitorController.php new file mode 100644 index 0000000000..63ba3cacb1 --- /dev/null +++ b/src/applications/daemon/controller/PhabricatorDaemonBulkJobMonitorController.php @@ -0,0 +1,165 @@ +getViewer(); + + $job = id(new PhabricatorWorkerBulkJobQuery()) + ->setViewer($viewer) + ->withIDs(array($request->getURIData('id'))) + ->executeOne(); + if (!$job) { + return new Aphront404Response(); + } + + // If the user clicks "Continue" on a completed job, take them back to + // whatever application sent them here. + if ($request->getStr('done')) { + if ($request->isFormPost()) { + $done_uri = $job->getDoneURI(); + return id(new AphrontRedirectResponse())->setURI($done_uri); + } + } + + $title = pht('Bulk Job %d', $job->getID()); + + if ($job->getStatus() == PhabricatorWorkerBulkJob::STATUS_CONFIRM) { + $can_edit = PhabricatorPolicyFilter::hasCapability( + $viewer, + $job, + PhabricatorPolicyCapability::CAN_EDIT); + + if ($can_edit) { + if ($request->isFormPost()) { + $type_status = PhabricatorWorkerBulkJobTransaction::TYPE_STATUS; + + $xactions = array(); + $xactions[] = id(new PhabricatorWorkerBulkJobTransaction()) + ->setTransactionType($type_status) + ->setNewValue(PhabricatorWorkerBulkJob::STATUS_WAITING); + + $editor = id(new PhabricatorWorkerBulkJobEditor()) + ->setActor($viewer) + ->setContentSourceFromRequest($request) + ->setContinueOnMissingFields(true) + ->applyTransactions($job, $xactions); + + return id(new AphrontRedirectResponse()) + ->setURI($job->getMonitorURI()); + } else { + return $this->newDialog() + ->setTitle(pht('Confirm Bulk Job')) + ->appendParagraph($job->getDescriptionForConfirm()) + ->appendParagraph( + pht('Start work on this bulk job?')) + ->addCancelButton($job->getManageURI(), pht('Details')) + ->addSubmitButton(pht('Start Work')); + } + } else { + return $this->newDialog() + ->setTitle(pht('Waiting For Confirmation')) + ->appendParagraph( + pht( + 'This job is waiting for confirmation before work begins.')) + ->addCancelButotn($job->getManageURI(), pht('Details')); + } + } + + + $dialog = $this->newDialog() + ->setTitle(pht('%s: %s', $title, $job->getStatusName())) + ->addCancelButton($job->getManageURI(), pht('Details')); + + switch ($job->getStatus()) { + case PhabricatorWorkerBulkJob::STATUS_WAITING: + $dialog->appendParagraph( + pht('This job is waiting for tasks to be queued.')); + break; + case PhabricatorWorkerBulkJob::STATUS_RUNNING: + $dialog->appendParagraph( + pht('This job is running.')); + break; + case PhabricatorWorkerBulkJob::STATUS_COMPLETE: + $dialog->appendParagraph( + pht('This job is complete.')); + break; + } + + $counts = $job->loadTaskStatusCounts(); + if ($counts) { + $dialog->appendParagraph($this->renderProgress($counts)); + } + + switch ($job->getStatus()) { + case PhabricatorWorkerBulkJob::STATUS_COMPLETE: + $dialog->addHiddenInput('done', true); + $dialog->addSubmitButton(pht('Continue')); + break; + default: + Javelin::initBehavior('bulk-job-reload'); + break; + } + + return $dialog; + } + + private function renderProgress(array $counts) { + $this->requireResource('bulk-job-css'); + + $states = array( + PhabricatorWorkerBulkTask::STATUS_DONE => array( + 'class' => 'bulk-job-progress-slice-green', + ), + PhabricatorWorkerBulkTask::STATUS_RUNNING => array( + 'class' => 'bulk-job-progress-slice-blue', + ), + PhabricatorWorkerBulkTask::STATUS_WAITING => array( + 'class' => 'bulk-job-progress-slice-empty', + ), + PhabricatorWorkerBulkTask::STATUS_FAIL => array( + 'class' => 'bulk-job-progress-slice-red', + ), + ); + + $total = array_sum($counts); + $offset = 0; + $bars = array(); + foreach ($states as $state => $spec) { + $size = idx($counts, $state, 0); + if (!$size) { + continue; + } + + $classes = array(); + $classes[] = 'bulk-job-progress-slice'; + $classes[] = $spec['class']; + + $width = ($size / $total); + $bars[] = phutil_tag( + 'div', + array( + 'class' => implode(' ', $classes), + 'style' => + 'left: '.sprintf('%.2f%%', 100 * $offset).'; '. + 'width: '.sprintf('%.2f%%', 100 * $width).';', + ), + ''); + + $offset += $width; + } + + return phutil_tag( + 'div', + array( + 'class' => 'bulk-job-progress-bar', + ), + $bars); + } + +} diff --git a/src/applications/daemon/controller/PhabricatorDaemonBulkJobViewController.php b/src/applications/daemon/controller/PhabricatorDaemonBulkJobViewController.php new file mode 100644 index 0000000000..6fc3af724e --- /dev/null +++ b/src/applications/daemon/controller/PhabricatorDaemonBulkJobViewController.php @@ -0,0 +1,83 @@ +getViewer(); + + $job = id(new PhabricatorWorkerBulkJobQuery()) + ->setViewer($viewer) + ->withIDs(array($request->getURIData('id'))) + ->executeOne(); + if (!$job) { + return new Aphront404Response(); + } + + $title = pht('Bulk Job %d', $job->getID()); + + $crumbs = $this->buildApplicationCrumbs(); + $crumbs->addTextCrumb(pht('Bulk Jobs'), '/daemon/bulk/'); + $crumbs->addTextCrumb($title); + + $properties = $this->renderProperties($job); + $actions = $this->renderActions($job); + $properties->setActionList($actions); + + $box = id(new PHUIObjectBoxView()) + ->setHeaderText($title) + ->addPropertyList($properties); + + $timeline = $this->buildTransactionTimeline( + $job, + new PhabricatorWorkerBulkJobTransactionQuery()); + $timeline->setShouldTerminate(true); + + return $this->buildApplicationPage( + array( + $crumbs, + $box, + $timeline, + ), + array( + 'title' => $title, + )); + } + + private function renderProperties(PhabricatorWorkerBulkJob $job) { + $viewer = $this->getViewer(); + + $view = id(new PHUIPropertyListView()) + ->setUser($viewer) + ->setObject($job); + + $view->addProperty( + pht('Author'), + $viewer->renderHandle($job->getAuthorPHID())); + + $view->addProperty(pht('Status'), $job->getStatusName()); + + return $view; + } + + private function renderActions(PhabricatorWorkerBulkJob $job) { + $viewer = $this->getViewer(); + + $actions = id(new PhabricatorActionListView()) + ->setUser($viewer) + ->setObject($job); + + $actions->addAction( + id(new PhabricatorActionView()) + ->setHref($job->getDoneURI()) + ->setIcon('fa-arrow-circle-o-right') + ->setName(pht('Continue'))); + + return $actions; + } + +} diff --git a/src/applications/daemon/controller/PhabricatorDaemonController.php b/src/applications/daemon/controller/PhabricatorDaemonController.php index 24fbceb710..3b1d17a70b 100644 --- a/src/applications/daemon/controller/PhabricatorDaemonController.php +++ b/src/applications/daemon/controller/PhabricatorDaemonController.php @@ -10,6 +10,9 @@ abstract class PhabricatorDaemonController extends PhabricatorController { $nav->addFilter('/', pht('Console')); $nav->addFilter('log', pht('All Daemons')); + $nav->addLabel(pht('Bulk Jobs')); + $nav->addFilter('bulk', pht('Manage Bulk Jobs')); + return $nav; } diff --git a/src/applications/maniphest/bulk/ManiphestTaskEditBulkJobType.php b/src/applications/maniphest/bulk/ManiphestTaskEditBulkJobType.php new file mode 100644 index 0000000000..e24b1e7bd4 --- /dev/null +++ b/src/applications/maniphest/bulk/ManiphestTaskEditBulkJobType.php @@ -0,0 +1,296 @@ +getSize())); + } + + public function getJobSize(PhabricatorWorkerBulkJob $job) { + return count($job->getParameter('taskPHIDs', array())); + } + + public function getDoneURI(PhabricatorWorkerBulkJob $job) { + return $job->getParameter('doneURI'); + } + + public function createTasks(PhabricatorWorkerBulkJob $job) { + $tasks = array(); + + foreach ($job->getParameter('taskPHIDs', array()) as $phid) { + $tasks[] = PhabricatorWorkerBulkTask::initializeNewTask($job, $phid); + } + + return $tasks; + } + + public function runTask( + PhabricatorUser $actor, + PhabricatorWorkerBulkJob $job, + PhabricatorWorkerBulkTask $task) { + + $object = id(new ManiphestTaskQuery()) + ->setViewer($actor) + ->requireCapabilities( + array( + PhabricatorPolicyCapability::CAN_VIEW, + PhabricatorPolicyCapability::CAN_EDIT, + )) + ->withPHIDs(array($task->getObjectPHID())) + ->executeOne(); + if (!$object) { + return; + } + + $field_list = PhabricatorCustomField::getObjectFields( + $object, + PhabricatorCustomField::ROLE_EDIT); + $field_list->readFieldsFromStorage($object); + + $actions = $job->getParameter('actions'); + $xactions = $this->buildTransactions($actions, $object); + + $editor = id(new ManiphestTransactionEditor()) + ->setActor($actor) + ->setContentSource($job->newContentSource()) + ->setContinueOnNoEffect(true) + ->setContinueOnMissingFields(true) + ->applyTransactions($object, $xactions); + } + + private function buildTransactions($actions, ManiphestTask $task) { + $value_map = array(); + $type_map = array( + 'add_comment' => PhabricatorTransactions::TYPE_COMMENT, + 'assign' => ManiphestTransaction::TYPE_OWNER, + 'status' => ManiphestTransaction::TYPE_STATUS, + 'priority' => ManiphestTransaction::TYPE_PRIORITY, + 'add_project' => PhabricatorTransactions::TYPE_EDGE, + 'remove_project' => PhabricatorTransactions::TYPE_EDGE, + 'add_ccs' => PhabricatorTransactions::TYPE_SUBSCRIBERS, + 'remove_ccs' => PhabricatorTransactions::TYPE_SUBSCRIBERS, + 'space' => PhabricatorTransactions::TYPE_SPACE, + ); + + $edge_edit_types = array( + 'add_project' => true, + 'remove_project' => true, + 'add_ccs' => true, + 'remove_ccs' => true, + ); + + $xactions = array(); + foreach ($actions as $action) { + if (empty($type_map[$action['action']])) { + throw new Exception(pht("Unknown batch edit action '%s'!", $action)); + } + + $type = $type_map[$action['action']]; + + // Figure out the current value, possibly after modifications by other + // batch actions of the same type. For example, if the user chooses to + // "Add Comment" twice, we should add both comments. More notably, if the + // user chooses "Remove Project..." and also "Add Project...", we should + // avoid restoring the removed project in the second transaction. + + if (array_key_exists($type, $value_map)) { + $current = $value_map[$type]; + } else { + switch ($type) { + case PhabricatorTransactions::TYPE_COMMENT: + $current = null; + break; + case ManiphestTransaction::TYPE_OWNER: + $current = $task->getOwnerPHID(); + break; + case ManiphestTransaction::TYPE_STATUS: + $current = $task->getStatus(); + break; + case ManiphestTransaction::TYPE_PRIORITY: + $current = $task->getPriority(); + break; + case PhabricatorTransactions::TYPE_EDGE: + $current = $task->getProjectPHIDs(); + break; + case PhabricatorTransactions::TYPE_SUBSCRIBERS: + $current = $task->getSubscriberPHIDs(); + break; + case PhabricatorTransactions::TYPE_SPACE: + $current = PhabricatorSpacesNamespaceQuery::getObjectSpacePHID( + $task); + break; + } + } + + // Check if the value is meaningful / provided, and normalize it if + // necessary. This discards, e.g., empty comments and empty owner + // changes. + + $value = $action['value']; + switch ($type) { + case PhabricatorTransactions::TYPE_COMMENT: + if (!strlen($value)) { + continue 2; + } + break; + case PhabricatorTransactions::TYPE_SPACE: + if (empty($value)) { + continue 2; + } + $value = head($value); + break; + case ManiphestTransaction::TYPE_OWNER: + if (empty($value)) { + continue 2; + } + $value = head($value); + $no_owner = PhabricatorPeopleNoOwnerDatasource::FUNCTION_TOKEN; + if ($value === $no_owner) { + $value = null; + } + break; + case PhabricatorTransactions::TYPE_EDGE: + if (empty($value)) { + continue 2; + } + break; + case PhabricatorTransactions::TYPE_SUBSCRIBERS: + if (empty($value)) { + continue 2; + } + break; + } + + // If the edit doesn't change anything, go to the next action. This + // check is only valid for changes like "owner", "status", etc, not + // for edge edits, because we should still apply an edit like + // "Remove Projects: A, B" to a task with projects "A, B". + + if (empty($edge_edit_types[$action['action']])) { + if ($value == $current) { + continue; + } + } + + // Apply the value change; for most edits this is just replacement, but + // some need to merge the current and edited values (add/remove project). + + switch ($type) { + case PhabricatorTransactions::TYPE_COMMENT: + if (strlen($current)) { + $value = $current."\n\n".$value; + } + break; + case PhabricatorTransactions::TYPE_EDGE: + $is_remove = $action['action'] == 'remove_project'; + + $current = array_fill_keys($current, true); + $value = array_fill_keys($value, true); + + $new = $current; + $did_something = false; + + if ($is_remove) { + foreach ($value as $phid => $ignored) { + if (isset($new[$phid])) { + unset($new[$phid]); + $did_something = true; + } + } + } else { + foreach ($value as $phid => $ignored) { + if (empty($new[$phid])) { + $new[$phid] = true; + $did_something = true; + } + } + } + + if (!$did_something) { + continue 2; + } + + $value = array_keys($new); + break; + case PhabricatorTransactions::TYPE_SUBSCRIBERS: + $is_remove = $action['action'] == 'remove_ccs'; + + $current = array_fill_keys($current, true); + + $new = array(); + $did_something = false; + + if ($is_remove) { + foreach ($value as $phid) { + if (isset($current[$phid])) { + $new[$phid] = true; + $did_something = true; + } + } + if ($new) { + $value = array('-' => array_keys($new)); + } + } else { + $new = array(); + foreach ($value as $phid) { + $new[$phid] = true; + $did_something = true; + } + if ($new) { + $value = array('+' => array_keys($new)); + } + } + if (!$did_something) { + continue 2; + } + + break; + } + + $value_map[$type] = $value; + } + + $template = new ManiphestTransaction(); + + foreach ($value_map as $type => $value) { + $xaction = clone $template; + $xaction->setTransactionType($type); + + switch ($type) { + case PhabricatorTransactions::TYPE_COMMENT: + $xaction->attachComment( + id(new ManiphestTransactionComment()) + ->setContent($value)); + break; + case PhabricatorTransactions::TYPE_EDGE: + $project_type = PhabricatorProjectObjectHasProjectEdgeType::EDGECONST; + $xaction + ->setMetadataValue('edge:type', $project_type) + ->setNewValue( + array( + '=' => array_fuse($value), + )); + break; + default: + $xaction->setNewValue($value); + break; + } + + $xactions[] = $xaction; + } + + return $xactions; + } +} diff --git a/src/applications/maniphest/controller/ManiphestBatchEditController.php b/src/applications/maniphest/controller/ManiphestBatchEditController.php index e4086b2d18..1cee711dd2 100644 --- a/src/applications/maniphest/controller/ManiphestBatchEditController.php +++ b/src/applications/maniphest/controller/ManiphestBatchEditController.php @@ -45,8 +45,7 @@ final class ManiphestBatchEditController extends ManiphestController { if (!$tasks) { throw new Exception( - pht( - "You don't have permission to edit any of the selected tasks.")); + pht("You don't have permission to edit any of the selected tasks.")); } if ($project) { @@ -62,27 +61,32 @@ final class ManiphestBatchEditController extends ManiphestController { $actions = phutil_json_decode($actions); } - if ($request->isFormPost() && is_array($actions)) { - foreach ($tasks as $task) { - $field_list = PhabricatorCustomField::getObjectFields( - $task, - PhabricatorCustomField::ROLE_EDIT); - $field_list->readFieldsFromStorage($task); - - $xactions = $this->buildTransactions($actions, $task); - if ($xactions) { - // TODO: Set content source to "batch edit". - - $editor = id(new ManiphestTransactionEditor()) - ->setActor($viewer) - ->setContentSourceFromRequest($request) - ->setContinueOnNoEffect(true) - ->setContinueOnMissingFields(true) - ->applyTransactions($task, $xactions); - } - } - - return id(new AphrontRedirectResponse())->setURI($redirect_uri); + if ($request->isFormPost() && $actions) { + $job = PhabricatorWorkerBulkJob::initializeNewJob( + $viewer, + new ManiphestTaskEditBulkJobType(), + array( + 'taskPHIDs' => mpull($tasks, 'getPHID'), + 'actions' => $actions, + 'cancelURI' => $cancel_uri, + 'doneURI' => $redirect_uri, + )); + + $type_status = PhabricatorWorkerBulkJobTransaction::TYPE_STATUS; + + $xactions = array(); + $xactions[] = id(new PhabricatorWorkerBulkJobTransaction()) + ->setTransactionType($type_status) + ->setNewValue(PhabricatorWorkerBulkJob::STATUS_CONFIRM); + + $editor = id(new PhabricatorWorkerBulkJobEditor()) + ->setActor($viewer) + ->setContentSourceFromRequest($request) + ->setContinueOnMissingFields(true) + ->applyTransactions($job, $xactions); + + return id(new AphrontRedirectResponse()) + ->setURI($job->getMonitorURI()); } $handles = ManiphestTaskListView::loadTaskHandles($viewer, $tasks); @@ -210,228 +214,4 @@ final class ManiphestBatchEditController extends ManiphestController { )); } - private function buildTransactions($actions, ManiphestTask $task) { - $value_map = array(); - $type_map = array( - 'add_comment' => PhabricatorTransactions::TYPE_COMMENT, - 'assign' => ManiphestTransaction::TYPE_OWNER, - 'status' => ManiphestTransaction::TYPE_STATUS, - 'priority' => ManiphestTransaction::TYPE_PRIORITY, - 'add_project' => PhabricatorTransactions::TYPE_EDGE, - 'remove_project' => PhabricatorTransactions::TYPE_EDGE, - 'add_ccs' => PhabricatorTransactions::TYPE_SUBSCRIBERS, - 'remove_ccs' => PhabricatorTransactions::TYPE_SUBSCRIBERS, - 'space' => PhabricatorTransactions::TYPE_SPACE, - ); - - $edge_edit_types = array( - 'add_project' => true, - 'remove_project' => true, - 'add_ccs' => true, - 'remove_ccs' => true, - ); - - $xactions = array(); - foreach ($actions as $action) { - if (empty($type_map[$action['action']])) { - throw new Exception(pht("Unknown batch edit action '%s'!", $action)); - } - - $type = $type_map[$action['action']]; - - // Figure out the current value, possibly after modifications by other - // batch actions of the same type. For example, if the user chooses to - // "Add Comment" twice, we should add both comments. More notably, if the - // user chooses "Remove Project..." and also "Add Project...", we should - // avoid restoring the removed project in the second transaction. - - if (array_key_exists($type, $value_map)) { - $current = $value_map[$type]; - } else { - switch ($type) { - case PhabricatorTransactions::TYPE_COMMENT: - $current = null; - break; - case ManiphestTransaction::TYPE_OWNER: - $current = $task->getOwnerPHID(); - break; - case ManiphestTransaction::TYPE_STATUS: - $current = $task->getStatus(); - break; - case ManiphestTransaction::TYPE_PRIORITY: - $current = $task->getPriority(); - break; - case PhabricatorTransactions::TYPE_EDGE: - $current = $task->getProjectPHIDs(); - break; - case PhabricatorTransactions::TYPE_SUBSCRIBERS: - $current = $task->getSubscriberPHIDs(); - break; - case PhabricatorTransactions::TYPE_SPACE: - $current = PhabricatorSpacesNamespaceQuery::getObjectSpacePHID( - $task); - break; - } - } - - // Check if the value is meaningful / provided, and normalize it if - // necessary. This discards, e.g., empty comments and empty owner - // changes. - - $value = $action['value']; - switch ($type) { - case PhabricatorTransactions::TYPE_COMMENT: - if (!strlen($value)) { - continue 2; - } - break; - case PhabricatorTransactions::TYPE_SPACE: - if (empty($value)) { - continue 2; - } - $value = head($value); - break; - case ManiphestTransaction::TYPE_OWNER: - if (empty($value)) { - continue 2; - } - $value = head($value); - $no_owner = PhabricatorPeopleNoOwnerDatasource::FUNCTION_TOKEN; - if ($value === $no_owner) { - $value = null; - } - break; - case PhabricatorTransactions::TYPE_EDGE: - if (empty($value)) { - continue 2; - } - break; - case PhabricatorTransactions::TYPE_SUBSCRIBERS: - if (empty($value)) { - continue 2; - } - break; - } - - // If the edit doesn't change anything, go to the next action. This - // check is only valid for changes like "owner", "status", etc, not - // for edge edits, because we should still apply an edit like - // "Remove Projects: A, B" to a task with projects "A, B". - - if (empty($edge_edit_types[$action['action']])) { - if ($value == $current) { - continue; - } - } - - // Apply the value change; for most edits this is just replacement, but - // some need to merge the current and edited values (add/remove project). - - switch ($type) { - case PhabricatorTransactions::TYPE_COMMENT: - if (strlen($current)) { - $value = $current."\n\n".$value; - } - break; - case PhabricatorTransactions::TYPE_EDGE: - $is_remove = $action['action'] == 'remove_project'; - - $current = array_fill_keys($current, true); - $value = array_fill_keys($value, true); - - $new = $current; - $did_something = false; - - if ($is_remove) { - foreach ($value as $phid => $ignored) { - if (isset($new[$phid])) { - unset($new[$phid]); - $did_something = true; - } - } - } else { - foreach ($value as $phid => $ignored) { - if (empty($new[$phid])) { - $new[$phid] = true; - $did_something = true; - } - } - } - - if (!$did_something) { - continue 2; - } - - $value = array_keys($new); - break; - case PhabricatorTransactions::TYPE_SUBSCRIBERS: - $is_remove = $action['action'] == 'remove_ccs'; - - $current = array_fill_keys($current, true); - - $new = array(); - $did_something = false; - - if ($is_remove) { - foreach ($value as $phid) { - if (isset($current[$phid])) { - $new[$phid] = true; - $did_something = true; - } - } - if ($new) { - $value = array('-' => array_keys($new)); - } - } else { - $new = array(); - foreach ($value as $phid) { - $new[$phid] = true; - $did_something = true; - } - if ($new) { - $value = array('+' => array_keys($new)); - } - } - if (!$did_something) { - continue 2; - } - - break; - } - - $value_map[$type] = $value; - } - - $template = new ManiphestTransaction(); - - foreach ($value_map as $type => $value) { - $xaction = clone $template; - $xaction->setTransactionType($type); - - switch ($type) { - case PhabricatorTransactions::TYPE_COMMENT: - $xaction->attachComment( - id(new ManiphestTransactionComment()) - ->setContent($value)); - break; - case PhabricatorTransactions::TYPE_EDGE: - $project_type = PhabricatorProjectObjectHasProjectEdgeType::EDGECONST; - $xaction - ->setMetadataValue('edge:type', $project_type) - ->setNewValue( - array( - '=' => array_fuse($value), - )); - break; - default: - $xaction->setNewValue($value); - break; - } - - $xactions[] = $xaction; - } - - return $xactions; - } - } diff --git a/src/applications/metamta/contentsource/PhabricatorContentSource.php b/src/applications/metamta/contentsource/PhabricatorContentSource.php index c291b2069b..6d6a8ee896 100644 --- a/src/applications/metamta/contentsource/PhabricatorContentSource.php +++ b/src/applications/metamta/contentsource/PhabricatorContentSource.php @@ -15,6 +15,7 @@ final class PhabricatorContentSource extends Phobject { const SOURCE_DAEMON = 'daemon'; const SOURCE_LIPSUM = 'lipsum'; const SOURCE_PHORTUNE = 'phortune'; + const SOURCE_BULK = 'bulk'; private $source; private $params = array(); @@ -79,6 +80,7 @@ final class PhabricatorContentSource extends Phobject { self::SOURCE_LIPSUM => pht('Lipsum'), self::SOURCE_UNKNOWN => pht('Old World'), self::SOURCE_PHORTUNE => pht('Phortune'), + self::SOURCE_BULK => pht('Bulk Edit'), ); } diff --git a/src/infrastructure/daemon/workers/__tests__/PhabricatorWorkerBulkJobTestCase.php b/src/infrastructure/daemon/workers/__tests__/PhabricatorWorkerBulkJobTestCase.php new file mode 100644 index 0000000000..58ddaf0339 --- /dev/null +++ b/src/infrastructure/daemon/workers/__tests__/PhabricatorWorkerBulkJobTestCase.php @@ -0,0 +1,10 @@ +assertTrue(true); + } + +} diff --git a/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobCreateWorker.php b/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobCreateWorker.php new file mode 100644 index 0000000000..7c1b5dcada --- /dev/null +++ b/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobCreateWorker.php @@ -0,0 +1,51 @@ +acquireJobLock(); + + $job = $this->loadJob(); + $actor = $this->loadActor($job); + + $status = $job->getStatus(); + switch ($status) { + case PhabricatorWorkerBulkJob::STATUS_WAITING: + // This is what we expect. Other statuses indicate some kind of race + // is afoot. + break; + default: + throw new PhabricatorWorkerPermanentFailureException( + pht( + 'Found unexpected job status ("%s").', + $status)); + } + + $tasks = $job->createTasks(); + foreach ($tasks as $task) { + $task->save(); + } + + $this->updateJobStatus( + $job, + PhabricatorWorkerBulkJob::STATUS_RUNNING); + + $lock->unlock(); + + foreach ($tasks as $task) { + PhabricatorWorker::scheduleTask( + 'PhabricatorWorkerBulkJobTaskWorker', + array( + 'jobID' => $job->getID(), + 'taskID' => $task->getID(), + ), + array( + 'priority' => PhabricatorWorker::PRIORITY_BULK, + )); + } + + $this->updateJob($job); + } + +} diff --git a/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobTaskWorker.php b/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobTaskWorker.php new file mode 100644 index 0000000000..12de74c3ec --- /dev/null +++ b/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobTaskWorker.php @@ -0,0 +1,46 @@ +acquireTaskLock(); + + $task = $this->loadTask(); + $status = $task->getStatus(); + switch ($task->getStatus()) { + case PhabricatorWorkerBulkTask::STATUS_WAITING: + // This is what we expect. + break; + default: + throw new PhabricatorWorkerPermanentFailureException( + pht( + 'Found unexpected task status ("%s").', + $status)); + } + + $task + ->setStatus(PhabricatorWorkerBulkTask::STATUS_RUNNING) + ->save(); + + $lock->unlock(); + + $job = $this->loadJob(); + $actor = $this->loadActor($job); + + try { + $job->runTask($actor, $task); + $status = PhabricatorWorkerBulkTask::STATUS_DONE; + } catch (Exception $ex) { + phlog($ex); + $status = PhabricatorWorkerBulkTask::STATUS_FAIL; + } + + $task + ->setStatus($status) + ->save(); + + $this->updateJob($job); + } + +} diff --git a/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobType.php b/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobType.php new file mode 100644 index 0000000000..a5e29bc101 --- /dev/null +++ b/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobType.php @@ -0,0 +1,28 @@ +getManageURI(); + } + + final public static function getAllJobTypes() { + return id(new PhutilClassMapQuery()) + ->setAncestorClass(__CLASS__) + ->setUniqueMethod('getBulkJobTypeKey') + ->execute(); + } + +} diff --git a/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobWorker.php b/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobWorker.php new file mode 100644 index 0000000000..1f6644ae71 --- /dev/null +++ b/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobWorker.php @@ -0,0 +1,138 @@ +getJobID()) + ->lock(15); + } + + final protected function acquireTaskLock() { + return PhabricatorGlobalLock::newLock('bulktask.'.$this->getTaskID()) + ->lock(15); + } + + final protected function getJobID() { + $data = $this->getTaskData(); + $id = idx($data, 'jobID'); + if (!$id) { + throw new PhabricatorWorkerPermanentFailureException( + pht('Worker has no job ID.')); + } + return $id; + } + + final protected function getTaskID() { + $data = $this->getTaskData(); + $id = idx($data, 'taskID'); + if (!$id) { + throw new PhabricatorWorkerPermanentFailureException( + pht('Worker has no task ID.')); + } + return $id; + } + + final protected function loadJob() { + $id = $this->getJobID(); + $job = id(new PhabricatorWorkerBulkJobQuery()) + ->setViewer(PhabricatorUser::getOmnipotentUser()) + ->withIDs(array($id)) + ->executeOne(); + if (!$job) { + throw new PhabricatorWorkerPermanentFailureException( + pht('Worker has invalid job ID ("%s").', $id)); + } + return $job; + } + + final protected function loadTask() { + $id = $this->getTaskID(); + $task = id(new PhabricatorWorkerBulkTask())->load($id); + if (!$task) { + throw new PhabricatorWorkerPermanentFailureException( + pht('Worker has invalid task ID ("%s").', $id)); + } + return $task; + } + + final protected function loadActor(PhabricatorWorkerBulkJob $job) { + $actor_phid = $job->getAuthorPHID(); + $actor = id(new PhabricatorPeopleQuery()) + ->setViewer(PhabricatorUser::getOmnipotentUser()) + ->withPHIDs(array($actor_phid)) + ->executeOne(); + if (!$actor) { + throw new PhabricatorWorkerPermanentFailureException( + pht('Worker has invalid actor PHID ("%s").', $actor_phid)); + } + + $can_edit = PhabricatorPolicyFilter::hasCapability( + $actor, + $job, + PhabricatorPolicyCapability::CAN_EDIT); + + if (!$can_edit) { + throw new PhabricatorWorkerPermanentFailureException( + pht('Job actor does not have permission to edit job.')); + } + + return $actor; + } + + final protected function updateJob(PhabricatorWorkerBulkJob $job) { + $has_work = $this->hasRemainingWork($job); + if ($has_work) { + return; + } + + $lock = $this->acquireJobLock(); + + $job = $this->loadJob(); + if ($job->getStatus() == PhabricatorWorkerBulkJob::STATUS_RUNNING) { + if (!$this->hasRemainingWork($job)) { + $this->updateJobStatus( + $job, + PhabricatorWorkerBulkJob::STATUS_COMPLETE); + } + } + + $lock->unlock(); + } + + private function hasRemainingWork(PhabricatorWorkerBulkJob $job) { + return (bool)queryfx_one( + $job->establishConnection('r'), + 'SELECT * FROM %T WHERE bulkJobPHID = %s + AND status NOT IN (%Ls) LIMIT 1', + id(new PhabricatorWorkerBulkTask())->getTableName(), + $job->getPHID(), + array( + PhabricatorWorkerBulkTask::STATUS_DONE, + PhabricatorWorkerBulkTask::STATUS_FAIL, + )); + } + + protected function updateJobStatus(PhabricatorWorkerBulkJob $job, $status) { + $type_status = PhabricatorWorkerBulkJobTransaction::TYPE_STATUS; + + $xactions = array(); + $xactions[] = id(new PhabricatorWorkerBulkJobTransaction()) + ->setTransactionType($type_status) + ->setNewValue($status); + + $daemon_source = PhabricatorContentSource::newForSource( + PhabricatorContentSource::SOURCE_DAEMON, + array()); + + $app_phid = id(new PhabricatorDaemonsApplication())->getPHID(); + + $editor = id(new PhabricatorWorkerBulkJobEditor()) + ->setActor(PhabricatorUser::getOmnipotentUser()) + ->setActingAsPHID($app_phid) + ->setContentSource($daemon_source) + ->setContinueOnMissingFields(true) + ->applyTransactions($job, $xactions); + } + +} diff --git a/src/infrastructure/daemon/workers/editor/PhabricatorWorkerBulkJobEditor.php b/src/infrastructure/daemon/workers/editor/PhabricatorWorkerBulkJobEditor.php new file mode 100644 index 0000000000..b23c987d6d --- /dev/null +++ b/src/infrastructure/daemon/workers/editor/PhabricatorWorkerBulkJobEditor.php @@ -0,0 +1,87 @@ +getTransactionType()) { + case PhabricatorWorkerBulkJobTransaction::TYPE_STATUS: + return $object->getStatus(); + } + } + + protected function getCustomTransactionNewValue( + PhabricatorLiskDAO $object, + PhabricatorApplicationTransaction $xaction) { + + switch ($xaction->getTransactionType()) { + case PhabricatorWorkerBulkJobTransaction::TYPE_STATUS: + return $xaction->getNewValue(); + } + } + + protected function applyCustomInternalTransaction( + PhabricatorLiskDAO $object, + PhabricatorApplicationTransaction $xaction) { + + $type = $xaction->getTransactionType(); + $new = $xaction->getNewValue(); + + switch ($type) { + case PhabricatorWorkerBulkJobTransaction::TYPE_STATUS: + $object->setStatus($xaction->getNewValue()); + return; + } + + return parent::applyCustomInternalTransaction($object, $xaction); + } + + protected function applyCustomExternalTransaction( + PhabricatorLiskDAO $object, + PhabricatorApplicationTransaction $xaction) { + + $type = $xaction->getTransactionType(); + $new = $xaction->getNewValue(); + + switch ($type) { + case PhabricatorWorkerBulkJobTransaction::TYPE_STATUS: + switch ($new) { + case PhabricatorWorkerBulkJob::STATUS_WAITING: + PhabricatorWorker::scheduleTask( + 'PhabricatorWorkerBulkJobCreateWorker', + array( + 'jobID' => $object->getID(), + ), + array( + 'priority' => PhabricatorWorker::PRIORITY_BULK, + )); + break; + } + return; + } + + return parent::applyCustomExternalTransaction($object, $xaction); + } + + + +} diff --git a/src/infrastructure/daemon/workers/phid/PhabricatorWorkerBulkJobPHIDType.php b/src/infrastructure/daemon/workers/phid/PhabricatorWorkerBulkJobPHIDType.php new file mode 100644 index 0000000000..7550d1cf02 --- /dev/null +++ b/src/infrastructure/daemon/workers/phid/PhabricatorWorkerBulkJobPHIDType.php @@ -0,0 +1,37 @@ +withPHIDs($phids); + } + + public function loadHandles( + PhabricatorHandleQuery $query, + array $handles, + array $objects) { + + foreach ($handles as $phid => $handle) { + $job = $objects[$phid]; + + $id = $job->getID(); + + $handle->setName(pht('Bulk Job %d', $id)); + } + } + +} diff --git a/src/infrastructure/daemon/workers/query/PhabricatorWorkerBulkJobQuery.php b/src/infrastructure/daemon/workers/query/PhabricatorWorkerBulkJobQuery.php new file mode 100644 index 0000000000..32a9419a33 --- /dev/null +++ b/src/infrastructure/daemon/workers/query/PhabricatorWorkerBulkJobQuery.php @@ -0,0 +1,106 @@ +ids = $ids; + return $this; + } + + public function withPHIDs(array $phids) { + $this->phids = $phids; + return $this; + } + + public function withAuthorPHIDs(array $author_phids) { + $this->authorPHIDs = $author_phids; + return $this; + } + + public function withBulkJobTypes(array $job_types) { + $this->bulkJobTypes = $job_types; + return $this; + } + + public function withStatuses(array $statuses) { + $this->statuses = $statuses; + return $this; + } + + public function newResultObject() { + return new PhabricatorWorkerBulkJob(); + } + + protected function loadPage() { + return $this->loadStandardPage($this->newResultObject()); + } + + protected function willFilterPage(array $page) { + $map = PhabricatorWorkerBulkJobType::getAllJobTypes(); + + foreach ($page as $key => $job) { + $implementation = idx($map, $job->getJobTypeKey()); + if (!$implementation) { + $this->didRejectResult($job); + unset($page[$key]); + continue; + } + $job->attachJobImplementation($implementation); + } + + return $page; + } + + protected function buildWhereClauseParts(AphrontDatabaseConnection $conn) { + $where = parent::buildWhereClauseParts($conn); + + if ($this->ids !== null) { + $where[] = qsprintf( + $conn, + 'id IN (%Ld)', + $this->ids); + } + + if ($this->phids !== null) { + $where[] = qsprintf( + $conn, + 'phid IN (%Ls)', + $this->phids); + } + + if ($this->authorPHIDs !== null) { + $where[] = qsprintf( + $conn, + 'authorPHID IN (%Ls)', + $this->authorPHIDs); + } + + if ($this->bulkJobTypes !== null) { + $where[] = qsprintf( + $conn, + 'bulkJobType IN (%Ls)', + $this->bulkJobTypes); + } + + if ($this->statuses !== null) { + $where[] = qsprintf( + $conn, + 'status IN (%Ls)', + $this->statuses); + } + + return $where; + } + + public function getQueryApplicationClass() { + return 'PhabricatorDaemonsApplication'; + } + +} diff --git a/src/infrastructure/daemon/workers/query/PhabricatorWorkerBulkJobSearchEngine.php b/src/infrastructure/daemon/workers/query/PhabricatorWorkerBulkJobSearchEngine.php new file mode 100644 index 0000000000..e27cd04f16 --- /dev/null +++ b/src/infrastructure/daemon/workers/query/PhabricatorWorkerBulkJobSearchEngine.php @@ -0,0 +1,98 @@ +newQuery(); + + if ($map['authorPHIDs']) { + $query->withAuthorPHIDs($map['authorPHIDs']); + } + + return $query; + } + + protected function buildCustomSearchFields() { + return array( + id(new PhabricatorSearchUsersField()) + ->setLabel(pht('Authors')) + ->setKey('authorPHIDs') + ->setAliases(array('author', 'authors')), + ); + } + + protected function getURI($path) { + return '/daemon/bulk/'.$path; + } + + protected function getBuiltinQueryNames() { + $names = array(); + + if ($this->requireViewer()->isLoggedIn()) { + $names['authored'] = pht('Authored Jobs'); + } + + $names['all'] = pht('All Jobs'); + + return $names; + } + + public function buildSavedQueryFromBuiltin($query_key) { + + $query = $this->newSavedQuery(); + $query->setQueryKey($query_key); + + switch ($query_key) { + case 'all': + return $query; + case 'authored': + return $query->setParameter( + 'authorPHIDs', + array($this->requireViewer()->getPHID())); + } + + return parent::buildSavedQueryFromBuiltin($query_key); + } + + protected function renderResultList( + array $jobs, + PhabricatorSavedQuery $query, + array $handles) { + assert_instances_of($jobs, 'PhabricatorWorkerBulkJob'); + + $viewer = $this->requireViewer(); + + $list = id(new PHUIObjectItemListView()) + ->setUser($viewer); + foreach ($jobs as $job) { + $size = pht('%s Bulk Task(s)', new PhutilNumber($job->getSize())); + + $item = id(new PHUIObjectItemView()) + ->setObjectName(pht('Bulk Job %d', $job->getID())) + ->setHeader($job->getJobName()) + ->addAttribute(phabricator_datetime($job->getDateCreated(), $viewer)) + ->setHref($job->getManageURI()) + ->addIcon($job->getStatusIcon(), $job->getStatusName()) + ->addIcon('none', $size); + + $list->addItem($item); + } + + // TODO: Needs new wrapper when merging to redesign. + + return $list; + } +} diff --git a/src/infrastructure/daemon/workers/query/PhabricatorWorkerBulkJobTransactionQuery.php b/src/infrastructure/daemon/workers/query/PhabricatorWorkerBulkJobTransactionQuery.php new file mode 100644 index 0000000000..350277a888 --- /dev/null +++ b/src/infrastructure/daemon/workers/query/PhabricatorWorkerBulkJobTransactionQuery.php @@ -0,0 +1,10 @@ + true, + self::CONFIG_SERIALIZATION => array( + 'parameters' => self::SERIALIZATION_JSON, + ), + self::CONFIG_COLUMN_SCHEMA => array( + 'jobTypeKey' => 'text32', + 'status' => 'text32', + 'size' => 'uint32', + ), + self::CONFIG_KEY_SCHEMA => array( + 'key_type' => array( + 'columns' => array('jobTypeKey'), + ), + 'key_author' => array( + 'columns' => array('authorPHID'), + ), + 'key_status' => array( + 'columns' => array('status'), + ), + ), + ) + parent::getConfiguration(); + } + + public static function initializeNewJob( + PhabricatorUser $actor, + PhabricatorWorkerBulkJobType $type, + array $parameters) { + + $job = id(new PhabricatorWorkerBulkJob()) + ->setAuthorPHID($actor->getPHID()) + ->setJobTypeKey($type->getBulkJobTypeKey()) + ->setParameters($parameters) + ->attachJobImplementation($type); + + $job->setSize($job->computeSize()); + + return $job; + } + + public function generatePHID() { + return PhabricatorPHID::generateNewPHID( + PhabricatorWorkerBulkJobPHIDType::TYPECONST); + } + + public function getMonitorURI() { + return '/daemon/bulk/monitor/'.$this->getID().'/'; + } + + public function getManageURI() { + return '/daemon/bulk/view/'.$this->getID().'/'; + } + + public function getParameter($key, $default = null) { + return idx($this->parameters, $key, $default); + } + + public function setParameter($key, $value) { + $this->parameters[$key] = $value; + return $this; + } + + public function loadTaskStatusCounts() { + $table = new PhabricatorWorkerBulkTask(); + $conn_r = $table->establishConnection('r'); + $rows = queryfx_all( + $conn_r, + 'SELECT status, COUNT(*) N FROM %T WHERE bulkJobPHID = %s + GROUP BY status', + $table->getTableName(), + $this->getPHID()); + + return ipull($rows, 'N', 'status'); + } + + public function newContentSource() { + return PhabricatorContentSource::newForSource( + PhabricatorContentSource::SOURCE_BULK, + array( + 'jobID' => $this->getID(), + )); + } + + public function getStatusIcon() { + $map = array( + self::STATUS_CONFIRM => 'fa-question', + self::STATUS_WAITING => 'fa-clock-o', + self::STATUS_RUNNING => 'fa-clock-o', + self::STATUS_COMPLETE => 'fa-check grey', + ); + + return idx($map, $this->getStatus(), 'none'); + } + + public function getStatusName() { + $map = array( + self::STATUS_CONFIRM => pht('Confirming'), + self::STATUS_WAITING => pht('Waiting'), + self::STATUS_RUNNING => pht('Running'), + self::STATUS_COMPLETE => pht('Complete'), + ); + + return idx($map, $this->getStatus(), $this->getStatus()); + } + + +/* -( Job Implementation )------------------------------------------------- */ + + + protected function getJobImplementation() { + return $this->assertAttached($this->jobImplementation); + } + + public function attachJobImplementation(PhabricatorWorkerBulkJobType $type) { + $this->jobImplementation = $type; + return $this; + } + + private function computeSize() { + return $this->getJobImplementation()->getJobSize($this); + } + + public function getCancelURI() { + return $this->getJobImplementation()->getCancelURI($this); + } + + public function getDoneURI() { + return $this->getJobImplementation()->getDoneURI($this); + } + + public function getDescriptionForConfirm() { + return $this->getJobImplementation()->getDescriptionForConfirm($this); + } + + public function createTasks() { + return $this->getJobImplementation()->createTasks($this); + } + + public function runTask( + PhabricatorUser $actor, + PhabricatorWorkerBulkTask $task) { + return $this->getJobImplementation()->runTask($actor, $this, $task); + } + + public function getJobName() { + return $this->getJobImplementation()->getJobName($this); + } + + +/* -( PhabricatorPolicyInterface )----------------------------------------- */ + + + public function getCapabilities() { + return array( + PhabricatorPolicyCapability::CAN_VIEW, + PhabricatorPolicyCapability::CAN_EDIT, + ); + } + + public function getPolicy($capability) { + switch ($capability) { + case PhabricatorPolicyCapability::CAN_VIEW: + return PhabricatorPolicies::getMostOpenPolicy(); + case PhabricatorPolicyCapability::CAN_EDIT: + return $this->getAuthorPHID(); + } + } + + public function hasAutomaticCapability($capability, PhabricatorUser $viewer) { + return false; + } + + public function describeAutomaticCapability($capability) { + switch ($capability) { + case PhabricatorPolicyCapability::CAN_EDIT: + return pht('Only the owner of a bulk job can edit it.'); + default: + return null; + } + } + + +/* -( PhabricatorSubscribableInterface )----------------------------------- */ + + + public function isAutomaticallySubscribed($phid) { + return false; + } + + public function shouldShowSubscribersProperty() { + return true; + } + + public function shouldAllowSubscription($phid) { + return true; + } + + +/* -( PhabricatorApplicationTransactionInterface )------------------------- */ + + + public function getApplicationTransactionEditor() { + return new PhabricatorWorkerBulkJobEditor(); + } + + public function getApplicationTransactionObject() { + return $this; + } + + public function getApplicationTransactionTemplate() { + return new PhabricatorWorkerBulkJobTransaction(); + } + + public function willRenderTimeline( + PhabricatorApplicationTransactionView $timeline, + AphrontRequest $request) { + return $timeline; + } + +/* -( PhabricatorDestructibleInterface )----------------------------------- */ + + + public function destroyObjectPermanently( + PhabricatorDestructionEngine $engine) { + + $this->openTransaction(); + + // We're only removing the actual task objects. This may leave stranded + // workers in the queue itself, but they'll just flush out automatically + // when they can't load bulk job data. + + $task_table = new PhabricatorWorkerBulkTask(); + $conn_w = $task_table->establishConnection('w'); + queryfx( + $conn_w, + 'DELETE FROM %T WHERE bulkJobPHID = %s', + $task_table->getPHID(), + $this->getPHID()); + + $this->delete(); + $this->saveTransaction(); + } + + +} diff --git a/src/infrastructure/daemon/workers/storage/PhabricatorWorkerBulkJobTransaction.php b/src/infrastructure/daemon/workers/storage/PhabricatorWorkerBulkJobTransaction.php new file mode 100644 index 0000000000..9ac6b19716 --- /dev/null +++ b/src/infrastructure/daemon/workers/storage/PhabricatorWorkerBulkJobTransaction.php @@ -0,0 +1,51 @@ +getAuthorPHID(); + + $old = $this->getOldValue(); + $new = $this->getNewValue(); + + $type = $this->getTransactionType(); + switch ($type) { + case self::TYPE_STATUS: + if ($old === null) { + return pht( + '%s created this bulk job.', + $this->renderHandleLink($author_phid)); + } else { + switch ($new) { + case PhabricatorWorkerBulkJob::STATUS_WAITING: + return pht( + '%s confirmed this job.', + $this->renderHandleLink($author_phid)); + case PhabricatorWorkerBulkJob::STATUS_RUNNING: + return pht( + '%s marked this job as running.', + $this->renderHandleLink($author_phid)); + case PhabricatorWorkerBulkJob::STATUS_COMPLETE: + return pht( + '%s marked this job complete.', + $this->renderHandleLink($author_phid)); + } + } + break; + } + + return parent::getTitle(); + } + +} diff --git a/src/infrastructure/daemon/workers/storage/PhabricatorWorkerBulkTask.php b/src/infrastructure/daemon/workers/storage/PhabricatorWorkerBulkTask.php new file mode 100644 index 0000000000..382e435571 --- /dev/null +++ b/src/infrastructure/daemon/workers/storage/PhabricatorWorkerBulkTask.php @@ -0,0 +1,46 @@ + false, + self::CONFIG_SERIALIZATION => array( + 'data' => self::SERIALIZATION_JSON, + ), + self::CONFIG_COLUMN_SCHEMA => array( + 'status' => 'text32', + ), + self::CONFIG_KEY_SCHEMA => array( + 'key_job' => array( + 'columns' => array('bulkJobPHID', 'status'), + ), + 'key_object' => array( + 'columns' => array('objectPHID'), + ), + ), + ) + parent::getConfiguration(); + } + + public static function initializeNewTask( + PhabricatorWorkerBulkJob $job, + $object_phid) { + + return id(new PhabricatorWorkerBulkTask()) + ->setBulkJobPHID($job->getPHID()) + ->setStatus(self::STATUS_WAITING) + ->setObjectPHID($object_phid); + } + +} diff --git a/src/infrastructure/daemon/workers/storage/PhabricatorWorkerSchemaSpec.php b/src/infrastructure/daemon/workers/storage/PhabricatorWorkerSchemaSpec.php new file mode 100644 index 0000000000..2897f523c6 --- /dev/null +++ b/src/infrastructure/daemon/workers/storage/PhabricatorWorkerSchemaSpec.php @@ -0,0 +1,10 @@ +buildEdgeSchemata(new PhabricatorWorkerBulkJob()); + } + +} diff --git a/src/infrastructure/internationalization/translation/PhabricatorUSEnglishTranslation.php b/src/infrastructure/internationalization/translation/PhabricatorUSEnglishTranslation.php index 64198ae14e..453312f837 100644 --- a/src/infrastructure/internationalization/translation/PhabricatorUSEnglishTranslation.php +++ b/src/infrastructure/internationalization/translation/PhabricatorUSEnglishTranslation.php @@ -1177,6 +1177,11 @@ final class PhabricatorUSEnglishTranslation '%s Broken Test(s)' => '%s Broken', '%s Unsound Test(s)' => '%s Unsound', '%s Other Test(s)' => '%s Other', + + '%s Bulk Task(s)' => array( + '%s Task', + '%s Tasks', + ), ); } diff --git a/webroot/rsrc/css/application/daemon/bulk-job.css b/webroot/rsrc/css/application/daemon/bulk-job.css new file mode 100644 index 0000000000..206a4dd831 --- /dev/null +++ b/webroot/rsrc/css/application/daemon/bulk-job.css @@ -0,0 +1,32 @@ +/** + * @provides bulk-job-css + */ + +.bulk-job-progress-bar { + position: relative; + width: 100%; + border: 1px solid {$lightgreyborder}; + height: 32px; +} + +.bulk-job-progress-slice { + position: absolute; + top: 0; + bottom: 0; +} + +.bulk-job-progress-slice-green { + background-color: {$green}; +} + +.bulk-job-progress-slice-blue { + background-color: {$blue}; +} + +.bulk-job-progress-slice-red { + background-color: {$red}; +} + +.bulk-job-progress-slice-empty { + background-color: {$lightbluebackground}; +} diff --git a/webroot/rsrc/js/application/daemon/behavior-bulk-job-reload.js b/webroot/rsrc/js/application/daemon/behavior-bulk-job-reload.js new file mode 100644 index 0000000000..293f18b22c --- /dev/null +++ b/webroot/rsrc/js/application/daemon/behavior-bulk-job-reload.js @@ -0,0 +1,18 @@ +/** + * @provides javelin-behavior-bulk-job-reload + * @requires javelin-behavior + * javelin-uri + */ + +JX.behavior('bulk-job-reload', function() { + + // TODO: It would be nice to have a pretty Ajax progress bar here, but just + // reload the page for now. + + function reload() { + JX.$U().go(); + } + + setTimeout(reload, 1000); + +});