From 88caa45854133507a118f9e6293895734dca534c Mon Sep 17 00:00:00 2001 From: Nick Harper Date: Wed, 1 Aug 2012 17:06:04 -0700 Subject: [PATCH] Save daemon state to database Summary: To make it easier to monitor daemons, let's store their current state (running, died, exited, or unknown) to the db. The purpose of this is to provide more information on the daemon console about the status of daemons, especially when they are running on multiple machines. This is mostly backend work, with only a few frontend changes. (It is also dependent on a change to libphutil.) These changes will make dead or stuck daemons more obvious, and will allow more work on the frontend to hide daemons (and logs) that have exited cleanly, i.e. ones we don't care about any more. Test Plan: - run db migration, check in db that all daemons were marked as exited - start up a daemon, check in db that it is marked as running - open web interface, check that daemon is listed as running - after daemon has been running for a little bit, check in db that dateModified is being updated (indicating daemon is properly sending heartbeat) - kill -9 daemon (but don't run bin/phd yet), and check that db still shows it as running - edit daemon db entry to show it as being on a different host, and backdate dateModified field by 3 minutes, and check the web ui to show that the status is unknown. - change db entry to have proper host, check in web ui that daemon status is displayed as dead. Check db to see that the status was saved. - run bin/phd stop, and see that the formerly dead daemon is now exited. Reviewers: epriestley, vrana Reviewed By: epriestley CC: aran, Korvin Differential Revision: https://secure.phabricator.com/D3126 --- resources/sql/patches/daemonstatus.sql | 4 + src/__phutil_library_map__.php | 2 + .../ConduitAPI_daemon_launched_Method.php | 1 + .../ConduitAPI_daemon_setstatus_Method.php | 66 +++++++++++++++++ .../view/PhabricatorDaemonLogListView.php | 74 ++++++++++++------- .../daemon/PhabricatorDaemonControl.php | 11 +++ .../control/PhabricatorDaemonReference.php | 13 ++++ .../daemon/storage/PhabricatorDaemonLog.php | 6 ++ .../patch/PhabricatorBuiltinPatchList.php | 4 + 9 files changed, 155 insertions(+), 26 deletions(-) create mode 100644 resources/sql/patches/daemonstatus.sql create mode 100644 src/applications/conduit/method/daemon/ConduitAPI_daemon_setstatus_Method.php diff --git a/resources/sql/patches/daemonstatus.sql b/resources/sql/patches/daemonstatus.sql new file mode 100644 index 0000000000..6a2f392495 --- /dev/null +++ b/resources/sql/patches/daemonstatus.sql @@ -0,0 +1,4 @@ +ALTER TABLE {$NAMESPACE}_daemon.daemon_log + ADD COLUMN `status` varchar(8) NOT NULL; + +UPDATE {$NAMESPACE}_daemon.daemon_log SET `status`='exit'; diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php index 3252d68843..0f5eeb3e70 100644 --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -112,6 +112,7 @@ phutil_register_library_map(array( 'ConduitAPI_conduit_ping_Method' => 'applications/conduit/method/conduit/ConduitAPI_conduit_ping_Method.php', 'ConduitAPI_daemon_launched_Method' => 'applications/conduit/method/daemon/ConduitAPI_daemon_launched_Method.php', 'ConduitAPI_daemon_log_Method' => 'applications/conduit/method/daemon/ConduitAPI_daemon_log_Method.php', + 'ConduitAPI_daemon_setstatus_Method' => 'applications/conduit/method/daemon/ConduitAPI_daemon_setstatus_Method.php', 'ConduitAPI_differential_Method' => 'applications/conduit/method/differential/ConduitAPI_differential_Method.php', 'ConduitAPI_differential_close_Method' => 'applications/conduit/method/differential/ConduitAPI_differential_close_Method.php', 'ConduitAPI_differential_createcomment_Method' => 'applications/conduit/method/differential/ConduitAPI_differential_createcomment_Method.php', @@ -1229,6 +1230,7 @@ phutil_register_library_map(array( 'ConduitAPI_conduit_ping_Method' => 'ConduitAPIMethod', 'ConduitAPI_daemon_launched_Method' => 'ConduitAPIMethod', 'ConduitAPI_daemon_log_Method' => 'ConduitAPIMethod', + 'ConduitAPI_daemon_setstatus_Method' => 'ConduitAPIMethod', 'ConduitAPI_differential_Method' => 'ConduitAPIMethod', 'ConduitAPI_differential_close_Method' => 'ConduitAPIMethod', 'ConduitAPI_differential_createcomment_Method' => 'ConduitAPIMethod', diff --git a/src/applications/conduit/method/daemon/ConduitAPI_daemon_launched_Method.php b/src/applications/conduit/method/daemon/ConduitAPI_daemon_launched_Method.php index 7081dfc1d6..f0fafd17d5 100644 --- a/src/applications/conduit/method/daemon/ConduitAPI_daemon_launched_Method.php +++ b/src/applications/conduit/method/daemon/ConduitAPI_daemon_launched_Method.php @@ -58,6 +58,7 @@ final class ConduitAPI_daemon_launched_Method extends ConduitAPIMethod { $daemon_log->setDaemon($request->getValue('daemon')); $daemon_log->setHost($request->getValue('host')); $daemon_log->setPID($request->getValue('pid')); + $daemon_log->setStatus(PhabricatorDaemonLog::STATUS_RUNNING); $daemon_log->setArgv(json_decode($request->getValue('argv'))); $daemon_log->save(); diff --git a/src/applications/conduit/method/daemon/ConduitAPI_daemon_setstatus_Method.php b/src/applications/conduit/method/daemon/ConduitAPI_daemon_setstatus_Method.php new file mode 100644 index 0000000000..6800518baa --- /dev/null +++ b/src/applications/conduit/method/daemon/ConduitAPI_daemon_setstatus_Method.php @@ -0,0 +1,66 @@ + 'required string', + 'status' => 'required enum', + ); + } + + public function defineReturnType() { + return 'void'; + } + + public function defineErrorTypes() { + return array( + 'ERR-INVALID-ID' => 'An invalid daemonLogID was provided.', + ); + } + + protected function execute(ConduitAPIRequest $request) { + + $daemon_log = id(new PhabricatorDaemonLog()) + ->load($request->getValue('daemonLogID')); + if (!$daemon_log) { + throw new ConduitException('ERR-INVALID-ID'); + } + $daemon_log->setStatus($request->getValue('status')); + + $daemon_log->save(); + } + +} diff --git a/src/applications/daemon/view/PhabricatorDaemonLogListView.php b/src/applications/daemon/view/PhabricatorDaemonLogListView.php index 8c637fb95c..03a04de53e 100644 --- a/src/applications/daemon/view/PhabricatorDaemonLogListView.php +++ b/src/applications/daemon/view/PhabricatorDaemonLogListView.php @@ -42,38 +42,60 @@ final class PhabricatorDaemonLogListView extends AphrontView { foreach ($this->daemonLogs as $log) { $epoch = $log->getDateCreated(); - if ($log->getHost() == php_uname('n')) { + $status = $log->getStatus(); + if ($log->getHost() == php_uname('n') && + $status != PhabricatorDaemonLog::STATUS_EXITED && + $status != PhabricatorDaemonLog::STATUS_DEAD) { $pid = $log->getPID(); $is_running = PhabricatorDaemonReference::isProcessRunning($pid); - - if ($is_running) { - $running = phutil_render_tag( - 'span', - array( - 'style' => 'color: #00cc00', - 'title' => 'Running', - ), - '•'); - } else { - $running = phutil_render_tag( - 'span', - array( - 'style' => 'color: #cc0000', - 'title' => 'Not running', - ), - '•'); + if (!$is_running) { + $guard = AphrontWriteGuard::beginScopedUnguardedWrites(); + $log->setStatus(PhabricatorDaemonLog::STATUS_DEAD); + $log->save(); + unset($guard); + $status = PhabricatorDaemonLog::STATUS_DEAD; } - } else { - $running = phutil_render_tag( - 'span', - array( - 'style' => 'color: #888888', - 'title' => 'Not on this host', - ), - '?'); } + $heartbeat_timeout = + $log->getDateModified() + 3 * PhutilDaemonOverseer::HEARTBEAT_WAIT; + if ($status == PhabricatorDaemonLog::STATUS_RUNNING && + $heartbeat_timeout < time()) { + $status = PhabricatorDaemonLog::STATUS_UNKNOWN; + } + + switch ($status) { + case PhabricatorDaemonLog::STATUS_RUNNING: + $style = 'color: #00cc00'; + $title = 'Running'; + $symbol = '•'; + break; + case PhabricatorDaemonLog::STATUS_DEAD: + $style = 'color: #cc0000'; + $title = 'Died'; + $symbol = '•'; + break; + case PhabricatorDaemonLog::STATUS_EXITED: + $style = 'color: #000000'; + $title = 'Exited'; + $symbol = '•'; + break; + case PhabricatorDaemonLog::STATUS_UNKNOWN: + default: // fallthrough + $style = 'color: #888888'; + $title = 'Unknown'; + $symbol = '?'; + } + + $running = phutil_render_tag( + 'span', + array( + 'style' => $style, + 'title' => $title, + ), + $symbol); + $rows[] = array( $running, phutil_escape_html($log->getDaemon()), diff --git a/src/infrastructure/daemon/PhabricatorDaemonControl.php b/src/infrastructure/daemon/PhabricatorDaemonControl.php index e07d5d2e6b..743df8d936 100644 --- a/src/infrastructure/daemon/PhabricatorDaemonControl.php +++ b/src/infrastructure/daemon/PhabricatorDaemonControl.php @@ -53,6 +53,12 @@ final class PhabricatorDaemonControl { foreach ($daemons as $daemon) { $name = $daemon->getName(); if (!$daemon->isRunning()) { + $daemon_log = $daemon->loadDaemonLog(); + if ($daemon_log) { + $daemon_log->setStatus(PhabricatorDaemonLog::STATUS_DEAD); + $daemon_log->save(); + } + $status = 2; $name = ' '.$name; } @@ -110,6 +116,11 @@ final class PhabricatorDaemonControl { if (!$daemon->isRunning()) { echo "Daemon is not running.\n"; unset($running[$key]); + $daemon_log = $daemon->loadDaemonLog(); + if ($daemon_log) { + $daemon_log->setStatus(PhabricatorDaemonLog::STATUS_EXITED); + $daemon_log->save(); + } } else { posix_kill($pid, SIGINT); } diff --git a/src/infrastructure/daemon/control/PhabricatorDaemonReference.php b/src/infrastructure/daemon/control/PhabricatorDaemonReference.php index 9bb57ba603..1af743ac35 100644 --- a/src/infrastructure/daemon/control/PhabricatorDaemonReference.php +++ b/src/infrastructure/daemon/control/PhabricatorDaemonReference.php @@ -23,6 +23,8 @@ final class PhabricatorDaemonReference { private $start; private $pidFile; + private $daemonLog; + public static function newFromDictionary(array $dict) { $ref = new PhabricatorDaemonReference(); @@ -33,6 +35,17 @@ final class PhabricatorDaemonReference { return $ref; } + public function loadDaemonLog() { + if (!$this->daemonLog) { + $this->daemonLog = id(new PhabricatorDaemonLog())->loadOneWhere( + 'daemon = %s AND pid = %d AND dateCreated = %d', + $this->name, + $this->pid, + $this->start); + } + return $this->daemonLog; + } + public function getPID() { return $this->pid; } diff --git a/src/infrastructure/daemon/storage/PhabricatorDaemonLog.php b/src/infrastructure/daemon/storage/PhabricatorDaemonLog.php index 0c0e8fc960..8a53fccf22 100644 --- a/src/infrastructure/daemon/storage/PhabricatorDaemonLog.php +++ b/src/infrastructure/daemon/storage/PhabricatorDaemonLog.php @@ -18,10 +18,16 @@ final class PhabricatorDaemonLog extends PhabricatorDaemonDAO { + const STATUS_UNKNOWN = 'unknown'; + const STATUS_RUNNING = 'run'; + const STATUS_DEAD = 'dead'; + const STATUS_EXITED = 'exit'; + protected $daemon; protected $host; protected $pid; protected $argv; + protected $status; public function getConfiguration() { return array( diff --git a/src/infrastructure/storage/patch/PhabricatorBuiltinPatchList.php b/src/infrastructure/storage/patch/PhabricatorBuiltinPatchList.php index 7729e5aef3..999e742df9 100644 --- a/src/infrastructure/storage/patch/PhabricatorBuiltinPatchList.php +++ b/src/infrastructure/storage/patch/PhabricatorBuiltinPatchList.php @@ -932,6 +932,10 @@ final class PhabricatorBuiltinPatchList extends PhabricatorSQLPatchList { 'type' => 'php', 'name' => $this->getPatchPath('migrate-maniphest-revisions.php'), ), + 'daemonstatus.sql' => array( + 'type' => 'sql', + 'name' => $this->getPatchPath('daemonstatus.sql'), + ), ); } -- 2.51.2