From 143033dc1f4b2f7689c1abd5b08126fdea2049d1 Mon Sep 17 00:00:00 2001 From: epriestley Date: Wed, 28 Feb 2018 06:17:03 -0800 Subject: [PATCH] When showing a small piece of a Harbormaster build log, load a small piece of data instead of the entire log Summary: Depends on D19148. Ref T13088. The new rendering always executes range requests for data it needs, and we can satisfy these requests by loading the smallest number of chunks which span that range. Test Plan: Piped 50,000 lines of Apache log into Harbormaster, viewed it in the new UI, got sensible rendering times and a reasonable amount of data actually going over the wire. Subscribers: PHID-OPKG-gm6ozazyms6q6i22gyam Maniphest Tasks: T13088 Differential Revision: https://secure.phabricator.com/D19149 --- .../autopatches/20180228.log.01.offset.sql | 5 + .../storage/build/HarbormasterBuildLog.php | 117 ++++++++++++++++-- .../build/HarbormasterBuildLogChunk.php | 8 +- 3 files changed, 120 insertions(+), 10 deletions(-) create mode 100644 resources/sql/autopatches/20180228.log.01.offset.sql diff --git a/resources/sql/autopatches/20180228.log.01.offset.sql b/resources/sql/autopatches/20180228.log.01.offset.sql new file mode 100644 index 0000000000..db20fc292e --- /dev/null +++ b/resources/sql/autopatches/20180228.log.01.offset.sql @@ -0,0 +1,5 @@ +ALTER TABLE {$NAMESPACE}_harbormaster.harbormaster_buildlogchunk + ADD headOffset BIGINT UNSIGNED NOT NULL; + +ALTER TABLE {$NAMESPACE}_harbormaster.harbormaster_buildlogchunk + ADD tailOffset BIGINT UNSIGNED NOT NULL; diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php index 64934bbea8..ccee7724c3 100644 --- a/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php +++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLog.php @@ -130,7 +130,85 @@ final class HarbormasterBuildLog } public function loadData($offset, $length) { - return substr($this->getLogText(), $offset, $length); + $end = ($offset + $length); + + $chunks = id(new HarbormasterBuildLogChunk())->loadAllWhere( + 'logID = %d AND headOffset < %d AND tailOffset >= %d + ORDER BY headOffset ASC', + $this->getID(), + $end, + $offset); + + // Make sure that whatever we read out of the database is a single + // contiguous range which contains all of the requested bytes. + $ranges = array(); + foreach ($chunks as $chunk) { + $ranges[] = array( + 'head' => $chunk->getHeadOffset(), + 'tail' => $chunk->getTailOffset(), + ); + } + + $ranges = isort($ranges, 'head'); + $ranges = array_values($ranges); + $count = count($ranges); + for ($ii = 0; $ii < ($count - 1); $ii++) { + if ($ranges[$ii + 1]['head'] === $ranges[$ii]['tail']) { + $ranges[$ii + 1]['head'] = $ranges[$ii]['head']; + unset($ranges[$ii]); + } + } + + if (count($ranges) !== 1) { + $display_ranges = array(); + foreach ($ranges as $range) { + $display_ranges[] = pht( + '(%d - %d)', + $range['head'], + $range['tail']); + } + + if (!$display_ranges) { + $display_ranges[] = pht(''); + } + + throw new Exception( + pht( + 'Attempt to load log bytes (%d - %d) failed: failed to '. + 'load a single contiguous range. Actual ranges: %s.', + implode('; ', $display_ranges))); + } + + $range = head($ranges); + if ($range['head'] > $offset || $range['tail'] < $end) { + throw new Exception( + pht( + 'Attempt to load log bytes (%d - %d) failed: the loaded range '. + '(%d - %d) does not span the requested range.', + $offset, + $end, + $range['head'], + $range['tail'])); + } + + $parts = array(); + foreach ($chunks as $chunk) { + $parts[] = $chunk->getChunkDisplayText(); + } + $parts = implode('', $parts); + + $chop_head = ($offset - $range['head']); + $chop_tail = ($range['tail'] - $end); + + if ($chop_head) { + $parts = substr($parts, $chop_head); + } + + if ($chop_tail) { + $parts = substr($parts, 0, -$chop_tail); + } + + return $parts; } public function getReadPosition($read_offset) { @@ -220,17 +298,18 @@ final class HarbormasterBuildLog $this->openTransaction(); + $offset = 0; foreach ($chunks as $chunk) { $rope->append($chunk->getChunkDisplayText()); $chunk->delete(); while ($rope->getByteLength() > $byte_limit) { - $this->writeEncodedChunk($rope, $byte_limit, $mode); + $offset += $this->writeEncodedChunk($rope, $offset, $byte_limit, $mode); } } while ($rope->getByteLength()) { - $this->writeEncodedChunk($rope, $byte_limit, $mode); + $offset += $this->writeEncodedChunk($rope, $offset, $byte_limit, $mode); } $this @@ -240,7 +319,12 @@ final class HarbormasterBuildLog $this->saveTransaction(); } - private function writeEncodedChunk(PhutilRope $rope, $length, $mode) { + private function writeEncodedChunk( + PhutilRope $rope, + $offset, + $length, + $mode) { + $data = $rope->getPrefixBytes($length); $size = strlen($data); @@ -258,15 +342,22 @@ final class HarbormasterBuildLog throw new Exception(pht('Unknown chunk encoding "%s"!', $mode)); } - $this->writeChunk($mode, $size, $data); + $this->writeChunk($mode, $offset, $size, $data); $rope->removeBytesFromHead($size); + + return $size; } - private function writeChunk($encoding, $raw_size, $data) { + private function writeChunk($encoding, $offset, $raw_size, $data) { + $head_offset = $offset; + $tail_offset = $offset + $raw_size; + return id(new HarbormasterBuildLogChunk()) ->setLogID($this->getID()) ->setEncoding($encoding) + ->setHeadOffset($head_offset) + ->setTailOffset($tail_offset) ->setSize($raw_size) ->setChunk($data) ->save(); @@ -397,13 +488,23 @@ final class HarbormasterBuildLog if ($append_id) { queryfx( $conn_w, - 'UPDATE %T SET chunk = CONCAT(chunk, %B), size = %d WHERE id = %d', + 'UPDATE %T SET + chunk = CONCAT(chunk, %B), + size = %d, + tailOffset = headOffset + %d, + WHERE + id = %d', $chunk_table, $append_data, $prefix_size + $data_size, + $prefix_size + $data_size, $append_id); } else { - $this->writeChunk($encoding_text, $data_size, $append_data); + $this->writeChunk( + $encoding_text, + $this->getByteLength(), + $data_size, + $append_data); } $this->updateLineMap($append_data); diff --git a/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php b/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php index e69bad4caf..ff3e817c58 100644 --- a/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php +++ b/src/applications/harbormaster/storage/build/HarbormasterBuildLogChunk.php @@ -5,6 +5,8 @@ final class HarbormasterBuildLogChunk protected $logID; protected $encoding; + protected $headOffset; + protected $tailOffset; protected $size; protected $chunk; @@ -20,6 +22,8 @@ final class HarbormasterBuildLogChunk self::CONFIG_COLUMN_SCHEMA => array( 'logID' => 'id', 'encoding' => 'text32', + 'headOffset' => 'uint64', + 'tailOffset' => 'uint64', // T6203/NULLABILITY // Both the type and nullability of this column are crazily wrong. @@ -28,8 +32,8 @@ final class HarbormasterBuildLogChunk 'chunk' => 'bytes', ), self::CONFIG_KEY_SCHEMA => array( - 'key_log' => array( - 'columns' => array('logID'), + 'key_offset' => array( + 'columns' => array('logID', 'headOffset', 'tailOffset'), ), ), ) + parent::getConfiguration(); -- 2.51.2