From 7398693b14d5d40d45a404b78bc1fb51b0c67171 Mon Sep 17 00:00:00 2001 From: mainframe98 Date: Fri, 13 Feb 2026 18:36:20 +0100 Subject: [PATCH] diviner: Atomize using PHP-Parser Summary: This allows Diviner to support atomizing PHP source files that use features of newer versions of PHP. Ref T16289 Test Plan: * (Optional) Edit source files to use new PHP features (enums, union types) * Run `./bin/diviner generate --clean` * Look at the generated documention on http://phorge.localhost/book/dev/ of the file modified Reviewers: O1 Blessed Committers, aklapper Reviewed By: O1 Blessed Committers, aklapper Subscribers: aklapper, tobiaswiese, valerio.bozzolan, Matthew, Cigaryno Maniphest Tasks: T16289 Differential Revision: https://we.phorge.it/D26755 --- src/applications/diviner/atom/DivinerAtom.php | 12 + .../diviner/atomizer/DivinerPHPAtomizer.php | 268 ++++++++++-------- 2 files changed, 163 insertions(+), 117 deletions(-) diff --git a/src/applications/diviner/atom/DivinerAtom.php b/src/applications/diviner/atom/DivinerAtom.php index 93e24d5d03..324c172c2c 100644 --- a/src/applications/diviner/atom/DivinerAtom.php +++ b/src/applications/diviner/atom/DivinerAtom.php @@ -4,10 +4,12 @@ final class DivinerAtom extends Phobject { const TYPE_ARTICLE = 'article'; const TYPE_CLASS = 'class'; + const TYPE_ENUM = 'enum'; const TYPE_FILE = 'file'; const TYPE_FUNCTION = 'function'; const TYPE_INTERFACE = 'interface'; const TYPE_METHOD = 'method'; + const TYPE_TRAIT = 'trait'; private $type; private $name; @@ -396,6 +398,10 @@ final class DivinerAtom extends Phobject { return pht('This interface is not documented.'); case self::TYPE_METHOD: return pht('This method is not documented.'); + case self::TYPE_TRAIT: + return pht('This trait is not documented.'); + case self::TYPE_ENUM: + return pht('This enum is not documented.'); default: phlog(pht("Need translation for '%s'.", $type)); return pht('This %s is not documented.', $type); @@ -410,6 +416,8 @@ final class DivinerAtom extends Phobject { self::TYPE_FUNCTION, self::TYPE_INTERFACE, self::TYPE_METHOD, + self::TYPE_TRAIT, + self::TYPE_ENUM, ); } @@ -425,6 +433,10 @@ final class DivinerAtom extends Phobject { return pht('Function'); case self::TYPE_INTERFACE: return pht('Interface'); + case self::TYPE_TRAIT: + return pht('Trait'); + case self::TYPE_ENUM: + return pht('Enum'); case self::TYPE_METHOD: return pht('Method'); default: diff --git a/src/applications/diviner/atomizer/DivinerPHPAtomizer.php b/src/applications/diviner/atomizer/DivinerPHPAtomizer.php index f3e07214ff..e4ff15b3df 100644 --- a/src/applications/diviner/atomizer/DivinerPHPAtomizer.php +++ b/src/applications/diviner/atomizer/DivinerPHPAtomizer.php @@ -1,5 +1,19 @@ resolve()); + $parser = PhutilPHPParserLibrary::getParser(); + $ast = $parser->parse($file_data); + + $classlike_finder = new PhpParser\NodeVisitor\FindingVisitor( + function ($node) { + return $node instanceof PhpParser\Node\Stmt\ClassLike; + }); + $function_finder = new PhpParser\NodeVisitor\FindingVisitor( + function ($node) { + return $node instanceof PhpParser\Node\Stmt\Function_; + }); + + $namespace_resolver = new PhpParser\NodeVisitor\NameResolver(); + $traverser = new PhpParser\NodeTraverser(); + $traverser->addVisitor($namespace_resolver); + $traverser->addVisitor($classlike_finder); + $traverser->addVisitor($function_finder); + $traverser->traverse($ast); $atoms = array(); - $root = $tree->getRootNode(); - - $func_decl = $root->selectDescendantsOfType('n_FUNCTION_DECLARATION'); - foreach ($func_decl as $func) { - $name = $func->getChildByIndex(2); - - // Don't atomize closures - if ($name->getTypeName() === 'n_EMPTY') { - continue; - } + foreach ($function_finder->getFoundNodes() as $func) { $atom = $this->newAtom(DivinerAtom::TYPE_FUNCTION) - ->setName($name->getConcreteString()) - ->setLine($func->getLineNumber()) + ->setName($func->namespacedName->toString()) + ->setLine($func->getStartLine()) ->setFile($file_name); $this->findAtomDocblock($atom, $func); @@ -37,99 +56,112 @@ final class DivinerPHPAtomizer extends DivinerAtomizer { } $class_types = array( - DivinerAtom::TYPE_CLASS => 'n_CLASS_DECLARATION', - DivinerAtom::TYPE_INTERFACE => 'n_INTERFACE_DECLARATION', + PhpParser\Node\Stmt\Class_::class => DivinerAtom::TYPE_CLASS, + PhpParser\Node\Stmt\Interface_::class => DivinerAtom::TYPE_INTERFACE, + PhpParser\Node\Stmt\Trait_::class => DivinerAtom::TYPE_TRAIT, + PhpParser\Node\Stmt\Enum_::class => DivinerAtom::TYPE_ENUM, ); - foreach ($class_types as $atom_type => $node_type) { - $class_decls = $root->selectDescendantsOfType($node_type); - foreach ($class_decls as $class) { - $name = $class->getChildByIndex(1, 'n_CLASS_NAME'); + foreach ($classlike_finder->getFoundNodes() as $class) { + $atom_type = $class_types[get_class($class)]; - $atom = $this->newAtom($atom_type) - ->setName($name->getConcreteString()) - ->setFile($file_name) - ->setLine($class->getLineNumber()); + // Don't analyze anonymous classes. + if (!$class->name) { + continue; + } - // This parses `final` and `abstract`. - $attributes = $class->getChildByIndex(0, 'n_CLASS_ATTRIBUTES'); - foreach ($attributes->selectDescendantsOfType('n_STRING') as $attr) { - $atom->setProperty($attr->getConcreteString(), true); + $atom = $this->newAtom($atom_type) + ->setName($class->namespacedName->toString()) + ->setFile($file_name) + ->setLine($class->getStartLine()); + + if ($class instanceof PhpParser\Node\Stmt\Class_) { + if ($class->isAbstract()) { + $atom->setProperty('abstract', true); + } else if ($class->isFinal()) { + $atom->setProperty('final', true); + } else if ($class->isReadonly()) { + $atom->setProperty('readonly', true); } - // If this exists, it is `n_EXTENDS_LIST`. - $extends = $class->getChildByIndex(2); - $extends_class = $extends->selectDescendantsOfType('n_CLASS_NAME'); - foreach ($extends_class as $parent_class) { + if ($class->extends) { $atom->addExtends( $this->newRef( DivinerAtom::TYPE_CLASS, - $parent_class->getConcreteString())); + $class->extends->toString())); } - // If this exists, it is `n_IMPLEMENTS_LIST`. - $implements = $class->getChildByIndex(3); - $iface_names = $implements->selectDescendantsOfType('n_CLASS_NAME'); - foreach ($iface_names as $iface_name) { + foreach ($class->implements as $implement) { + $atom->addExtends( + $this->newRef( + DivinerAtom::TYPE_INTERFACE, + $implement->toString())); + } + } else if ($class instanceof PhpParser\Node\Stmt\Interface_) { + foreach ($class->extends as $extend) { $atom->addExtends( $this->newRef( DivinerAtom::TYPE_INTERFACE, - $iface_name->getConcreteString())); + $extend->toString())); } + } else if ($class instanceof PhpParser\Node\Stmt\Enum_) { + foreach ($class->implements as $implement) { + $atom->addExtends( + $this->newRef( + DivinerAtom::TYPE_INTERFACE, + $implement->toString())); + } + } - $this->findAtomDocblock($atom, $class); - - $methods = $class->selectDescendantsOfType('n_METHOD_DECLARATION'); - foreach ($methods as $method) { - $matom = $this->newAtom(DivinerAtom::TYPE_METHOD); - - $this->findAtomDocblock($matom, $method); - - $attribute_list = $method->getChildByIndex(0); - $attributes = $attribute_list->selectDescendantsOfType('n_STRING'); - if ($attributes) { - foreach ($attributes as $attribute) { - $attr = strtolower($attribute->getConcreteString()); - switch ($attr) { - case 'final': - case 'abstract': - case 'static': - $matom->setProperty($attr, true); - break; - case 'public': - case 'protected': - case 'private': - $matom->setProperty('access', $attr); - break; - } - } - } else { - $matom->setProperty('access', 'public'); - } + $this->findAtomDocblock($atom, $class); + + foreach ($class->getMethods() as $method) { + $matom = $this->newAtom(DivinerAtom::TYPE_METHOD) + ->setName($method->name->toString()) + ->setLine($method->getStartLine()) + ->setFile($file_name); - $this->parseParams($matom, $method); + $this->findAtomDocblock($matom, $method); + + if ($method->isFinal()) { + $matom->setProperty('final', true); + } - $matom->setName($method->getChildByIndex(2)->getConcreteString()); - $matom->setLine($method->getLineNumber()); - $matom->setFile($file_name); + if ($method->isAbstract()) { + $matom->setProperty('abstract', true); + } - $this->parseReturnType($matom, $method); - $atom->addChild($matom); + if ($method->isStatic()) { + $matom->setProperty('static', true); + } - $atoms[] = $matom; + if ($method->isPrivate()) { + $matom->setProperty('access', 'private'); + } else if ($method->isProtected()) { + $matom->setProperty('access', 'protected'); + } else { + $matom->setProperty('access', 'public'); } - $atoms[] = $atom; + $this->parseParams($matom, $method); + + $this->parseReturnType($matom, $method); + $atom->addChild($matom); + + $atoms[] = $matom; } + + $atoms[] = $atom; } return $atoms; } - private function parseParams(DivinerAtom $atom, AASTNode $func) { - $params = $func - ->getChildOfType(3, 'n_DECLARATION_PARAMETER_LIST') - ->selectDescendantsOfType('n_DECLARATION_PARAMETER'); + private function parseParams( + DivinerAtom $atom, + PhpParser\Node\FunctionLike $func) { + + $params = $func->getParams(); $param_spec = array(); @@ -158,10 +190,10 @@ final class DivinerPHPAtomizer extends DivinerAtomizer { } foreach ($params as $param) { - $name = $param->getChildByIndex(1)->getConcreteString(); + $name = '$'.$param->var->name; $dict = array( - 'type' => $param->getChildByIndex(0)->getConcreteString(), - 'default' => $param->getChildByIndex(2)->getConcreteString(), + 'type' => $this->stringify($param->type), + 'default' => $this->stringify($param->default), ); if ($docs) { @@ -190,42 +222,32 @@ final class DivinerPHPAtomizer extends DivinerAtomizer { $atom->setProperty('parameters', $param_spec); } - private function findAtomDocblock(DivinerAtom $atom, XHPASTNode $node) { - $token = $node->getDocblockToken(); - if ($token) { - $atom->setDocblockRaw($token->getValue()); - return true; - } else { - $tokens = $node->getTokens(); - if ($tokens) { - $prev = head($tokens); - while ($prev = $prev->getPrevToken()) { - if ($prev->isAnyWhitespace()) { - continue; - } - break; - } + private function findAtomDocblock(DivinerAtom $atom, PhpParser\Node $node) { + $doc_comment = $node->getDocComment(); - if ($prev && $prev->isComment()) { - $value = $prev->getValue(); - $matches = null; - if (preg_match('/@(return|param|task|author)/', $value, $matches)) { - $atom->addWarning( - pht( - 'Atom "%s" is preceded by a comment containing `%s`, but '. - 'the comment is not a documentation comment. Documentation '. - 'comments must begin with `%s`, followed by a newline. Did '. - 'you mean to use a documentation comment? (As the comment is '. - 'not a documentation comment, it will be ignored.)', - $atom->getName(), - '@'.$matches[1], - '/**')); - } + if ($doc_comment) { + $atom->setDocblockRaw($doc_comment->getText()); + } else { + $comments = $node->getComments(); + + foreach ($comments as $comment) { + $value = $comment->getText(); + $matches = null; + if (preg_match('/@(return|param|task|author)/', $value, $matches)) { + $atom->addWarning( + pht( + 'Atom "%s" is preceded by a comment containing `%s`, but '. + 'the comment is not a documentation comment. Documentation '. + 'comments must begin with `%s`, followed by a newline. Did '. + 'you mean to use a documentation comment? (As the comment is '. + 'not a documentation comment, it will be ignored.)', + $atom->getName(), + '@'.$matches[1], + '/**')); } } $atom->setDocblockRaw(''); - return false; } } @@ -263,7 +285,10 @@ final class DivinerPHPAtomizer extends DivinerAtomizer { return $dict; } - private function parseReturnType(DivinerAtom $atom, XHPASTNode $decl) { + private function parseReturnType( + DivinerAtom $atom, + PhpParser\Node\FunctionLike $decl) { + $return_spec = array(); $metadata = $atom->getDocblockMeta(); @@ -314,7 +339,7 @@ final class DivinerPHPAtomizer extends DivinerAtomizer { $type = $split[0]; } - if ($decl->getChildByIndex(1)->getTypeName() == 'n_REFERENCE') { + if ($decl->returnsByRef()) { $type = $type.' &'; } @@ -335,4 +360,13 @@ final class DivinerPHPAtomizer extends DivinerAtomizer { $atom->setProperty('return', $return_spec); } + private function stringify(?PhpParser\Node $node) { + if (!$node) { + return ''; + } + + return id(new PhpParser\PrettyPrinter\Standard()) + ->prettyPrint(array($node)); + } + } -- 2.51.2