diff --git a/packages/lex_gen/lib/lex_gen.dart b/packages/lex_gen/lib/lex_gen.dart index 89a788bca..d75222e9d 100644 --- a/packages/lex_gen/lib/lex_gen.dart +++ b/packages/lex_gen/lib/lex_gen.dart @@ -2,4 +2,6 @@ // All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +export 'package:lex_gen/src/config.dart'; export 'package:lex_gen/src/gen.dart'; +export 'package:lex_gen/src/doc_loader.dart'; diff --git a/packages/lex_gen/lib/src/commands/lex_command_generator.dart b/packages/lex_gen/lib/src/commands/lex_command_generator.dart index afcbf2af1..837716bd8 100644 --- a/packages/lex_gen/lib/src/commands/lex_command_generator.dart +++ b/packages/lex_gen/lib/src/commands/lex_command_generator.dart @@ -6,7 +6,6 @@ import 'dart:io'; // Package imports: -import 'package:lexicon/docs.dart'; import 'package:lexicon/lexicon.dart'; // Project imports: @@ -16,11 +15,9 @@ import 'types/lex_parameter.dart'; import 'types/lex_parent_command.dart'; import 'types/lex_root_command.dart'; -void generateLexCommands() { +void generateLexCommands(final List docs) { _cleanWorkspace(); - final docs = lexicons.map(LexiconDoc.fromJson).toList(); - final result = >{}; for (final doc in docs) { for (final def in doc.defs.entries) { diff --git a/packages/lex_gen/lib/src/commands/rule.dart b/packages/lex_gen/lib/src/commands/rule.dart index 7b24aca5d..2d0d12154 100644 --- a/packages/lex_gen/lib/src/commands/rule.dart +++ b/packages/lex_gen/lib/src/commands/rule.dart @@ -3,8 +3,15 @@ // BSD-style license that can be found in the LICENSE file. // Project imports: +import '../config.dart'; import '../utils.dart'; +LexCommandRuleConfig? _config; + +void setLexCommandRuleConfig(final LexCommandRuleConfig config) { + _config = config; +} + String getServiceId(final String lexiconId) { return lexiconId.split('.').sublist(0, 3).join('.'); } @@ -40,7 +47,12 @@ String getAbsoluteFilePathForParent(final String lexiconId) { } String getHomeDir() { - return 'packages/bluesky_cli/lib/src/commands/codegen'; + final config = _config; + if (config == null) { + throw StateError('Lex command rule config is not set'); + } + + return config.homeDir; } String getFilePath(final String lexiconId) { diff --git a/packages/lex_gen/lib/src/config.dart b/packages/lex_gen/lib/src/config.dart new file mode 100644 index 000000000..0dd140838 --- /dev/null +++ b/packages/lex_gen/lib/src/config.dart @@ -0,0 +1,72 @@ +// Copyright (c) 2023-2026, Shinya Kato. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +// Package imports: +import 'package:lexicon/lexicon.dart'; + +// Project imports: +import 'doc_loader.dart'; + +typedef LexiconDocsProvider = List Function(); + +LexiconDocsProvider lexiconDocsProviderFromPaths( + final List paths, { + bool recursive = true, +}) { + return () => loadLexiconDocsFromPaths(paths, recursive: recursive); +} + +final class LexiconNamespaceRule { + final List prefixes; + final String homeDir; + final String exportCodegenPath; + final String servicePackagePath; + final String rootPackageName; + + const LexiconNamespaceRule({ + required this.prefixes, + required this.homeDir, + required this.exportCodegenPath, + required this.servicePackagePath, + required this.rootPackageName, + }); + + bool matches(final String lexiconId) { + for (final prefix in prefixes) { + if (lexiconId.startsWith(prefix)) { + return true; + } + } + + return false; + } +} + +base class LexServiceRuleConfig { + final List namespaceRules; + + const LexServiceRuleConfig({required this.namespaceRules}); +} + +base class LexCommandRuleConfig { + final String homeDir; + + const LexCommandRuleConfig({required this.homeDir}); +} + +base class LexGenConfig { + final List services; + final List packages; + final LexiconDocsProvider docsProvider; + final LexServiceRuleConfig serviceRuleConfig; + final LexCommandRuleConfig commandRuleConfig; + + const LexGenConfig({ + required this.services, + required this.packages, + required this.docsProvider, + required this.serviceRuleConfig, + required this.commandRuleConfig, + }); +} diff --git a/packages/lex_gen/lib/src/doc_loader.dart b/packages/lex_gen/lib/src/doc_loader.dart new file mode 100644 index 000000000..df24d72bc --- /dev/null +++ b/packages/lex_gen/lib/src/doc_loader.dart @@ -0,0 +1,83 @@ +// Copyright (c) 2023-2026, Shinya Kato. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +// Dart imports: +import 'dart:convert'; +import 'dart:io'; + +// Package imports: +import 'package:lexicon/lexicon.dart'; + +/// Loads lexicon docs from [paths]. +/// +/// Supports file paths (must be `.json`) and directory paths. +/// When a directory is provided, `.json` files are discovered recursively. +List loadLexiconDocsFromPaths( + final List paths, { + bool recursive = true, +}) { + if (paths.isEmpty) { + throw ArgumentError.value(paths, 'paths', 'Must not be empty'); + } + + final files = []; + for (final path in paths) { + final type = FileSystemEntity.typeSync(path); + if (type == FileSystemEntityType.notFound) { + throw ArgumentError.value(path, 'paths', 'Path does not exist'); + } + + if (type == FileSystemEntityType.file) { + if (!path.endsWith('.json')) { + throw ArgumentError.value( + path, + 'paths', + 'Only .json files are supported', + ); + } + + files.add(File(path)); + continue; + } + + if (type == FileSystemEntityType.directory) { + final directory = Directory(path); + final entities = directory.listSync(recursive: recursive); + + for (final entity in entities) { + if (entity is! File) continue; + if (!entity.path.endsWith('.json')) continue; + + files.add(entity); + } + } + } + + if (files.isEmpty) { + throw ArgumentError.value(paths, 'paths', 'No lexicon .json files found'); + } + + files.sort((a, b) => a.absolute.path.compareTo(b.absolute.path)); + + final docs = []; + final ids = {}; + for (final file in files) { + final content = file.readAsStringSync(); + final dynamic json = jsonDecode(content); + + if (json is! Map) { + throw FormatException('Lexicon JSON must be an object', file.path); + } + + final doc = LexiconDoc.fromJson(json); + final id = doc.id.toString(); + if (!ids.add(id)) { + throw StateError('Duplicate lexicon id found: $id (${file.path})'); + } + + docs.add(doc); + } + + return docs; +} diff --git a/packages/lex_gen/lib/src/gen.dart b/packages/lex_gen/lib/src/gen.dart index 4baed02b9..4c463f4b0 100644 --- a/packages/lex_gen/lib/src/gen.dart +++ b/packages/lex_gen/lib/src/gen.dart @@ -4,20 +4,14 @@ // Project imports: import 'commands/lex_command_generator.dart'; +import 'commands/rule.dart' as command_rule; +import 'config.dart'; import 'services/lex_service_generator.dart'; import 'services/lex_tools_generator.dart'; import 'services/lex_type_generator.dart'; +import 'services/rule.dart' as service_rule; import 'utils.dart'; -const services = [ - 'com.atproto', - 'app.bsky', - 'chat.bsky', - 'tools.ozone', -]; - -const packages = ['atproto', 'bluesky']; - sealed class Gen { const Gen(); @@ -25,24 +19,37 @@ sealed class Gen { } final class ServiceGen implements Gen { - const ServiceGen(); + final LexGenConfig config; + + const ServiceGen({required this.config}); @override void execute() { print(kLexGeneratorLogo); - final types = generateLexTypes(services, packages); - generateLexServices(services, packages, types); + final docs = config.docsProvider(); + + service_rule.setLexServiceRuleConfig(config.serviceRuleConfig); + service_rule.setLexiconDocs(docs); - generateLexTools(); + final types = generateLexTypes(config.services, config.packages, docs); + generateLexServices(config.services, config.packages, types, docs); + + generateLexTools(docs); } } final class CommandGen implements Gen { - const CommandGen(); + final LexGenConfig config; + + const CommandGen({required this.config}); @override void execute() { - generateLexCommands(); + final docs = config.docsProvider(); + + command_rule.setLexCommandRuleConfig(config.commandRuleConfig); + + generateLexCommands(docs); } } diff --git a/packages/lex_gen/lib/src/services/lex_service_generator.dart b/packages/lex_gen/lib/src/services/lex_service_generator.dart index fadcdc357..69e2c27e6 100644 --- a/packages/lex_gen/lib/src/services/lex_service_generator.dart +++ b/packages/lex_gen/lib/src/services/lex_service_generator.dart @@ -6,7 +6,6 @@ import 'dart:io'; // Package imports: -import 'package:lexicon/docs.dart'; import 'package:lexicon/lexicon.dart'; // Project imports: @@ -21,22 +20,27 @@ void generateLexServices( final List services, final List packages, final List types, + final List docs, ) { - return _LexServiceGenerator(services, packages, types).execute(); + return _LexServiceGenerator(services, packages, types, docs).execute(); } final class _LexServiceGenerator { final List services; final List packages; final List types; + final List docs; - const _LexServiceGenerator(this.services, this.packages, this.types); + const _LexServiceGenerator( + this.services, + this.packages, + this.types, + this.docs, + ); void execute() { final services = []; - final docsByService = _groupDocsByService( - _filterLexicons().map(LexiconDoc.fromJson).toList(), - ); + final docsByService = _groupDocsByService(_filterLexicons(docs)); for (final entry in docsByService.entries) { final apis = []; @@ -92,9 +96,9 @@ final class _LexServiceGenerator { _generateLexPackages(services); } - List> _filterLexicons() { - return lexicons.where((lexicon) { - final id = lexicon['id'] as String; + List _filterLexicons(final List docs) { + return docs.where((lexicon) { + final id = lexicon.id.toString(); final service = id.split('.').sublist(0, 2).join('.'); return services.contains(service); diff --git a/packages/lex_gen/lib/src/services/lex_tools_generator.dart b/packages/lex_gen/lib/src/services/lex_tools_generator.dart index d00e607c1..93f0cdaab 100644 --- a/packages/lex_gen/lib/src/services/lex_tools_generator.dart +++ b/packages/lex_gen/lib/src/services/lex_tools_generator.dart @@ -6,7 +6,6 @@ import 'dart:io'; // Package imports: -import 'package:lexicon/docs.dart'; import 'package:lexicon/lexicon.dart'; // Project imports: @@ -14,12 +13,14 @@ import 'object/at_uri_extension.dart'; import 'object/repo_commit_handler.dart'; import 'rule.dart'; -void generateLexTools() { - return _LexToolsGenerator().execute(); +void generateLexTools(final List docs) { + return _LexToolsGenerator(docs).execute(); } final class _LexToolsGenerator { - const _LexToolsGenerator(); + final List docs; + + const _LexToolsGenerator(this.docs); void execute() { final recordLexiconIds = _getRecordLexiconIds(); @@ -37,7 +38,6 @@ final class _LexToolsGenerator { List _getRecordLexiconIds() { final recordLexiconIds = []; - final docs = lexicons.map(LexiconDoc.fromJson).toList(); for (final doc in docs) { if (_isRecord(doc)) { recordLexiconIds.add(doc.id.toString()); diff --git a/packages/lex_gen/lib/src/services/lex_type_generator.dart b/packages/lex_gen/lib/src/services/lex_type_generator.dart index c813be1e3..0f69218d0 100644 --- a/packages/lex_gen/lib/src/services/lex_type_generator.dart +++ b/packages/lex_gen/lib/src/services/lex_type_generator.dart @@ -6,7 +6,6 @@ import 'dart:io'; // Package imports: -import 'package:lexicon/docs.dart'; import 'package:lexicon/lexicon.dart' as lex; // Project imports: @@ -25,24 +24,23 @@ import 'rule.dart' as rule; List generateLexTypes( final List services, final List packages, + final List docs, ) { - return _LexTypeGenerator(services, packages).execute(); + return _LexTypeGenerator(services, packages, docs).execute(); } final class _LexTypeGenerator { final List services; final List packages; + final List docs; - const _LexTypeGenerator(this.services, this.packages); + const _LexTypeGenerator(this.services, this.packages, this.docs); List execute() { _cleanWorkspace(); final types = []; - final filteredLexicons = _filterLexicons( - lexicons, - services, - ).map(lex.LexiconDoc.fromJson).toList(); + final filteredLexicons = _filterLexicons(docs, services); final mainVariants = _checkMainVariants(filteredLexicons); @@ -173,12 +171,12 @@ final class _LexTypeGenerator { } } - List> _filterLexicons( - final List> lexicons, + List _filterLexicons( + final List lexicons, final List services, ) { return lexicons.where((lexicon) { - final id = lexicon['id'] as String; + final id = lexicon.id.toString(); final service = id.split('.').sublist(0, 2).join('.'); return services.contains(service); diff --git a/packages/lex_gen/lib/src/services/rule.dart b/packages/lex_gen/lib/src/services/rule.dart index 3b405857c..cfdb5ad05 100644 --- a/packages/lex_gen/lib/src/services/rule.dart +++ b/packages/lex_gen/lib/src/services/rule.dart @@ -3,13 +3,24 @@ // BSD-style license that can be found in the LICENSE file. // Package imports: -import 'package:lexicon/docs.dart'; import 'package:lexicon/lexicon.dart'; // Project imports: +import '../config.dart'; import '../utils.dart'; import 'object/lex_type.dart'; +LexServiceRuleConfig? _config; +List _lexiconDocs = const []; + +void setLexServiceRuleConfig(final LexServiceRuleConfig config) { + _config = config; +} + +void setLexiconDocs(final List docs) { + _lexiconDocs = List.unmodifiable(docs); +} + bool isDeprecated(final String? description) { if (description == null) return false; return description.toLowerCase().contains('deprecated'); @@ -147,39 +158,15 @@ String getLexObjectTypeId(final String lexiconId, final String defName) { } String getHomeDir(final String lexiconId) { - if (lexiconId.startsWith('com.atproto.')) { - return 'packages/atproto/lib/src/services/codegen'; - } else if (lexiconId.startsWith('app.bsky.') || - lexiconId.startsWith('chat.bsky.') || - lexiconId.startsWith('tools.ozone.')) { - return 'packages/bluesky/lib/src/services/codegen'; - } - - throw ArgumentError('Unsupported lexicon ID: $lexiconId'); + return _getNamespaceRule(lexiconId).homeDir; } String _getHomeDirForExport(final String lexiconId) { - if (lexiconId.startsWith('com.atproto.')) { - return 'package:atproto/src/services/codegen'; - } else if (lexiconId.startsWith('app.bsky.') || - lexiconId.startsWith('chat.bsky.') || - lexiconId.startsWith('tools.ozone.')) { - return 'package:bluesky/src/services/codegen'; - } - - throw ArgumentError('Unsupported lexicon ID: $lexiconId'); + return _getNamespaceRule(lexiconId).exportCodegenPath; } String _getHomeDirForService(final String lexiconId) { - if (lexiconId.startsWith('com.atproto.')) { - return 'package:atproto'; - } else if (lexiconId.startsWith('app.bsky.') || - lexiconId.startsWith('chat.bsky.') || - lexiconId.startsWith('tools.ozone.')) { - return 'package:bluesky'; - } - - throw ArgumentError('Unsupported lexicon ID: $lexiconId'); + return _getNamespaceRule(lexiconId).servicePackagePath; } String _getFileDir(final String lexiconId) { @@ -315,12 +302,19 @@ String getRecordTypeName(final String lexiconId) { } String getRootPackageName(final String lexiconId) { - if (lexiconId.startsWith('com.atproto.')) { - return 'atproto'; - } else if (lexiconId.startsWith('app.bsky.') || - lexiconId.startsWith('chat.bsky.') || - lexiconId.startsWith('tools.ozone.')) { - return 'bluesky'; + return _getNamespaceRule(lexiconId).rootPackageName; +} + +LexiconNamespaceRule _getNamespaceRule(final String lexiconId) { + final config = _config; + if (config == null) { + throw StateError('Lex service rule config is not set'); + } + + for (final rule in config.namespaceRules) { + if (rule.matches(lexiconId)) { + return rule; + } } throw ArgumentError('Unsupported lexicon ID: $lexiconId'); @@ -399,7 +393,7 @@ LexUserType? getRelatedDocFromRef(final String? ref) { final lexiconId = parts.first; final defName = parts.last; - for (final doc in lexicons.map(LexiconDoc.fromJson).toList()) { + for (final doc in _lexiconDocs) { for (final def in doc.defs.entries) { if (doc.id.toString() == lexiconId && def.key == defName) { return def.value; diff --git a/scripts/gen_codes.dart b/scripts/gen_codes.dart index bf23323c9..bf9e45f3c 100644 --- a/scripts/gen_codes.dart +++ b/scripts/gen_codes.dart @@ -17,6 +17,15 @@ import 'shared/error_handler.dart'; import 'shared/logger.dart'; import 'shared/progress_reporter.dart'; +const _lexiconServices = [ + 'com.atproto', + 'app.bsky', + 'chat.bsky', + 'tools.ozone', +]; + +const _lexiconPackages = ['atproto', 'bluesky']; + /// Optimized code generation script using BaseScript infrastructure. class GenCodesScript extends BaseScript { final ScriptConfig config; @@ -72,9 +81,14 @@ class GenCodesScript extends BaseScript { logger.info('Generating services and types...'); progress.updateProgress(0, currentItem: 'Initializing service generation'); + final lexGenConfig = _buildLexGenConfig( + lexiconsPath: config.lexiconsPath, + packagesPath: config.packagesPath, + ); + final serviceStartTime = DateTime.now(); try { - const ServiceGen().execute(); + ServiceGen(config: lexGenConfig).execute(); final serviceDuration = DateTime.now().difference(serviceStartTime); progress.updateProgress(1, currentItem: 'Service generation complete'); logger.debug( @@ -97,7 +111,7 @@ class GenCodesScript extends BaseScript { final commandStartTime = DateTime.now(); try { - const CommandGen().execute(); + CommandGen(config: lexGenConfig).execute(); final commandDuration = DateTime.now().difference(commandStartTime); progress.updateProgress(2, currentItem: 'Command generation complete'); logger.debug( @@ -179,8 +193,16 @@ class GenCodesScript extends BaseScript { try { // Spawn isolates for parallel execution await Future.wait([ - Isolate.spawn(_serviceGenerationIsolate, servicePort.sendPort), - Isolate.spawn(_commandGenerationIsolate, commandPort.sendPort), + Isolate.spawn(_serviceGenerationIsolate, { + 'sendPort': servicePort.sendPort, + 'lexiconsPath': config.lexiconsPath, + 'packagesPath': config.packagesPath, + }), + Isolate.spawn(_commandGenerationIsolate, { + 'sendPort': commandPort.sendPort, + 'lexiconsPath': config.lexiconsPath, + 'packagesPath': config.packagesPath, + }), ]); // Wait for both tasks to complete @@ -234,9 +256,18 @@ class _GenerationOptions { } /// Isolate entry point for service generation. -void _serviceGenerationIsolate(SendPort sendPort) { +void _serviceGenerationIsolate(Map args) { + final sendPort = args['sendPort']! as SendPort; + final lexiconsPath = args['lexiconsPath']! as String; + final packagesPath = args['packagesPath']! as String; + try { - const ServiceGen().execute(); + final config = _buildLexGenConfig( + lexiconsPath: lexiconsPath, + packagesPath: packagesPath, + ); + + ServiceGen(config: config).execute(); sendPort.send('complete'); } catch (error) { sendPort.send('error:$error'); @@ -244,15 +275,56 @@ void _serviceGenerationIsolate(SendPort sendPort) { } /// Isolate entry point for command generation. -void _commandGenerationIsolate(SendPort sendPort) { +void _commandGenerationIsolate(Map args) { + final sendPort = args['sendPort']! as SendPort; + final lexiconsPath = args['lexiconsPath']! as String; + final packagesPath = args['packagesPath']! as String; + try { - const CommandGen().execute(); + final config = _buildLexGenConfig( + lexiconsPath: lexiconsPath, + packagesPath: packagesPath, + ); + + CommandGen(config: config).execute(); sendPort.send('complete'); } catch (error) { sendPort.send('error:$error'); } } +LexGenConfig _buildLexGenConfig({ + required String lexiconsPath, + required String packagesPath, +}) { + return LexGenConfig( + services: _lexiconServices, + packages: _lexiconPackages, + docsProvider: lexiconDocsProviderFromPaths([lexiconsPath]), + serviceRuleConfig: LexServiceRuleConfig( + namespaceRules: [ + LexiconNamespaceRule( + prefixes: ['com.atproto.'], + homeDir: '$packagesPath/atproto/lib/src/services/codegen', + exportCodegenPath: 'package:atproto/src/services/codegen', + servicePackagePath: 'package:atproto', + rootPackageName: 'atproto', + ), + LexiconNamespaceRule( + prefixes: ['app.bsky.', 'chat.bsky.', 'tools.ozone.'], + homeDir: '$packagesPath/bluesky/lib/src/services/codegen', + exportCodegenPath: 'package:bluesky/src/services/codegen', + servicePackagePath: 'package:bluesky', + rootPackageName: 'bluesky', + ), + ], + ), + commandRuleConfig: LexCommandRuleConfig( + homeDir: '$packagesPath/bluesky_cli/lib/src/commands/codegen', + ), + ); +} + /// Main entry point for the script. Future main(List args) async { // Handle help request early