diff --git a/.gitignore b/.gitignore index 7feaf93cbc..1efc0f7784 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,8 @@ packages .settings/ .DS_Store +pub.dartlang.org/ + # Or the files created by dart2js. *.dart.js *.js.deps diff --git a/lib/markdown_processor.dart b/lib/markdown_processor.dart index 0532b96f51..b04d832ed4 100644 --- a/lib/markdown_processor.dart +++ b/lib/markdown_processor.dart @@ -76,23 +76,21 @@ const List _oneLinerSkipTags = const ["code", "pre"]; String oneLinerWithoutReferences(String text) { if (text == null) return ''; // Parse with Markdown, but only care about the first block or paragraph. - var lines = text.replaceAll('\r\n', '\n').split('\n'); - var document = new md.Document(inlineSyntaxes: _markdown_syntaxes); + Iterable lines = text.replaceAll('\r\n', '\n').split('\n'); + md.Document document = new md.Document(inlineSyntaxes: _markdown_syntaxes); document.parseRefLinks(lines); - var blocks = document.parseLines(lines); + List blocks = document.parseLines(lines); while (blocks.isNotEmpty && ((blocks.first is md.Element && _oneLinerSkipTags.contains(blocks.first.tag)) || - blocks.first.isEmpty)) { + (blocks.first is md.Text && blocks.first.text.isEmpty))) { blocks.removeAt(0); } - if (blocks.isEmpty) { - return ''; - } + if (blocks.isEmpty) return ''; - var firstPara = new PlainTextRenderer().render([blocks.first]); + String firstPara = new PlainTextRenderer().render([blocks.first]); if (firstPara.length > 200) { firstPara = firstPara.substring(0, 200) + '...'; } diff --git a/lib/src/html_generator.dart b/lib/src/html_generator.dart index 55bdf6f171..b28cde139a 100644 --- a/lib/src/html_generator.dart +++ b/lib/src/html_generator.dart @@ -222,6 +222,8 @@ class HtmlGeneratorInstance { } void generateLibrary(Package package, Library lib) { + print('generating docs for library ${lib.path}...'); + // TODO should we add _this_ to the context and avoid putting stuff // in the map? Map data = { @@ -277,6 +279,7 @@ class HtmlGeneratorInstance { 'htmlBase': '..' }; + // TODO: `clazz.href` can be null here. _build(path.joinAll(clazz.href.split('/')), _templates.classTemplate, data); } @@ -493,7 +496,6 @@ class HtmlGeneratorInstance { File f = new File(path.join(out.path, filename)); if (!f.existsSync()) f.createSync(recursive: true); _htmlFiles.add(filename); - print('generating ${f.path}'); return f; } diff --git a/lib/src/model.dart b/lib/src/model.dart index 806c8fd7b1..e34486ae53 100644 --- a/lib/src/model.dart +++ b/lib/src/model.dart @@ -491,6 +491,8 @@ class Library extends ModelElement { return _name; } + String get path => _library.definingCompilationUnit.name; + String get nameForFile => name.replaceAll(':', '-'); bool get isInSdk => _library.isInSdk; diff --git a/pubspec.lock b/pubspec.lock index 267dc57696..236e382521 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -12,7 +12,7 @@ packages: args: description: args source: hosted - version: "0.13.1" + version: "0.13.2" barback: description: barback source: hosted @@ -68,7 +68,7 @@ packages: http_parser: description: http_parser source: hosted - version: "0.0.2+6" + version: "0.0.2+7" http_server: description: http_server source: hosted @@ -132,7 +132,7 @@ packages: quiver: description: quiver source: hosted - version: "0.21.3+1" + version: "0.21.4" shelf: description: shelf source: hosted @@ -160,7 +160,7 @@ packages: stack_trace: description: stack_trace source: hosted - version: "1.3.2" + version: "1.3.3" string_scanner: description: string_scanner source: hosted @@ -184,7 +184,7 @@ packages: watcher: description: watcher source: hosted - version: "0.9.5" + version: "0.9.6" webkit_inspection_protocol: description: webkit_inspection_protocol source: hosted @@ -204,4 +204,4 @@ packages: yaml: description: yaml source: hosted - version: "2.1.2" + version: "2.1.3" diff --git a/pubspec.yaml b/pubspec.yaml index e8903b0d68..a864c14325 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -23,6 +23,7 @@ dev_dependencies: den_api: ^0.1.0 grinder: ^0.7.1 librato: any + pub_semver: ^1.0.0 test: ^0.12.0 executables: dartdoc: null diff --git a/tool/doc_packages.dart b/tool/doc_packages.dart new file mode 100644 index 0000000000..658ebab6c9 --- /dev/null +++ b/tool/doc_packages.dart @@ -0,0 +1,257 @@ +// Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +/// A CLI tool to generate documentation for packages from pub.dartlang.org. +library dartdoc.doc_packages; + +import 'dart:async'; +import 'dart:convert' show JSON, UTF8; +import 'dart:io'; + +import 'package:args/args.dart'; +import 'package:http/http.dart' as http; +import 'package:pub_semver/pub_semver.dart'; +import 'package:yaml/yaml.dart'; + +// TODO: Use isolates; Platform.numberOfProcessors. + +const String _rootDir = 'pub.dartlang.org'; + +/// To use: +/// +/// dart tool/doc_packages.dart foo_package bar_package +/// +/// or: +/// +/// dart tool/doc_packages.dart --list --page=1 +/// +/// or: +/// +/// dart tool/doc_packages.dart --generate --page=3 +/// +void main(List _args) { + var parser = _createArgsParser(); + var args = parser.parse(_args); + + if (args['help']) { + _printUsageAndExit(parser); + } else if (args['list']) { + performList(int.parse(args['page'])); + } else if (args['generate']) { + performGenerate(int.parse(args['page'])); + } else if (args.rest.isEmpty) { + _printUsageAndExit(parser, exitCode: 1); + } else { + // Handle args.rest. + generateForPackages(args.rest); + } +} + +ArgParser _createArgsParser() { + var parser = new ArgParser(); + parser.addFlag('help', + abbr: 'h', negatable: false, help: 'Show command help.'); + parser.addFlag('list', help: 'Show available pub packages', negatable: false); + parser.addFlag('generate', + help: 'Generate docs for available pub packages.', negatable: false); + parser.addOption('page', + help: 'The pub.dartlang.org page to list or generate for.', + defaultsTo: '1'); + return parser; +} + +/// Print help if we are passed the help option or invalid arguments. +void _printUsageAndExit(ArgParser parser, {int exitCode: 0}) { + print('Generate documentation for published pub packages.\n'); + print('Usage: _doc_packages [OPTIONS] \n'); + print(parser.usage); + exit(exitCode); +} + +void performList(int page) { + print('Listing pub packages for page ${page}'); + print(''); + + _packageUrls(page).then((List packages) { + return _getPackageInfos(packages).then((List infos) { + infos.forEach(print); + }); + }); +} + +void performGenerate(int page) { + print('Generating docs for page ${page} into ${_rootDir}/'); + print(''); + + _packageUrls(page).then((List packages) { + return _getPackageInfos(packages).then((List infos) { + return Future.forEach(infos, (info) { + return _printGenerationResult(info, _generateFor(info)); + }); + }); + }); +} + +void generateForPackages(List packages) { + print('Generating docs into ${_rootDir}/'); + print(''); + + List urls = packages + .map((s) => 'https://pub.dartlang.org/packages/${s}.json') + .toList(); + + _getPackageInfos(urls).then((List infos) { + return Future.forEach(infos, (PackageInfo info) { + return _printGenerationResult(info, _generateFor(info)); + }); + }); +} + +Future _printGenerationResult(PackageInfo package, Future generationResult) { + String name = package.name.padRight(20); + + return generationResult.then((bool result) { + if (result) { + print('${name}: passed'); + } else { + print('${name}: (skipped)'); + } + }).catchError((e) { + print('${name}: * failed *'); + }); +} + +Future> _packageUrls(int page) { + return http + .get('https://pub.dartlang.org/packages.json?page=${page}') + .then((response) { + return JSON.decode(response.body)['packages']; + }); +} + +Future> _getPackageInfos(List packageUrls) { + List futures = packageUrls.map((String p) { + return http.get(p).then((response) { + var json = JSON.decode(response.body); + String name = json['name']; + List versions = + json['versions'].map((v) => new Version.parse(v)).toList(); + return new PackageInfo(name, Version.primary(versions)); + }); + }).toList(); + + return Future.wait(futures); +} + +StringBuffer _logBuffer; + +/// Generate the docs for the given package into _rootDir. Return whether +/// generation was performed or was skipped (due to an older package). +Future _generateFor(PackageInfo package) async { + _logBuffer = new StringBuffer(); + + // Get the package archive (tar zxvf foo.tar.gz). + var response = await http.get(package.archiveUrl); + if (response.statusCode != 200) throw response; + + Directory output = new Directory('${_rootDir}/${package.name}'); + output.createSync(recursive: true); + + try { + new File(output.path + '/archive.tar.gz') + .writeAsBytesSync(response.bodyBytes); + + await _exec('tar', ['zxvf', 'archive.tar.gz'], + cwd: output.path, quiet: true); + + // Rule out any old packages (old sdk constraints). + File pubspecFile = new File(output.path + '/pubspec.yaml'); + var pubspecInfo = loadYaml(pubspecFile.readAsStringSync()); + + // Check for old versions. + if (_isOldSdkConstraint(pubspecInfo)) { + _log('skipping ${package.name} - non-matching sdk constraint'); + return false; + } + + // Run pub get. + await _exec('pub', ['get'], + cwd: output.path, timeout: new Duration(seconds: 30)); + + // Run dartdoc. + await _exec('dart', ['../../bin/dartdoc.dart'], cwd: output.path); + + return true; + } catch (e, st) { + _log(e.toString()); + _log(st.toString()); + rethrow; + } finally { + new File(output.path + '/output.txt') + .writeAsStringSync(_logBuffer.toString()); + } +} + +Future _exec(String command, List args, {String cwd, bool quiet: false, + Duration timeout: const Duration(seconds: 60)}) { + return Process + .start(command, args, workingDirectory: cwd) + .then((Process process) { + if (!quiet) { + process.stdout.listen((bytes) => _log(UTF8.decode(bytes))); + process.stderr.listen((bytes) => _log(UTF8.decode(bytes))); + } + + Future f = process.exitCode.then((code) { + if (code != 0) throw code; + }); + + if (timeout != null) { + return f.timeout(timeout, onTimeout: () { + _log('Timing out operation ${command}.'); + process.kill(); + throw 'timeout on ${command}'; + }); + } else { + return f; + } + }); +} + +bool _isOldSdkConstraint(var pubspecInfo) { + var environment = pubspecInfo['environment']; + if (environment != null) { + var sdk = environment['sdk']; + if (sdk != null) { + VersionConstraint constraint = new VersionConstraint.parse(sdk); + String version = Platform.version; + if (version.contains(' ')) version = + version.substring(0, version.indexOf(' ')); + if (!constraint.allows(new Version.parse(version))) { + _log('sdk constraint = ${constraint}'); + return true; + } else { + return false; + } + } + } + + return false; +} + +void _log(String str) { + _logBuffer.write(str); +} + +class PackageInfo { + final String name; + final Version version; + + PackageInfo(this.name, this.version); + + String get archiveUrl => + 'https://storage.googleapis.com/pub.dartlang.org/packages/${name}-${version}.tar.gz'; + + String toString() => '${name}, ${version}'; +}