Devoncarew gen script

.gitignore

@@ -9,6 +9,8 @@ packages
 .settings/
 .DS_Store
 
+pub.dartlang.org/
+
 # Or the files created by dart2js.
 *.dart.js
 *.js.deps

lib/markdown_processor.dart

@@ -76,23 +76,21 @@ const List<String> _oneLinerSkipTags = const ["code", "pre"];
 String oneLinerWithoutReferences(String text) {
   if (text == null) return '';
   // Parse with Markdown, but only care about the first block or paragraph.
-  var lines = text.replaceAll('\r\n', '\n').split('\n');
-  var document = new md.Document(inlineSyntaxes: _markdown_syntaxes);
+  Iterable<String> lines = text.replaceAll('\r\n', '\n').split('\n');
+  md.Document document = new md.Document(inlineSyntaxes: _markdown_syntaxes);
   document.parseRefLinks(lines);
-  var blocks = document.parseLines(lines);
+  List blocks = document.parseLines(lines);
 
   while (blocks.isNotEmpty &&
       ((blocks.first is md.Element &&
               _oneLinerSkipTags.contains(blocks.first.tag)) ||
-          blocks.first.isEmpty)) {
+          (blocks.first is md.Text && blocks.first.text.isEmpty))) {
     blocks.removeAt(0);
   }
 
-  if (blocks.isEmpty) {
-    return '';
-  }
+  if (blocks.isEmpty) return '';
 
-  var firstPara = new PlainTextRenderer().render([blocks.first]);
+  String firstPara = new PlainTextRenderer().render([blocks.first]);
   if (firstPara.length > 200) {
     firstPara = firstPara.substring(0, 200) + '...';
   }

lib/src/html_generator.dart

@@ -222,6 +222,8 @@ class HtmlGeneratorInstance {
   }
 
   void generateLibrary(Package package, Library lib) {
+    print('generating docs for library ${lib.path}...');
+
     // TODO should we add _this_ to the context and avoid putting stuff
     // in the map?
     Map data = {
@@ -277,6 +279,7 @@ class HtmlGeneratorInstance {
       'htmlBase': '..'
     };
 
+    // TODO: `clazz.href` can be null here.
     _build(path.joinAll(clazz.href.split('/')), _templates.classTemplate, data);
   }
 
@@ -493,7 +496,6 @@ class HtmlGeneratorInstance {
     File f = new File(path.join(out.path, filename));
     if (!f.existsSync()) f.createSync(recursive: true);
     _htmlFiles.add(filename);
-    print('generating ${f.path}');
     return f;
   }
 

lib/src/model.dart

@@ -491,6 +491,8 @@ class Library extends ModelElement {
     return _name;
   }
 
+  String get path => _library.definingCompilationUnit.name;
+
   String get nameForFile => name.replaceAll(':', '-');
 
   bool get isInSdk => _library.isInSdk;

pubspec.lock

@@ -12,7 +12,7 @@ packages:
   args:
     description: args
     source: hosted
-    version: "0.13.1"
+    version: "0.13.2"
   barback:
     description: barback
     source: hosted
@@ -68,7 +68,7 @@ packages:
   http_parser:
     description: http_parser
     source: hosted
-    version: "0.0.2+6"
+    version: "0.0.2+7"
   http_server:
     description: http_server
     source: hosted
@@ -132,7 +132,7 @@ packages:
   quiver:
     description: quiver
     source: hosted
-    version: "0.21.3+1"
+    version: "0.21.4"
   shelf:
     description: shelf
     source: hosted
@@ -160,7 +160,7 @@ packages:
   stack_trace:
     description: stack_trace
     source: hosted
-    version: "1.3.2"
+    version: "1.3.3"
   string_scanner:
     description: string_scanner
     source: hosted
@@ -184,7 +184,7 @@ packages:
   watcher:
     description: watcher
     source: hosted
-    version: "0.9.5"
+    version: "0.9.6"
   webkit_inspection_protocol:
     description: webkit_inspection_protocol
     source: hosted
@@ -204,4 +204,4 @@ packages:
   yaml:
     description: yaml
     source: hosted
-    version: "2.1.2"
+    version: "2.1.3"

pubspec.yaml

@@ -23,6 +23,7 @@ dev_dependencies:
   den_api: ^0.1.0
   grinder: ^0.7.1
   librato: any
+  pub_semver: ^1.0.0
   test: ^0.12.0
 executables:
   dartdoc: null

tool/doc_packages.dart

@@ -0,0 +1,257 @@
+// Copyright (c) 2015, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+/// A CLI tool to generate documentation for packages from pub.dartlang.org.
+library dartdoc.doc_packages;
+
+import 'dart:async';
+import 'dart:convert' show JSON, UTF8;
+import 'dart:io';
+
+import 'package:args/args.dart';
+import 'package:http/http.dart' as http;
+import 'package:pub_semver/pub_semver.dart';
+import 'package:yaml/yaml.dart';
+
+// TODO: Use isolates; Platform.numberOfProcessors.
+
+const String _rootDir = 'pub.dartlang.org';
+
+/// To use:
+///
+///     dart tool/doc_packages.dart foo_package bar_package
+///
+/// or:
+///
+///     dart tool/doc_packages.dart --list --page=1
+///
+/// or:
+///
+///     dart tool/doc_packages.dart --generate --page=3
+///
+void main(List<String> _args) {
+  var parser = _createArgsParser();
+  var args = parser.parse(_args);
+
+  if (args['help']) {
+    _printUsageAndExit(parser);
+  } else if (args['list']) {
+    performList(int.parse(args['page']));
+  } else if (args['generate']) {
+    performGenerate(int.parse(args['page']));
+  } else if (args.rest.isEmpty) {
+    _printUsageAndExit(parser, exitCode: 1);
+  } else {
+    // Handle args.rest.
+    generateForPackages(args.rest);
+  }
+}
+
+ArgParser _createArgsParser() {
+  var parser = new ArgParser();
+  parser.addFlag('help',
+      abbr: 'h', negatable: false, help: 'Show command help.');
+  parser.addFlag('list', help: 'Show available pub packages', negatable: false);
+  parser.addFlag('generate',
+      help: 'Generate docs for available pub packages.', negatable: false);
+  parser.addOption('page',
+      help: 'The pub.dartlang.org page to list or generate for.',
+      defaultsTo: '1');
+  return parser;
+}
+
+/// Print help if we are passed the help option or invalid arguments.
+void _printUsageAndExit(ArgParser parser, {int exitCode: 0}) {
+  print('Generate documentation for published pub packages.\n');
+  print('Usage: _doc_packages [OPTIONS] <package1> <package2>\n');
+  print(parser.usage);
+  exit(exitCode);
+}
+
+void performList(int page) {
+  print('Listing pub packages for page ${page}');
+  print('');
+
+  _packageUrls(page).then((List<String> packages) {
+    return _getPackageInfos(packages).then((List<PackageInfo> infos) {
+      infos.forEach(print);
+    });
+  });
+}
+
+void performGenerate(int page) {
+  print('Generating docs for page ${page} into ${_rootDir}/');
+  print('');
+
+  _packageUrls(page).then((List<String> packages) {
+    return _getPackageInfos(packages).then((List<PackageInfo> infos) {
+      return Future.forEach(infos, (info) {
+        return _printGenerationResult(info, _generateFor(info));
+      });
+    });
+  });
+}
+
+void generateForPackages(List<String> packages) {
+  print('Generating docs into ${_rootDir}/');
+  print('');
+
+  List<String> urls = packages
+      .map((s) => 'https://pub.dartlang.org/packages/${s}.json')
+      .toList();
+
+  _getPackageInfos(urls).then((List<PackageInfo> infos) {
+    return Future.forEach(infos, (PackageInfo info) {
+      return _printGenerationResult(info, _generateFor(info));
+    });
+  });
+}
+
+Future _printGenerationResult(PackageInfo package, Future generationResult) {
+  String name = package.name.padRight(20);
+
+  return generationResult.then((bool result) {
+    if (result) {
+      print('${name}: passed');
+    } else {
+      print('${name}: (skipped)');
+    }
+  }).catchError((e) {
+    print('${name}: * failed *');
+  });
+}
+
+Future<List<String>> _packageUrls(int page) {
+  return http
+      .get('https://pub.dartlang.org/packages.json?page=${page}')
+      .then((response) {
+    return JSON.decode(response.body)['packages'];
+  });
+}
+
+Future<List<PackageInfo>> _getPackageInfos(List<String> packageUrls) {
+  List<Future> futures = packageUrls.map((String p) {
+    return http.get(p).then((response) {
+      var json = JSON.decode(response.body);
+      String name = json['name'];
+      List<Version> versions =
+          json['versions'].map((v) => new Version.parse(v)).toList();
+      return new PackageInfo(name, Version.primary(versions));
+    });
+  }).toList();
+
+  return Future.wait(futures);
+}
+
+StringBuffer _logBuffer;
+
+/// Generate the docs for the given package into _rootDir. Return whether
+/// generation was performed or was skipped (due to an older package).
+Future<bool> _generateFor(PackageInfo package) async {
+  _logBuffer = new StringBuffer();
+
+  // Get the package archive (tar zxvf foo.tar.gz).
+  var response = await http.get(package.archiveUrl);
+  if (response.statusCode != 200) throw response;
+
+  Directory output = new Directory('${_rootDir}/${package.name}');
+  output.createSync(recursive: true);
+
+  try {
+    new File(output.path + '/archive.tar.gz')
+        .writeAsBytesSync(response.bodyBytes);
+
+    await _exec('tar', ['zxvf', 'archive.tar.gz'],
+        cwd: output.path, quiet: true);
+
+    // Rule out any old packages (old sdk constraints).
+    File pubspecFile = new File(output.path + '/pubspec.yaml');
+    var pubspecInfo = loadYaml(pubspecFile.readAsStringSync());
+
+    // Check for old versions.
+    if (_isOldSdkConstraint(pubspecInfo)) {
+      _log('skipping ${package.name} - non-matching sdk constraint');
+      return false;
+    }
+
+    // Run pub get.
+    await _exec('pub', ['get'],
+        cwd: output.path, timeout: new Duration(seconds: 30));
+
+    // Run dartdoc.
+    await _exec('dart', ['../../bin/dartdoc.dart'], cwd: output.path);
+
+    return true;
+  } catch (e, st) {
+    _log(e.toString());
+    _log(st.toString());
+    rethrow;
+  } finally {
+    new File(output.path + '/output.txt')
+        .writeAsStringSync(_logBuffer.toString());
+  }
+}
+
+Future _exec(String command, List<String> args, {String cwd, bool quiet: false,
+    Duration timeout: const Duration(seconds: 60)}) {
+  return Process
+      .start(command, args, workingDirectory: cwd)
+      .then((Process process) {
+    if (!quiet) {
+      process.stdout.listen((bytes) => _log(UTF8.decode(bytes)));
+      process.stderr.listen((bytes) => _log(UTF8.decode(bytes)));
+    }
+
+    Future f = process.exitCode.then((code) {
+      if (code != 0) throw code;
+    });
+
+    if (timeout != null) {
+      return f.timeout(timeout, onTimeout: () {
+        _log('Timing out operation ${command}.');
+        process.kill();
+        throw 'timeout on ${command}';
+      });
+    } else {
+      return f;
+    }
+  });
+}
+
+bool _isOldSdkConstraint(var pubspecInfo) {
+  var environment = pubspecInfo['environment'];
+  if (environment != null) {
+    var sdk = environment['sdk'];
+    if (sdk != null) {
+      VersionConstraint constraint = new VersionConstraint.parse(sdk);
+      String version = Platform.version;
+      if (version.contains(' ')) version =
+          version.substring(0, version.indexOf(' '));
+      if (!constraint.allows(new Version.parse(version))) {
+        _log('sdk constraint = ${constraint}');
+        return true;
+      } else {
+        return false;
+      }
+    }
+  }
+
+  return false;
+}
+
+void _log(String str) {
+  _logBuffer.write(str);
+}
+
+class PackageInfo {
+  final String name;
+  final Version version;
+
+  PackageInfo(this.name, this.version);
+
+  String get archiveUrl =>
+      'https://storage.googleapis.com/pub.dartlang.org/packages/${name}-${version}.tar.gz';
+
+  String toString() => '${name}, ${version}';
+}