diff --git a/CHANGELOG.md b/CHANGELOG.md index a6b28cf2d4..27ea625b43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ AppEngine version, listed here to ease deployment and troubleshooting. ## Next Release (replace with git tag when deployed) * Bump runtimeVersion to `2025.08.25`. * Upgraded dependencies (incl. `googleapis` and `googleapis_auth`) + * Note: upload emails may contain a changelog excerpt. ## `20250821t095300-all` * Bump runtimeVersion to `2025.08.15`. diff --git a/app/lib/package/backend.dart b/app/lib/package/backend.dart index ab2e4376b0..8fb311441b 100644 --- a/app/lib/package/backend.dart +++ b/app/lib/package/backend.dart @@ -9,6 +9,7 @@ import 'package:_pub_shared/data/account_api.dart' as account_api; import 'package:_pub_shared/data/package_api.dart' as api; import 'package:_pub_shared/utils/sdk_version_cache.dart'; import 'package:clock/clock.dart'; +import 'package:collection/collection.dart'; import 'package:convert/convert.dart'; import 'package:crypto/crypto.dart'; import 'package:gcloud/service_scope.dart' as ss; @@ -21,6 +22,8 @@ import 'package:pub_dev/package/tarball_storage.dart'; import 'package:pub_dev/scorecard/backend.dart'; import 'package:pub_dev/service/async_queue/async_queue.dart'; import 'package:pub_dev/service/rate_limit/rate_limit.dart'; +import 'package:pub_dev/shared/changelog.dart'; +import 'package:pub_dev/shared/monitoring.dart'; import 'package:pub_dev/shared/versions.dart'; import 'package:pub_dev/task/backend.dart'; import 'package:pub_package_reader/pub_package_reader.dart'; @@ -1168,6 +1171,15 @@ class PackageBackend { .run() .toList(); + final changelogExcerpt = _createChangelogExcerpt( + versionKey: newVersion.qualifiedVersionKey, + changelogContent: entities.changelogAsset?.textContent, + ); + if (changelogExcerpt != null && changelogExcerpt.isNotEmpty) { + uploadMessages + .add('Excerpt of the changelog:\n```\n$changelogExcerpt\n```'); + } + // Add the new package to the repository by storing the tarball and // inserting metadata to datastore (which happens atomically). final (pv, outgoingEmail) = await withRetryTransaction(db, (tx) async { @@ -1315,6 +1327,50 @@ class PackageBackend { return (pv, uploadMessages); } + String? _createChangelogExcerpt({ + required QualifiedVersionKey versionKey, + required String? changelogContent, + }) { + if (changelogContent == null) { + return null; + } + try { + final parsed = ChangelogParser().parseMarkdownText(changelogContent); + final version = parsed.releases + .firstWhereOrNull((r) => r.version == versionKey.version); + if (version == null) { + return null; + } + final text = version.content.asMarkdownText; + + /// Limit the changelog to 10 lines, 75 characters each: + final lines = text.split('\n'); + final excerpt = lines + // prevent accidental HTML-tag creation + .map((line) => line + .replaceAll('<', '[') + .replaceAll('>', ']') + .replaceAll('&', ' ') + .trim()) + // filter empty or decorative lines to maximalize usefulness + .where((line) => + line.isNotEmpty && + line != '-' && // empty list item + line != '1.' && // empty list item + !line.startsWith('```') && // also removes the need to escape it + !line.startsWith('---')) + .take(10) + .map((line) => + line.length < 76 ? line : '${line.substring(0, 70)}[...]') + .join('\n'); + return excerpt; + } catch (e, st) { + _logger.pubNoticeShout('changelog-parse-error', + 'Unable to parse changelog for $versionKey', e, st); + return null; + } + } + /// The post-upload tasks are not critical and could fail without any impact on /// the uploaded package version. Important operations (e.g. email sending) are /// retried periodically, others (e.g. triggering re-analysis of dependent @@ -1903,6 +1959,9 @@ class _UploadEntities { this.packageVersionInfo, this.assets, ); + + late final changelogAsset = + assets.firstWhereOrNull((e) => e.kind == AssetKind.changelog); } class DerivedPackageVersionEntities { diff --git a/app/lib/shared/changelog.dart b/app/lib/shared/changelog.dart index acc74292db..db602ed7f3 100644 --- a/app/lib/shared/changelog.dart +++ b/app/lib/shared/changelog.dart @@ -19,7 +19,9 @@ library; import 'package:collection/collection.dart'; import 'package:html/dom.dart' as html; +import 'package:html/dom_parsing.dart' as dom_parsing; import 'package:html/parser.dart' as html_parser; +import 'package:markdown/markdown.dart' as m; import 'package:pub_semver/pub_semver.dart'; /// Represents the entire changelog, containing a list of releases. @@ -101,6 +103,177 @@ class Content { if (_asNode != null) return _asNode!; return html_parser.parseFragment(_asText!); }(); + + late final asMarkdownText = () { + final visitor = _MarkdownVisitor()..visit(asHtmlNode); + return visitor.toString(); + }(); +} + +class _MarkdownVisitor extends dom_parsing.TreeVisitor { + final _result = StringBuffer(); + int _listDepth = 0; + + void _write(String text) { + _result.write(text); + } + + void _writeln([String? text]) { + if (text != null) { + _write(text); + } + _write('\n'); + } + + void _visitChildrenInline(html.Element node) { + for (var i = 0; i < node.nodes.length; i++) { + final child = node.nodes[i]; + if (i > 0 && (node.nodes[i - 1].text?.endsWithWhitespace() ?? false)) { + _result.write(' '); + } + visit(child); + } + } + + @override + void visitElement(html.Element node) { + final localName = node.localName!; + + switch (localName) { + case 'h1': + _write('# '); + _visitChildrenInline(node); + _writeln(); + _writeln(); + break; + case 'h2': + _write('## '); + _visitChildrenInline(node); + _writeln(); + _writeln(); + break; + case 'h3': + _write('### '); + _visitChildrenInline(node); + _writeln(); + _writeln(); + break; + case 'h4': + _write('#### '); + _visitChildrenInline(node); + _writeln(); + _writeln(); + break; + case 'h5': + _write('##### '); + _visitChildrenInline(node); + _writeln(); + _writeln(); + break; + case 'h6': + _write('###### '); + _visitChildrenInline(node); + _writeln(); + _writeln(); + break; + case 'p': + _visitChildrenInline(node); + _writeln(); + _writeln(); + break; + case 'br': + _writeln(); + break; + case 'strong': + case 'b': + _write('**'); + _visitChildrenInline(node); + _write('**'); + break; + case 'em': + case 'i': + _write('*'); + _visitChildrenInline(node); + _write('*'); + break; + case 'code': + _write('`'); + _visitChildrenInline(node); + _write('`'); + break; + case 'pre': + _writeln('```'); + visitChildren(node); + _writeln('```'); + break; + case 'blockquote': + _write('>'); + _visitChildrenInline(node); + _writeln(); + break; + case 'a': + final href = node.attributes['href']; + if (href != null) { + _write('['); + _visitChildrenInline(node); + _write(']($href)'); + } else { + visitChildren(node); + } + break; + case 'ul': + _listDepth++; + visitChildren(node); + _listDepth--; + if (_listDepth == 0) _writeln(); + break; + case 'ol': + _listDepth++; + visitChildren(node); + _listDepth--; + if (_listDepth == 0) _writeln(); + break; + case 'li': + final parent = node.parent?.localName; + final indent = ' ' * (_listDepth - 1); + _write(indent); + if (parent == 'ol') { + final childIndex = (node.parent?.children.indexOf(node) ?? 0) + 1; + _write('$childIndex. '); + } else { + _write('- '); + } + _visitChildrenInline(node); + _writeln(); + break; + case 'hr': + _writeln('---'); + break; + default: + visitChildren(node); + break; + } + } + + @override + void visitText(html.Text node) { + _result.write(node.text.normalizeAndTrim()); + } + + @override + String toString() => _result.toString().trim(); +} + +extension on String { + String normalizeAndTrim() { + return replaceAll(RegExp(r'\s+'), ' ').trim(); + } + + bool endsWithWhitespace() { + if (isEmpty) return false; + final last = this[length - 1]; + return last == ' ' || last == '\n'; + } } /// Parses the changelog with pre-configured options. @@ -115,7 +288,16 @@ class ChangelogParser { }) : _strictLevels = strictLevels, _partOfLevelThreshold = partOfLevelThreshold; - /// Parses markdown nodes into a [Changelog] structure. + /// Parses markdown text into a [Changelog] structure. + Changelog parseMarkdownText(String input) { + final nodes = + m.Document(extensionSet: m.ExtensionSet.gitHubWeb).parse(input); + final rawHtml = m.renderToHtml(nodes); + final root = html_parser.parseFragment(rawHtml); + return parseHtmlNodes(root.nodes); + } + + /// Parses HTML nodes into a [Changelog] structure. Changelog parseHtmlNodes(List input) { String? title; Content? description; diff --git a/app/test/package/backend_test_utils.dart b/app/test/package/backend_test_utils.dart index cc6b03e9b6..28d41d375e 100644 --- a/app/test/package/backend_test_utils.dart +++ b/app/test/package/backend_test_utils.dart @@ -19,10 +19,13 @@ Future withTempDirectory(Future Function(String temp) func) async { } } -Future> packageArchiveBytes({required String pubspecContent}) async { +Future> packageArchiveBytes({ + required String pubspecContent, + String? changelogContent, +}) async { final builder = ArchiveBuilder(); builder.addFile('README.md', foobarReadmeContent); - builder.addFile('CHANGELOG.md', foobarChangelogContent); + builder.addFile('CHANGELOG.md', changelogContent ?? foobarChangelogContent); builder.addFile('pubspec.yaml', pubspecContent); builder.addFile('LICENSE', 'BSD LICENSE 2.0'); builder.addFile('lib/test_library.dart', 'hello() => print("hello");'); diff --git a/app/test/package/upload_test.dart b/app/test/package/upload_test.dart index 9349ce2d88..8d341decbb 100644 --- a/app/test/package/upload_test.dart +++ b/app/test/package/upload_test.dart @@ -128,6 +128,8 @@ void main() { expect(email.subject, 'Package uploaded: new_package 1.2.3'); expect(email.bodyText, contains('https://pub.dev/packages/new_package/versions/1.2.3\n')); + // No relevant changelog entry for this version. + expect(email.bodyText, isNot(contains('Excerpt of the changelog'))); final audits = await auditBackend.listRecordsForPackageVersion( 'new_package', '1.2.3'); @@ -1193,7 +1195,9 @@ void main() { final p1 = await packageBackend.lookupPackage('oxygen'); expect(p1!.versionCount, 3); final tarball = await packageArchiveBytes( - pubspecContent: generatePubspecYaml('oxygen', '3.0.0')); + pubspecContent: generatePubspecYaml('oxygen', '3.0.0'), + changelogContent: + '# Changelog\n\n## v3.0.0\n\nSome bug fixes:\n- one,\n- two\n\n'); final message = await createPubApiClient(authToken: adminClientToken) .uploadPackageBytes(tarball); expect(message.success.message, contains('Successfully uploaded')); @@ -1210,6 +1214,15 @@ void main() { expect(email.subject, 'Package uploaded: oxygen 3.0.0'); expect(email.bodyText, contains('https://pub.dev/packages/oxygen/versions/3.0.0\n')); + expect( + email.bodyText, + contains('\n' + 'Excerpt of the changelog:\n' + '```\n' + 'Some bug fixes:\n' + '- one,\n' + '- two\n' + '```\n\n')); await nameTracker.reloadFromDatastore(); final lastPublished = diff --git a/app/test/shared/changelog_test.dart b/app/test/shared/changelog_test.dart index 9c2d11374f..f9703758fd 100644 --- a/app/test/shared/changelog_test.dart +++ b/app/test/shared/changelog_test.dart @@ -2,16 +2,11 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. -import 'package:html/parser.dart' as html_parser; -import 'package:markdown/markdown.dart' as m; import 'package:pub_dev/shared/changelog.dart'; import 'package:test/test.dart'; Changelog _parse(String input) { - final nodes = m.Document(extensionSet: m.ExtensionSet.gitHubWeb).parse(input); - final rawHtml = m.renderToHtml(nodes); - final root = html_parser.parseFragment(rawHtml); - return ChangelogParser().parseHtmlNodes(root.nodes); + return ChangelogParser().parseMarkdownText(input); } void main() { @@ -47,6 +42,7 @@ void main() { expect(firstRelease.date, equals(DateTime(2025, 7, 10))); expect(firstRelease.content.asHtmlText, contains('New feature A')); expect(firstRelease.content.asHtmlText, contains('Bug fix 1')); + expect(firstRelease.content.asMarkdownText, contains('Bug fix 1')); final secondRelease = changelog.releases[1]; expect(secondRelease.version, equals('1.1.0')); @@ -538,5 +534,73 @@ This is the changelog for the project. expect(changelog.releases, hasLength(1)); expect(changelog.releases[0].version, equals('1.0.0')); }); + + test('markdown rendering with different styles', () { + const markdown = ''' +# Changelog + +## Header 2 + +Text 2 over +two lines. + +Multiple paragraphs with [link](https://pub.dev), `code`, and *different* **emphasis**. + +--- + +Also: +- unordered + multiline +- list + +And: +1. order +1. list + +>With multiline quoted +> `code` and *text*. + +### Header 3 +#### Header 4 +##### Header 5 +###### Header 6 +'''; + final changelog = _parse(markdown); + expect( + changelog.description?.asMarkdownText, + '## Header 2\n' + '\n' + 'Text 2 over two lines.\n' + '\n' + 'Multiple paragraphs with [link](https://pub.dev), `code`, and *different* **emphasis**.\n' + '\n' + '---\n' + 'Also:\n' + '\n' + '- unordered multiline\n' + '- list\n' + '\n' + 'And:\n' + '\n' + '1. order\n' + '2. list\n' + '\n' + '> With multiline quoted `code`and *text*.\n' + '\n' + '\n' + '### Header 3\n' + '\n' + '#### Header 4\n' + '\n' + '##### Header 5\n' + '\n' + '###### Header 6'); + + // check stability: another round of the markdown output yields the same result + final changelog2 = + _parse('# Changelog\n${changelog.description?.asMarkdownText}'); + expect(changelog2.description?.asMarkdownText, + changelog.description?.asMarkdownText); + }); }); }