Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ AppEngine version, listed here to ease deployment and troubleshooting.
## Next Release (replace with git tag when deployed)
* Bump runtimeVersion to `2025.08.25`.
* Upgraded dependencies (incl. `googleapis` and `googleapis_auth`)
* Note: upload emails may contain a changelog excerpt.

## `20250821t095300-all`
* Bump runtimeVersion to `2025.08.15`.
Expand Down
59 changes: 59 additions & 0 deletions app/lib/package/backend.dart
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import 'package:_pub_shared/data/account_api.dart' as account_api;
import 'package:_pub_shared/data/package_api.dart' as api;
import 'package:_pub_shared/utils/sdk_version_cache.dart';
import 'package:clock/clock.dart';
import 'package:collection/collection.dart';
import 'package:convert/convert.dart';
import 'package:crypto/crypto.dart';
import 'package:gcloud/service_scope.dart' as ss;
Expand All @@ -21,6 +22,8 @@ import 'package:pub_dev/package/tarball_storage.dart';
import 'package:pub_dev/scorecard/backend.dart';
import 'package:pub_dev/service/async_queue/async_queue.dart';
import 'package:pub_dev/service/rate_limit/rate_limit.dart';
import 'package:pub_dev/shared/changelog.dart';
import 'package:pub_dev/shared/monitoring.dart';
import 'package:pub_dev/shared/versions.dart';
import 'package:pub_dev/task/backend.dart';
import 'package:pub_package_reader/pub_package_reader.dart';
Expand Down Expand Up @@ -1168,6 +1171,15 @@ class PackageBackend {
.run()
.toList();

final changelogExcerpt = _createChangelogExcerpt(
versionKey: newVersion.qualifiedVersionKey,
changelogContent: entities.changelogAsset?.textContent,
);
if (changelogExcerpt != null && changelogExcerpt.isNotEmpty) {
uploadMessages
.add('Excerpt of the changelog:\n```\n$changelogExcerpt\n```');
}

// Add the new package to the repository by storing the tarball and
// inserting metadata to datastore (which happens atomically).
final (pv, outgoingEmail) = await withRetryTransaction(db, (tx) async {
Expand Down Expand Up @@ -1315,6 +1327,50 @@ class PackageBackend {
return (pv, uploadMessages);
}

String? _createChangelogExcerpt({
required QualifiedVersionKey versionKey,
required String? changelogContent,
}) {
if (changelogContent == null) {
return null;
}
try {
final parsed = ChangelogParser().parseMarkdownText(changelogContent);
final version = parsed.releases
.firstWhereOrNull((r) => r.version == versionKey.version);
if (version == null) {
return null;
}
final text = version.content.asMarkdownText;

/// Limit the changelog to 10 lines, 75 characters each:
final lines = text.split('\n');
final excerpt = lines
// prevent accidental HTML-tag creation
.map((line) => line
.replaceAll('<', '[')
.replaceAll('>', ']')
.replaceAll('&', ' ')
.trim())
// filter empty or decorative lines to maximalize usefulness
.where((line) =>
line.isNotEmpty &&
line != '-' && // empty list item
line != '1.' && // empty list item
!line.startsWith('```') && // also removes the need to escape it
!line.startsWith('---'))
.take(10)
.map((line) =>
line.length < 76 ? line : '${line.substring(0, 70)}[...]')
.join('\n');
return excerpt;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we return null if the excerpt is empty/only whitespace?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We are filtering out most of the empty spaces and lines, maybe empty list items will be still kept. I'll add a bit more conditions to prevent / post-filter those.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was thinking if the "whole message" is whitespace only - it would look weird in the email.

Here's an excerpt from the CHANGELOG:

And now for something completely different... 

} catch (e, st) {
_logger.pubNoticeShout('changelog-parse-error',
'Unable to parse changelog for $versionKey', e, st);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Will this log the package name also?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, it is <package>/<version>.

return null;
}
}

/// The post-upload tasks are not critical and could fail without any impact on
/// the uploaded package version. Important operations (e.g. email sending) are
/// retried periodically, others (e.g. triggering re-analysis of dependent
Expand Down Expand Up @@ -1903,6 +1959,9 @@ class _UploadEntities {
this.packageVersionInfo,
this.assets,
);

late final changelogAsset =
assets.firstWhereOrNull((e) => e.kind == AssetKind.changelog);
}

class DerivedPackageVersionEntities {
Expand Down
184 changes: 183 additions & 1 deletion app/lib/shared/changelog.dart
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,9 @@ library;

import 'package:collection/collection.dart';
import 'package:html/dom.dart' as html;
import 'package:html/dom_parsing.dart' as dom_parsing;
import 'package:html/parser.dart' as html_parser;
import 'package:markdown/markdown.dart' as m;
import 'package:pub_semver/pub_semver.dart';

/// Represents the entire changelog, containing a list of releases.
Expand Down Expand Up @@ -101,6 +103,177 @@ class Content {
if (_asNode != null) return _asNode!;
return html_parser.parseFragment(_asText!);
}();

late final asMarkdownText = () {
final visitor = _MarkdownVisitor()..visit(asHtmlNode);
return visitor.toString();
}();
}

class _MarkdownVisitor extends dom_parsing.TreeVisitor {
final _result = StringBuffer();
int _listDepth = 0;

void _write(String text) {
_result.write(text);
}

void _writeln([String? text]) {
if (text != null) {
_write(text);
}
_write('\n');
}

void _visitChildrenInline(html.Element node) {
for (var i = 0; i < node.nodes.length; i++) {
final child = node.nodes[i];
if (i > 0 && (node.nodes[i - 1].text?.endsWithWhitespace() ?? false)) {
_result.write(' ');
}
visit(child);
}
}

@override
void visitElement(html.Element node) {
final localName = node.localName!;

switch (localName) {
case 'h1':
_write('# ');
_visitChildrenInline(node);
_writeln();
_writeln();
break;
case 'h2':
_write('## ');
_visitChildrenInline(node);
_writeln();
_writeln();
break;
case 'h3':
_write('### ');
_visitChildrenInline(node);
_writeln();
_writeln();
break;
case 'h4':
_write('#### ');
_visitChildrenInline(node);
_writeln();
_writeln();
break;
case 'h5':
_write('##### ');
_visitChildrenInline(node);
_writeln();
_writeln();
break;
case 'h6':
_write('###### ');
_visitChildrenInline(node);
_writeln();
_writeln();
break;
case 'p':
_visitChildrenInline(node);
_writeln();
_writeln();
break;
case 'br':
_writeln();
break;
case 'strong':
case 'b':
_write('**');
_visitChildrenInline(node);
_write('**');
break;
case 'em':
case 'i':
_write('*');
_visitChildrenInline(node);
_write('*');
break;
case 'code':
_write('`');
_visitChildrenInline(node);
_write('`');
break;
case 'pre':
_writeln('```');
visitChildren(node);
_writeln('```');
break;
case 'blockquote':
_write('>');
_visitChildrenInline(node);
_writeln();
break;
case 'a':
final href = node.attributes['href'];
if (href != null) {
_write('[');
_visitChildrenInline(node);
_write(']($href)');
} else {
visitChildren(node);
}
break;
case 'ul':
_listDepth++;
visitChildren(node);
_listDepth--;
if (_listDepth == 0) _writeln();
break;
case 'ol':
_listDepth++;
visitChildren(node);
_listDepth--;
if (_listDepth == 0) _writeln();
break;
case 'li':
final parent = node.parent?.localName;
final indent = ' ' * (_listDepth - 1);
_write(indent);
if (parent == 'ol') {
final childIndex = (node.parent?.children.indexOf(node) ?? 0) + 1;
_write('$childIndex. ');
} else {
_write('- ');
}
_visitChildrenInline(node);
_writeln();
break;
case 'hr':
_writeln('---');
break;
default:
visitChildren(node);
break;
}
}

@override
void visitText(html.Text node) {
_result.write(node.text.normalizeAndTrim());
}

@override
String toString() => _result.toString().trim();
}

extension on String {
String normalizeAndTrim() {
return replaceAll(RegExp(r'\s+'), ' ').trim();
}

bool endsWithWhitespace() {
if (isEmpty) return false;
final last = this[length - 1];
return last == ' ' || last == '\n';
}
}

/// Parses the changelog with pre-configured options.
Expand All @@ -115,7 +288,16 @@ class ChangelogParser {
}) : _strictLevels = strictLevels,
_partOfLevelThreshold = partOfLevelThreshold;

/// Parses markdown nodes into a [Changelog] structure.
/// Parses markdown text into a [Changelog] structure.
Changelog parseMarkdownText(String input) {
final nodes =
m.Document(extensionSet: m.ExtensionSet.gitHubWeb).parse(input);
final rawHtml = m.renderToHtml(nodes);
final root = html_parser.parseFragment(rawHtml);
return parseHtmlNodes(root.nodes);
}

/// Parses HTML nodes into a [Changelog] structure.
Changelog parseHtmlNodes(List<html.Node> input) {
String? title;
Content? description;
Expand Down
7 changes: 5 additions & 2 deletions app/test/package/backend_test_utils.dart
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,13 @@ Future<T> withTempDirectory<T>(Future<T> Function(String temp) func) async {
}
}

Future<List<int>> packageArchiveBytes({required String pubspecContent}) async {
Future<List<int>> packageArchiveBytes({
required String pubspecContent,
String? changelogContent,
}) async {
final builder = ArchiveBuilder();
builder.addFile('README.md', foobarReadmeContent);
builder.addFile('CHANGELOG.md', foobarChangelogContent);
builder.addFile('CHANGELOG.md', changelogContent ?? foobarChangelogContent);
builder.addFile('pubspec.yaml', pubspecContent);
builder.addFile('LICENSE', 'BSD LICENSE 2.0');
builder.addFile('lib/test_library.dart', 'hello() => print("hello");');
Expand Down
15 changes: 14 additions & 1 deletion app/test/package/upload_test.dart
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,8 @@ void main() {
expect(email.subject, 'Package uploaded: new_package 1.2.3');
expect(email.bodyText,
contains('https://pub.dev/packages/new_package/versions/1.2.3\n'));
// No relevant changelog entry for this version.
expect(email.bodyText, isNot(contains('Excerpt of the changelog')));

final audits = await auditBackend.listRecordsForPackageVersion(
'new_package', '1.2.3');
Expand Down Expand Up @@ -1193,7 +1195,9 @@ void main() {
final p1 = await packageBackend.lookupPackage('oxygen');
expect(p1!.versionCount, 3);
final tarball = await packageArchiveBytes(
pubspecContent: generatePubspecYaml('oxygen', '3.0.0'));
pubspecContent: generatePubspecYaml('oxygen', '3.0.0'),
changelogContent:
'# Changelog\n\n## v3.0.0\n\nSome bug fixes:\n- one,\n- two\n\n');
final message = await createPubApiClient(authToken: adminClientToken)
.uploadPackageBytes(tarball);
expect(message.success.message, contains('Successfully uploaded'));
Expand All @@ -1210,6 +1214,15 @@ void main() {
expect(email.subject, 'Package uploaded: oxygen 3.0.0');
expect(email.bodyText,
contains('https://pub.dev/packages/oxygen/versions/3.0.0\n'));
expect(
email.bodyText,
contains('\n'
'Excerpt of the changelog:\n'
'```\n'
'Some bug fixes:\n'
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we consider also including the header of the relevant section?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure. For most packages, it will be only the version string. Maybe if the changelog itself is not too long?

'- one,\n'
'- two\n'
'```\n\n'));

await nameTracker.reloadFromDatastore();
final lastPublished =
Expand Down
Loading