relnote: add Merge function

Add a function that merges a tree of markdown files into a single file.

For golang/go#64169.

Change-Id: Ie3200d6cbe0e65f9c878de92c2d812b0ffbccc83
Reviewed-on: https://go-review.googlesource.com/c/build/+/556159
Reviewed-by: Dmitri Shuralyov <dmitshur@golang.org>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Dmitri Shuralyov <dmitshur@google.com>

go.mod

@@ -70,7 +70,7 @@ require (
 	google.golang.org/grpc v1.58.2
 	google.golang.org/protobuf v1.31.0
 	gopkg.in/inf.v0 v0.9.1
-	rsc.io/markdown v0.0.0-20231215200646-988871efbd85
+	rsc.io/markdown v0.0.0-20240117044121-669d2fdf1650
 )
 
 require (

go.sum

@@ -1434,8 +1434,8 @@ modernc.org/z v1.0.1-0.20210308123920-1f282aa71362/go.mod h1:8/SRk5C/HgiQWCgXdfp
 modernc.org/z v1.0.1/go.mod h1:8/SRk5C/HgiQWCgXdfpb+1RvhORdkz5sw72d3jjtyqA=
 modernc.org/zappy v1.0.0/go.mod h1:hHe+oGahLVII/aTTyWK/b53VDHMAGCBYYeZ9sn83HC4=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
-rsc.io/markdown v0.0.0-20231215200646-988871efbd85 h1:Dxuj19cMfSZFI3G8f4Q59tfHqctUo9zE94x1N2k6JTQ=
-rsc.io/markdown v0.0.0-20231215200646-988871efbd85/go.mod h1:8xcPgWmwlZONN1D9bjxtHEjrUtSEa3fakVF8iaewYKQ=
+rsc.io/markdown v0.0.0-20240117044121-669d2fdf1650 h1:fuOABZYWclLVNotDsHVaFixLdtoC7+UQZJ0KSC1ocm0=
+rsc.io/markdown v0.0.0-20240117044121-669d2fdf1650/go.mod h1:8xcPgWmwlZONN1D9bjxtHEjrUtSEa3fakVF8iaewYKQ=
 rsc.io/pdf v0.1.1 h1:k1MczvYDUvJBe93bYd7wrZLLUEcLZAuF824/I4e5Xr4=
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=

relnote/relnote.go

@@ -29,6 +29,8 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
+	"slices"
 	"strings"
 
 	md "rsc.io/markdown"
@@ -138,3 +140,111 @@ func inlineText(ins []md.Inline) string {
 	}
 	return buf.String()
 }
+
+// Merge combines the markdown documents (files ending in ".md") in the tree rooted
+// at fs into a single document.
+// The blocks of the documents are concatenated in lexicographic order by filename.
+// Heading with no content are removed.
+// The link keys must be unique, and are combined into a single map.
+func Merge(fsys fs.FS) (*md.Document, error) {
+	filenames, err := sortedMarkdownFilenames(fsys)
+	if err != nil {
+		return nil, err
+	}
+	doc := &md.Document{}
+	for _, filename := range filenames {
+		fd, err := parseFile(fsys, filename)
+		if err != nil {
+			return nil, err
+		}
+		if len(fd.Blocks) == 0 {
+			continue
+		}
+		if len(doc.Blocks) > 0 {
+			// Put a blank line between the current and new blocks.
+			lastLine := lastBlock(doc).Pos().EndLine
+			delta := lastLine + 2 - fd.Blocks[0].Pos().StartLine
+			for _, b := range fd.Blocks {
+				addLines(b, delta)
+			}
+		}
+		doc.Blocks = append(doc.Blocks, fd.Blocks...)
+		// TODO(jba): merge links
+		// TODO(jba): add headings for package sections under "Minor changes to the library".
+	}
+	// TODO(jba): remove headings with empty contents
+	return doc, nil
+}
+
+func sortedMarkdownFilenames(fsys fs.FS) ([]string, error) {
+	var filenames []string
+	err := fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if !d.IsDir() && strings.HasSuffix(path, ".md") {
+			filenames = append(filenames, path)
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	// '.' comes before '/', which comes before alphanumeric characters.
+	// So just sorting the list will put a filename like "net.md" before
+	// the directory "net". That is what we want.
+	slices.Sort(filenames)
+	return filenames, nil
+}
+
+// lastBlock returns the last block in the document.
+// It panics if the document has no blocks.
+func lastBlock(doc *md.Document) md.Block {
+	return doc.Blocks[len(doc.Blocks)-1]
+}
+
+func addLines(b md.Block, n int) {
+	pos := position(b)
+	pos.StartLine += n
+	pos.EndLine += n
+}
+
+func position(b md.Block) *md.Position {
+	switch b := b.(type) {
+	case *md.Heading:
+		return &b.Position
+	case *md.Text:
+		return &b.Position
+	case *md.CodeBlock:
+		return &b.Position
+	case *md.HTMLBlock:
+		return &b.Position
+	case *md.List:
+		return &b.Position
+	case *md.Item:
+		return &b.Position
+	case *md.Empty:
+		return &b.Position
+	case *md.Paragraph:
+		return &b.Position
+	case *md.Quote:
+		return &b.Position
+	default:
+		panic(fmt.Sprintf("unknown block type %T", b))
+	}
+}
+
+func parseFile(fsys fs.FS, path string) (*md.Document, error) {
+	f, err := fsys.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	data, err := io.ReadAll(f)
+	if err != nil {
+		return nil, err
+	}
+	in := string(data)
+	doc := NewParser().Parse(in)
+	return doc, nil
+}

relnote/relnote_test.go

@@ -5,8 +5,17 @@
 package relnote
 
 import (
+	"fmt"
+	"io/fs"
+	"path/filepath"
+	"slices"
 	"strings"
 	"testing"
+	"testing/fstest"
+
+	"github.com/google/go-cmp/cmp"
+	"golang.org/x/tools/txtar"
+	md "rsc.io/markdown"
 )
 
 func TestCheckFragment(t *testing.T) {
@@ -71,3 +80,69 @@ func TestCheckFragment(t *testing.T) {
 		}
 	}
 }
+
+func TestMerge(t *testing.T) {
+	testFiles, err := filepath.Glob(filepath.Join("testdata", "*.txt"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, f := range testFiles {
+		t.Run(strings.TrimSuffix(filepath.Base(f), ".txt"), func(t *testing.T) {
+			fsys, want, err := parseTestFile(f)
+			if err != nil {
+				t.Fatal(err)
+			}
+			gotDoc, err := Merge(fsys)
+			if err != nil {
+				t.Fatal(err)
+			}
+			got := md.ToMarkdown(gotDoc)
+			if diff := cmp.Diff(want, got); diff != "" {
+				t.Errorf("mismatch (-want, +got)\n%s", diff)
+			}
+		})
+	}
+}
+
+func parseTestFile(filename string) (fsys fs.FS, want string, err error) {
+	ar, err := txtar.ParseFile(filename)
+	if err != nil {
+		return nil, "", err
+	}
+	mfs := make(fstest.MapFS)
+	for _, f := range ar.Files {
+		if f.Name == "want" {
+			want = string(f.Data)
+		} else {
+			mfs[f.Name] = &fstest.MapFile{Data: f.Data}
+		}
+	}
+	if want == "" {
+		return nil, "", fmt.Errorf("%s: missing 'want'", filename)
+	}
+	return mfs, want, nil
+}
+
+func TestSortedMarkdownFilenames(t *testing.T) {
+	want := []string{
+		"a.md",
+		"b.md",
+		"b/a.md",
+		"b/c.md",
+		"ba/a.md",
+	}
+	mfs := make(fstest.MapFS)
+	for _, fn := range want {
+		mfs[fn] = &fstest.MapFile{}
+	}
+	mfs["README"] = &fstest.MapFile{}
+	mfs["b/other.txt"] = &fstest.MapFile{}
+	got, err := sortedMarkdownFilenames(mfs)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !slices.Equal(got, want) {
+		t.Errorf("\ngot  %v\nwant %v", got, want)
+	}
+
+}