Skip to content

Commit 56d5d0a

Browse files
authored
feat(internal/godocfx): handle Markdown content (#3816)
The first commit copied the go/doc package into `third_party`. Then, I added ToMarkdown to the doc package. I got the idea from @dmitshur, who pointed me to http://golang.org/issues/34875. Once the comment is converted to Markdown, it gets converted to HTML by `goldmark`. Future changes will: * Add syntax highlighting for code blocks. * Maybe add \` tags around code elements in the comment. * Maybe support Markdown in other doc comments.
1 parent 1068f9a commit 56d5d0a

File tree

19 files changed

+4021
-86
lines changed

19 files changed

+4021
-86
lines changed

header_test.go

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,19 @@ func TestLicense(t *testing.T) {
3939
"cmd/go-cloud-debug-agent/internal/debug/elf/elf.go": true,
4040

4141
// From https://github.com/golang/pkgsite.
42-
"third_party/pkgsite/print_type.go": true,
43-
"third_party/pkgsite/synopsis.go": true,
42+
"third_party/pkgsite/print_type.go": true,
43+
"third_party/pkgsite/synopsis.go": true,
44+
"third_party/go/doc/comment.go": true,
45+
"third_party/go/doc/comment_test.go": true,
46+
"third_party/go/doc/doc.go": true,
47+
"third_party/go/doc/example.go": true,
48+
"third_party/go/doc/example_test.go": true,
49+
"third_party/go/doc/exports.go": true,
50+
"third_party/go/doc/filter.go": true,
51+
"third_party/go/doc/headscan.go": true,
52+
"third_party/go/doc/reader.go": true,
53+
"third_party/go/doc/synopsis.go": true,
54+
"third_party/go/doc/synopsis_test.go": true,
4455
}
4556
err := filepath.Walk(".", func(path string, fi os.FileInfo, err error) error {
4657
if err != nil {

internal/godocfx/go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ require (
88
cloud.google.com/go/datastore v1.1.0
99
cloud.google.com/go/storage v1.11.0
1010
github.com/kr/pretty v0.2.1 // indirect
11+
github.com/yuin/goldmark v1.3.2
1112
golang.org/x/mod v0.4.1 // indirect
1213
golang.org/x/tools v0.1.0
1314
gopkg.in/yaml.v2 v2.4.0

internal/godocfx/go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,8 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
9696
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
9797
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
9898
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
99+
github.com/yuin/goldmark v1.3.2 h1:YjHC5TgyMmHpicTgEqDN0Q96Xo8K6tLXPnmNOHXCgs0=
100+
github.com/yuin/goldmark v1.3.2/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
99101
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
100102
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
101103
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=

internal/godocfx/parse.go

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@ import (
2626
"bytes"
2727
"fmt"
2828
"go/ast"
29-
"go/doc"
3029
"go/format"
3130
"go/parser"
3231
"go/printer"
@@ -39,7 +38,10 @@ import (
3938
"strconv"
4039
"strings"
4140

41+
"cloud.google.com/go/third_party/go/doc"
4242
"cloud.google.com/go/third_party/pkgsite"
43+
"github.com/yuin/goldmark"
44+
"github.com/yuin/goldmark/renderer/html"
4345
"golang.org/x/tools/go/packages"
4446
)
4547

@@ -575,8 +577,17 @@ func buildTOC(mod string, pis []pkgInfo, extraFiles []extraFile) tableOfContents
575577

576578
func toHTML(s string) string {
577579
buf := &bytes.Buffer{}
578-
doc.ToHTML(buf, s, nil)
579-
return buf.String()
580+
// First, convert to Markdown.
581+
doc.ToMarkdown(buf, s, nil)
582+
583+
// Then, handle Markdown stuff, like lists and links.
584+
md := goldmark.New(goldmark.WithRendererOptions(html.WithUnsafe()))
585+
mdBuf := &bytes.Buffer{}
586+
if err := md.Convert(buf.Bytes(), mdBuf); err != nil {
587+
panic(err)
588+
}
589+
590+
return mdBuf.String()
580591
}
581592

582593
type pkgInfo struct {

internal/godocfx/testdata/golden/index.yml

Lines changed: 74 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -3,95 +3,91 @@ items:
33
- uid: cloud.google.com/go/storage
44
name: cloud.google.com/go/storage
55
id: storage
6-
summary: "<p>\nPackage storage provides an easy way to work with Google Cloud Storage.\nGoogle
7-
Cloud Storage stores data in named objects, which are grouped into buckets.\n</p>\n<p>\nMore
8-
information about Google Cloud Storage is available at\n<a href=\"https://cloud.google.com/storage/docs\">https://cloud.google.com/storage/docs</a>.\n</p>\n<p>\nSee
9-
<a href=\"https://godoc.org/cloud.google.com/go\">https://godoc.org/cloud.google.com/go</a>
10-
for authentication, timeouts,\nconnection pooling and similar aspects of this
11-
package.\n</p>\n<p>\nAll of the methods of this package use exponential backoff
12-
to retry calls that fail\nwith certain errors, as described in\n<a href=\"https://cloud.google.com/storage/docs/exponential-backoff\">https://cloud.google.com/storage/docs/exponential-backoff</a>.
13-
Retrying continues\nindefinitely unless the controlling context is canceled or
14-
the client is closed. See\ncontext.WithTimeout and context.WithCancel.\n</p>\n<h3
15-
id=\"hdr-Creating_a_Client\">Creating a Client</h3>\n<p>\nTo start working with
16-
this package, create a client:\n</p>\n<pre>ctx := context.Background()\nclient,
17-
err := storage.NewClient(ctx)\nif err != nil {\n // TODO: Handle error.\n}\n</pre>\n<p>\nThe
18-
client will use your default application credentials. Clients should be\nreused
19-
instead of created as needed. The methods of Client are safe for\nconcurrent use
20-
by multiple goroutines.\n</p>\n<p>\nIf you only wish to access public data, you
21-
can create\nan unauthenticated client with\n</p>\n<pre>client, err := storage.NewClient(ctx,
22-
option.WithoutAuthentication())\n</pre>\n<h3 id=\"hdr-Buckets\">Buckets</h3>\n<p>\nA
6+
summary: "<p>Package storage provides an easy way to work with Google Cloud Storage.\nGoogle
7+
Cloud Storage stores data in named objects, which are grouped into buckets.</p>\n<p>More
8+
information about Google Cloud Storage is available at\nhttps://cloud.google.com/storage/docs.</p>\n<p>See
9+
https://godoc.org/cloud.google.com/go for authentication, timeouts,\nconnection
10+
pooling and similar aspects of this package.</p>\n<p>All of the methods of this
11+
package use exponential backoff to retry calls that fail\nwith certain errors,
12+
as described in\nhttps://cloud.google.com/storage/docs/exponential-backoff. Retrying
13+
continues\nindefinitely unless the controlling context is canceled or the client
14+
is closed. See\ncontext.WithTimeout and context.WithCancel.</p>\n<h3>Creating
15+
a Client</h3>\n<p>To start working with this package, create a client:</p>\n<pre><code>ctx
16+
:= context.Background()\nclient, err := storage.NewClient(ctx)\nif err != nil
17+
{\n // TODO: Handle error.\n}\n</code></pre>\n<p>The client will use your default
18+
application credentials. Clients should be\nreused instead of created as needed.
19+
The methods of Client are safe for\nconcurrent use by multiple goroutines.</p>\n<p>If
20+
you only wish to access public data, you can create\nan unauthenticated client
21+
with</p>\n<pre><code>client, err := storage.NewClient(ctx, option.WithoutAuthentication())\n</code></pre>\n<h3>Buckets</h3>\n<p>A
2322
Google Cloud Storage bucket is a collection of objects. To work with a\nbucket,
24-
make a bucket handle:\n</p>\n<pre>bkt := client.Bucket(bucketName)\n</pre>\n<p>\nA
25-
handle is a reference to a bucket. You can have a handle even if the\nbucket doesn&#39;t
26-
exist yet. To create a bucket in Google Cloud Storage,\ncall Create on the handle:\n</p>\n<pre>if
27-
err := bkt.Create(ctx, projectID, nil); err != nil {\n // TODO: Handle error.\n}\n</pre>\n<p>\nNote
23+
make a bucket handle:</p>\n<pre><code>bkt := client.Bucket(bucketName)\n</code></pre>\n<p>A
24+
handle is a reference to a bucket. You can have a handle even if the\nbucket doesn't
25+
exist yet. To create a bucket in Google Cloud Storage,\ncall Create on the handle:</p>\n<pre><code>if
26+
err := bkt.Create(ctx, projectID, nil); err != nil {\n // TODO: Handle error.\n}\n</code></pre>\n<p>Note
2827
that although buckets are associated with projects, bucket names are\nglobal across
29-
all projects.\n</p>\n<p>\nEach bucket has associated metadata, represented in
30-
this package by\nBucketAttrs. The third argument to BucketHandle.Create allows
31-
you to set\nthe initial BucketAttrs of a bucket. To retrieve a bucket&#39;s attributes,
32-
use\nAttrs:\n</p>\n<pre>attrs, err := bkt.Attrs(ctx)\nif err != nil {\n //
33-
TODO: Handle error.\n}\nfmt.Printf(&#34;bucket %s, created at %s, is located in
34-
%s with storage class %s\\n&#34;,\n attrs.Name, attrs.Created, attrs.Location,
35-
attrs.StorageClass)\n</pre>\n<h3 id=\"hdr-Objects\">Objects</h3>\n<p>\nAn object
36-
holds arbitrary data as a sequence of bytes, like a file. You\nrefer to objects
37-
using a handle, just as with buckets, but unlike buckets\nyou don&#39;t explicitly
38-
create an object. Instead, the first time you write\nto an object it will be created.
39-
You can use the standard Go io.Reader\nand io.Writer interfaces to read and write
40-
object data:\n</p>\n<pre>obj := bkt.Object(&#34;data&#34;)\n// Write something
41-
to obj.\n// w implements io.Writer.\nw := obj.NewWriter(ctx)\n// Write some text
42-
to obj. This will either create the object or overwrite whatever is there already.\nif
43-
_, err := fmt.Fprintf(w, &#34;This object contains text.\\n&#34;); err != nil
44-
{\n // TODO: Handle error.\n}\n// Close, just like writing a file.\nif err
45-
:= w.Close(); err != nil {\n // TODO: Handle error.\n}\n\n// Read it back.\nr,
46-
err := obj.NewReader(ctx)\nif err != nil {\n // TODO: Handle error.\n}\ndefer
47-
r.Close()\nif _, err := io.Copy(os.Stdout, r); err != nil {\n // TODO: Handle
48-
error.\n}\n// Prints &#34;This object contains text.&#34;\n</pre>\n<p>\nObjects
49-
also have attributes, which you can fetch with Attrs:\n</p>\n<pre>objAttrs, err
50-
:= obj.Attrs(ctx)\nif err != nil {\n // TODO: Handle error.\n}\nfmt.Printf(&#34;object
51-
%s has size %d and can be read using %s\\n&#34;,\n objAttrs.Name, objAttrs.Size,
52-
objAttrs.MediaLink)\n</pre>\n<h3 id=\"hdr-Listing_objects\">Listing objects</h3>\n<p>\nListing
53-
objects in a bucket is done with the Bucket.Objects method:\n</p>\n<pre>query
54-
:= &amp;storage.Query{Prefix: &#34;&#34;}\n\nvar names []string\nit := bkt.Objects(ctx,
28+
all projects.</p>\n<p>Each bucket has associated metadata, represented in this
29+
package by\nBucketAttrs. The third argument to BucketHandle.Create allows you
30+
to set\nthe initial BucketAttrs of a bucket. To retrieve a bucket's attributes,
31+
use\nAttrs:</p>\n<pre><code>attrs, err := bkt.Attrs(ctx)\nif err != nil {\n //
32+
TODO: Handle error.\n}\nfmt.Printf(&quot;bucket %s, created at %s, is located
33+
in %s with storage class %s\\n&quot;,\n attrs.Name, attrs.Created, attrs.Location,
34+
attrs.StorageClass)\n</code></pre>\n<h3>Objects</h3>\n<p>An object holds arbitrary
35+
data as a sequence of bytes, like a file. You\nrefer to objects using a handle,
36+
just as with buckets, but unlike buckets\nyou don't explicitly create an object.
37+
Instead, the first time you write\nto an object it will be created. You can use
38+
the standard Go io.Reader\nand io.Writer interfaces to read and write object data:</p>\n<pre><code>obj
39+
:= bkt.Object(&quot;data&quot;)\n// Write something to obj.\n// w implements io.Writer.\nw
40+
:= obj.NewWriter(ctx)\n// Write some text to obj. This will either create the
41+
object or overwrite whatever is there already.\nif _, err := fmt.Fprintf(w, &quot;This
42+
object contains text.\\n&quot;); err != nil {\n // TODO: Handle error.\n}\n//
43+
Close, just like writing a file.\nif err := w.Close(); err != nil {\n // TODO:
44+
Handle error.\n}\n\n// Read it back.\nr, err := obj.NewReader(ctx)\nif err !=
45+
nil {\n // TODO: Handle error.\n}\ndefer r.Close()\nif _, err := io.Copy(os.Stdout,
46+
r); err != nil {\n // TODO: Handle error.\n}\n// Prints &quot;This object contains
47+
text.&quot;\n</code></pre>\n<p>Objects also have attributes, which you can fetch
48+
with Attrs:</p>\n<pre><code>objAttrs, err := obj.Attrs(ctx)\nif err != nil {\n
49+
\ // TODO: Handle error.\n}\nfmt.Printf(&quot;object %s has size %d and can
50+
be read using %s\\n&quot;,\n objAttrs.Name, objAttrs.Size, objAttrs.MediaLink)\n</code></pre>\n<h3>Listing
51+
objects</h3>\n<p>Listing objects in a bucket is done with the Bucket.Objects method:</p>\n<pre><code>query
52+
:= &amp;storage.Query{Prefix: &quot;&quot;}\n\nvar names []string\nit := bkt.Objects(ctx,
5553
query)\nfor {\n attrs, err := it.Next()\n if err == iterator.Done {\n break\n
5654
\ }\n if err != nil {\n log.Fatal(err)\n }\n names = append(names,
57-
attrs.Name)\n}\n</pre>\n<p>\nIf only a subset of object attributes is needed when
58-
listing, specifying this\nsubset using Query.SetAttrSelection may speed up the
59-
listing process:\n</p>\n<pre>query := &amp;storage.Query{Prefix: &#34;&#34;}\nquery.SetAttrSelection([]string{&#34;Name&#34;})\n\n//
60-
... as before\n</pre>\n<h3 id=\"hdr-ACLs\">ACLs</h3>\n<p>\nBoth objects and buckets
61-
have ACLs (Access Control Lists). An ACL is a list of\nACLRules, each of which
62-
specifies the role of a user, group or project. ACLs\nare suitable for fine-grained
63-
control, but you may prefer using IAM to control\naccess at the project level
64-
(see\n<a href=\"https://cloud.google.com/storage/docs/access-control/iam\">https://cloud.google.com/storage/docs/access-control/iam</a>).\n</p>\n<p>\nTo
65-
list the ACLs of a bucket or object, obtain an ACLHandle and call its List method:\n</p>\n<pre>acls,
55+
attrs.Name)\n}\n</code></pre>\n<p>If only a subset of object attributes is needed
56+
when listing, specifying this\nsubset using Query.SetAttrSelection may speed up
57+
the listing process:</p>\n<pre><code>query := &amp;storage.Query{Prefix: &quot;&quot;}\nquery.SetAttrSelection([]string{&quot;Name&quot;})\n\n//
58+
... as before\n</code></pre>\n<h3>ACLs</h3>\n<p>Both objects and buckets have
59+
ACLs (Access Control Lists). An ACL is a list of\nACLRules, each of which specifies
60+
the role of a user, group or project. ACLs\nare suitable for fine-grained control,
61+
but you may prefer using IAM to control\naccess at the project level (see\nhttps://cloud.google.com/storage/docs/access-control/iam).</p>\n<p>To
62+
list the ACLs of a bucket or object, obtain an ACLHandle and call its List method:</p>\n<pre><code>acls,
6663
err := obj.ACL().List(ctx)\nif err != nil {\n // TODO: Handle error.\n}\nfor
67-
_, rule := range acls {\n fmt.Printf(&#34;%s has role %s\\n&#34;, rule.Entity,
68-
rule.Role)\n}\n</pre>\n<p>\nYou can also set and delete ACLs.\n</p>\n<h3 id=\"hdr-Conditions\">Conditions</h3>\n<p>\nEvery
64+
_, rule := range acls {\n fmt.Printf(&quot;%s has role %s\\n&quot;, rule.Entity,
65+
rule.Role)\n}\n</code></pre>\n<p>You can also set and delete ACLs.</p>\n<h3>Conditions</h3>\n<p>Every
6966
object has a generation and a metageneration. The generation changes\nwhenever
7067
the content changes, and the metageneration changes whenever the\nmetadata changes.
7168
Conditions let you check these values before an operation;\nthe operation only
7269
executes if the conditions match. You can use conditions to\nprevent race conditions
73-
in read-modify-write operations.\n</p>\n<p>\nFor example, say you&#39;ve read
74-
an object&#39;s metadata into objAttrs. Now\nyou want to write to that object,
75-
but only if its contents haven&#39;t changed\nsince you read it. Here is how to
76-
express that:\n</p>\n<pre>w = obj.If(storage.Conditions{GenerationMatch: objAttrs.Generation}).NewWriter(ctx)\n//
77-
Proceed with writing as above.\n</pre>\n<h3 id=\"hdr-Signed_URLs\">Signed URLs</h3>\n<p>\nYou
78-
can obtain a URL that lets anyone read or write an object for a limited time.\nYou
79-
don&#39;t need to create a client to do this. See the documentation of\nSignedURL
80-
for details.\n</p>\n<pre>url, err := storage.SignedURL(bucketName, &#34;shared-object&#34;,
81-
opts)\nif err != nil {\n // TODO: Handle error.\n}\nfmt.Println(url)\n</pre>\n<h3
82-
id=\"hdr-Post_Policy_V4_Signed_Request\">Post Policy V4 Signed Request</h3>\n<p>\nA
83-
type of signed request that allows uploads through HTML forms directly to Cloud
84-
Storage with\ntemporary permission. Conditions can be applied to restrict how
85-
the HTML form is used and exercised\nby a user.\n</p>\n<p>\nFor more information,
86-
please see <a href=\"https://cloud.google.com/storage/docs/xml-api/post-object\">https://cloud.google.com/storage/docs/xml-api/post-object</a>
87-
as well\nas the documentation of GenerateSignedPostPolicyV4.\n</p>\n<pre>pv4,
70+
in read-modify-write operations.</p>\n<p>For example, say you've read an object's
71+
metadata into objAttrs. Now\nyou want to write to that object, but only if its
72+
contents haven't changed\nsince you read it. Here is how to express that:</p>\n<pre><code>w
73+
= obj.If(storage.Conditions{GenerationMatch: objAttrs.Generation}).NewWriter(ctx)\n//
74+
Proceed with writing as above.\n</code></pre>\n<h3>Signed URLs</h3>\n<p>You can
75+
obtain a URL that lets anyone read or write an object for a limited time.\nYou
76+
don't need to create a client to do this. See the documentation of\nSignedURL
77+
for details.</p>\n<pre><code>url, err := storage.SignedURL(bucketName, &quot;shared-object&quot;,
78+
opts)\nif err != nil {\n // TODO: Handle error.\n}\nfmt.Println(url)\n</code></pre>\n<h3>Post
79+
Policy V4 Signed Request</h3>\n<p>A type of signed request that allows uploads
80+
through HTML forms directly to Cloud Storage with\ntemporary permission. Conditions
81+
can be applied to restrict how the HTML form is used and exercised\nby a user.</p>\n<p>For
82+
more information, please see https://cloud.google.com/storage/docs/xml-api/post-object
83+
as well\nas the documentation of GenerateSignedPostPolicyV4.</p>\n<pre><code>pv4,
8884
err := storage.GenerateSignedPostPolicyV4(bucketName, objectName, opts)\nif err
89-
!= nil {\n // TODO: Handle error.\n}\nfmt.Printf(&#34;URL: %s\\nFields; %v\\n&#34;,
90-
pv4.URL, pv4.Fields)\n</pre>\n<h3 id=\"hdr-Errors\">Errors</h3>\n<p>\nErrors returned
91-
by this client are often of the type [`googleapi.Error`](<a href=\"https://godoc.org/google.golang.org/api/googleapi#Error\">https://godoc.org/google.golang.org/api/googleapi#Error</a>).\nThese
85+
!= nil {\n // TODO: Handle error.\n}\nfmt.Printf(&quot;URL: %s\\nFields; %v\\n&quot;,
86+
pv4.URL, pv4.Fields)\n</code></pre>\n<h3>Errors</h3>\n<p>Errors returned by this
87+
client are often of the type <a href=\"https://godoc.org/google.golang.org/api/googleapi#Error\"><code>googleapi.Error</code></a>.\nThese
9288
errors can be introspected for more information by type asserting to the richer
93-
`googleapi.Error` type. For example:\n</p>\n<pre>if e, ok := err.(*googleapi.Error);
94-
ok {\n\t if e.Code == 409 { ... }\n}\n</pre>\n"
89+
<code>googleapi.Error</code> type. For example:</p>\n<pre><code>if e, ok := err.(*googleapi.Error);
90+
ok {\n\t if e.Code == 409 { ... }\n}\n</code></pre>\n"
9591
type: package
9692
langs:
9793
- go

internal/kokoro/vet.sh

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,8 @@ golint ./... 2>&1 | (
7878
grep -v "a blank import should be only in a main or test package" |
7979
grep -v "method ExecuteSql should be ExecuteSQL" |
8080
grep -vE "spanner/spansql/(sql|types).go:.*should have comment" |
81-
grep -vE "\.pb\.go:"
81+
grep -vE "\.pb\.go:" |
82+
grep -v "third_party/go/doc"
8283
) |
8384
tee /dev/stderr | (! read)
8485

@@ -100,7 +101,8 @@ staticcheck -go 1.11 ./... 2>&1 | (
100101
grep -v bigtable/reader.go |
101102
grep -v internal/btree/btree.go |
102103
grep -v container/apiv1/mock_test.go |
103-
grep -v third_party/pkgsite/synopsis.go
104+
grep -v third_party/pkgsite/synopsis.go |
105+
grep -v third_party/go/doc
104106
) |
105107
tee /dev/stderr | (! read)
106108

third_party/go/doc/Makefile

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# Copyright 2009 The Go Authors. All rights reserved.
2+
# Use of this source code is governed by a BSD-style
3+
# license that can be found in the LICENSE file.
4+
5+
# Script to test heading detection heuristic
6+
headscan: headscan.go
7+
go build headscan.go

0 commit comments

Comments
 (0)