Skip to content

Commit

Permalink
feat(internal/godocfx): handle Markdown content (#3816)
Browse files Browse the repository at this point in the history 
The first commit copied the go/doc package into `third_party`.

Then, I added ToMarkdown to the doc package. I got the idea from @dmitshur,
who pointed me to http://golang.org/issues/34875.

Once the comment is converted to Markdown, it gets converted to HTML by
`goldmark`.

Future changes will:
* Add syntax highlighting for code blocks.
* Maybe add \` tags around code elements in the comment.
* Maybe support Markdown in other doc comments.
  • Loading branch information
@tbpg
tbpg authored Mar 15, 2021
1 parent 1068f9a commit 56d5d0a
Show file tree
Hide file tree
Showing 19 changed files with 4,021 additions and 86 deletions.
15 changes: 13 additions & 2 deletions header_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,19 @@ func TestLicense(t *testing.T) {
"cmd/go-cloud-debug-agent/internal/debug/elf/elf.go": true,

// From https://github.com/golang/pkgsite.
"third_party/pkgsite/print_type.go": true,
"third_party/pkgsite/synopsis.go": true,
"third_party/pkgsite/print_type.go": true,
"third_party/pkgsite/synopsis.go": true,
"third_party/go/doc/comment.go": true,
"third_party/go/doc/comment_test.go": true,
"third_party/go/doc/doc.go": true,
"third_party/go/doc/example.go": true,
"third_party/go/doc/example_test.go": true,
"third_party/go/doc/exports.go": true,
"third_party/go/doc/filter.go": true,
"third_party/go/doc/headscan.go": true,
"third_party/go/doc/reader.go": true,
"third_party/go/doc/synopsis.go": true,
"third_party/go/doc/synopsis_test.go": true,
}
err := filepath.Walk(".", func(path string, fi os.FileInfo, err error) error {
if err != nil {
Expand Down
1 change: 1 addition & 0 deletions internal/godocfx/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ require (
cloud.google.com/go/datastore v1.1.0
cloud.google.com/go/storage v1.11.0
github.com/kr/pretty v0.2.1 // indirect
github.com/yuin/goldmark v1.3.2
golang.org/x/mod v0.4.1 // indirect
golang.org/x/tools v0.1.0
gopkg.in/yaml.v2 v2.4.0
Expand Down
2 changes: 2 additions & 0 deletions internal/godocfx/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,8 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.2 h1:YjHC5TgyMmHpicTgEqDN0Q96Xo8K6tLXPnmNOHXCgs0=
github.com/yuin/goldmark v1.3.2/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
Expand Down
17 changes: 14 additions & 3 deletions internal/godocfx/parse.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ import (
"bytes"
"fmt"
"go/ast"
"go/doc"
"go/format"
"go/parser"
"go/printer"
Expand All @@ -39,7 +38,10 @@ import (
"strconv"
"strings"

"cloud.google.com/go/third_party/go/doc"
"cloud.google.com/go/third_party/pkgsite"
"github.com/yuin/goldmark"
"github.com/yuin/goldmark/renderer/html"
"golang.org/x/tools/go/packages"
)

Expand Down Expand Up @@ -575,8 +577,17 @@ func buildTOC(mod string, pis []pkgInfo, extraFiles []extraFile) tableOfContents

func toHTML(s string) string {
buf := &bytes.Buffer{}
doc.ToHTML(buf, s, nil)
return buf.String()
// First, convert to Markdown.
doc.ToMarkdown(buf, s, nil)

// Then, handle Markdown stuff, like lists and links.
md := goldmark.New(goldmark.WithRendererOptions(html.WithUnsafe()))
mdBuf := &bytes.Buffer{}
if err := md.Convert(buf.Bytes(), mdBuf); err != nil {
panic(err)
}

return mdBuf.String()
}

type pkgInfo struct {
Expand Down
152 changes: 74 additions & 78 deletions internal/godocfx/testdata/golden/index.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,95 +3,91 @@ items:
- uid: cloud.google.com/go/storage
name: cloud.google.com/go/storage
id: storage
summary: "<p>\nPackage storage provides an easy way to work with Google Cloud Storage.\nGoogle
Cloud Storage stores data in named objects, which are grouped into buckets.\n</p>\n<p>\nMore
information about Google Cloud Storage is available at\n<a href=\"https://cloud.google.com/storage/docs\">https://cloud.google.com/storage/docs</a>.\n</p>\n<p>\nSee
<a href=\"https://godoc.org/cloud.google.com/go\">https://godoc.org/cloud.google.com/go</a>
for authentication, timeouts,\nconnection pooling and similar aspects of this
package.\n</p>\n<p>\nAll of the methods of this package use exponential backoff
to retry calls that fail\nwith certain errors, as described in\n<a href=\"https://cloud.google.com/storage/docs/exponential-backoff\">https://cloud.google.com/storage/docs/exponential-backoff</a>.
Retrying continues\nindefinitely unless the controlling context is canceled or
the client is closed. See\ncontext.WithTimeout and context.WithCancel.\n</p>\n<h3
id=\"hdr-Creating_a_Client\">Creating a Client</h3>\n<p>\nTo start working with
this package, create a client:\n</p>\n<pre>ctx := context.Background()\nclient,
err := storage.NewClient(ctx)\nif err != nil {\n // TODO: Handle error.\n}\n</pre>\n<p>\nThe
client will use your default application credentials. Clients should be\nreused
instead of created as needed. The methods of Client are safe for\nconcurrent use
by multiple goroutines.\n</p>\n<p>\nIf you only wish to access public data, you
can create\nan unauthenticated client with\n</p>\n<pre>client, err := storage.NewClient(ctx,
option.WithoutAuthentication())\n</pre>\n<h3 id=\"hdr-Buckets\">Buckets</h3>\n<p>\nA
summary: "<p>Package storage provides an easy way to work with Google Cloud Storage.\nGoogle
Cloud Storage stores data in named objects, which are grouped into buckets.</p>\n<p>More
information about Google Cloud Storage is available at\nhttps://cloud.google.com/storage/docs.</p>\n<p>See
https://godoc.org/cloud.google.com/go for authentication, timeouts,\nconnection
pooling and similar aspects of this package.</p>\n<p>All of the methods of this
package use exponential backoff to retry calls that fail\nwith certain errors,
as described in\nhttps://cloud.google.com/storage/docs/exponential-backoff. Retrying
continues\nindefinitely unless the controlling context is canceled or the client
is closed. See\ncontext.WithTimeout and context.WithCancel.</p>\n<h3>Creating
a Client</h3>\n<p>To start working with this package, create a client:</p>\n<pre><code>ctx
:= context.Background()\nclient, err := storage.NewClient(ctx)\nif err != nil
{\n // TODO: Handle error.\n}\n</code></pre>\n<p>The client will use your default
application credentials. Clients should be\nreused instead of created as needed.
The methods of Client are safe for\nconcurrent use by multiple goroutines.</p>\n<p>If
you only wish to access public data, you can create\nan unauthenticated client
with</p>\n<pre><code>client, err := storage.NewClient(ctx, option.WithoutAuthentication())\n</code></pre>\n<h3>Buckets</h3>\n<p>A
Google Cloud Storage bucket is a collection of objects. To work with a\nbucket,
make a bucket handle:\n</p>\n<pre>bkt := client.Bucket(bucketName)\n</pre>\n<p>\nA
handle is a reference to a bucket. You can have a handle even if the\nbucket doesn&#39;t
exist yet. To create a bucket in Google Cloud Storage,\ncall Create on the handle:\n</p>\n<pre>if
err := bkt.Create(ctx, projectID, nil); err != nil {\n // TODO: Handle error.\n}\n</pre>\n<p>\nNote
make a bucket handle:</p>\n<pre><code>bkt := client.Bucket(bucketName)\n</code></pre>\n<p>A
handle is a reference to a bucket. You can have a handle even if the\nbucket doesn't
exist yet. To create a bucket in Google Cloud Storage,\ncall Create on the handle:</p>\n<pre><code>if
err := bkt.Create(ctx, projectID, nil); err != nil {\n // TODO: Handle error.\n}\n</code></pre>\n<p>Note
that although buckets are associated with projects, bucket names are\nglobal across
all projects.\n</p>\n<p>\nEach bucket has associated metadata, represented in
this package by\nBucketAttrs. The third argument to BucketHandle.Create allows
you to set\nthe initial BucketAttrs of a bucket. To retrieve a bucket&#39;s attributes,
use\nAttrs:\n</p>\n<pre>attrs, err := bkt.Attrs(ctx)\nif err != nil {\n //
TODO: Handle error.\n}\nfmt.Printf(&#34;bucket %s, created at %s, is located in
%s with storage class %s\\n&#34;,\n attrs.Name, attrs.Created, attrs.Location,
attrs.StorageClass)\n</pre>\n<h3 id=\"hdr-Objects\">Objects</h3>\n<p>\nAn object
holds arbitrary data as a sequence of bytes, like a file. You\nrefer to objects
using a handle, just as with buckets, but unlike buckets\nyou don&#39;t explicitly
create an object. Instead, the first time you write\nto an object it will be created.
You can use the standard Go io.Reader\nand io.Writer interfaces to read and write
object data:\n</p>\n<pre>obj := bkt.Object(&#34;data&#34;)\n// Write something
to obj.\n// w implements io.Writer.\nw := obj.NewWriter(ctx)\n// Write some text
to obj. This will either create the object or overwrite whatever is there already.\nif
_, err := fmt.Fprintf(w, &#34;This object contains text.\\n&#34;); err != nil
{\n // TODO: Handle error.\n}\n// Close, just like writing a file.\nif err
:= w.Close(); err != nil {\n // TODO: Handle error.\n}\n\n// Read it back.\nr,
err := obj.NewReader(ctx)\nif err != nil {\n // TODO: Handle error.\n}\ndefer
r.Close()\nif _, err := io.Copy(os.Stdout, r); err != nil {\n // TODO: Handle
error.\n}\n// Prints &#34;This object contains text.&#34;\n</pre>\n<p>\nObjects
also have attributes, which you can fetch with Attrs:\n</p>\n<pre>objAttrs, err
:= obj.Attrs(ctx)\nif err != nil {\n // TODO: Handle error.\n}\nfmt.Printf(&#34;object
%s has size %d and can be read using %s\\n&#34;,\n objAttrs.Name, objAttrs.Size,
objAttrs.MediaLink)\n</pre>\n<h3 id=\"hdr-Listing_objects\">Listing objects</h3>\n<p>\nListing
objects in a bucket is done with the Bucket.Objects method:\n</p>\n<pre>query
:= &amp;storage.Query{Prefix: &#34;&#34;}\n\nvar names []string\nit := bkt.Objects(ctx,
all projects.</p>\n<p>Each bucket has associated metadata, represented in this
package by\nBucketAttrs. The third argument to BucketHandle.Create allows you
to set\nthe initial BucketAttrs of a bucket. To retrieve a bucket's attributes,
use\nAttrs:</p>\n<pre><code>attrs, err := bkt.Attrs(ctx)\nif err != nil {\n //
TODO: Handle error.\n}\nfmt.Printf(&quot;bucket %s, created at %s, is located
in %s with storage class %s\\n&quot;,\n attrs.Name, attrs.Created, attrs.Location,
attrs.StorageClass)\n</code></pre>\n<h3>Objects</h3>\n<p>An object holds arbitrary
data as a sequence of bytes, like a file. You\nrefer to objects using a handle,
just as with buckets, but unlike buckets\nyou don't explicitly create an object.
Instead, the first time you write\nto an object it will be created. You can use
the standard Go io.Reader\nand io.Writer interfaces to read and write object data:</p>\n<pre><code>obj
:= bkt.Object(&quot;data&quot;)\n// Write something to obj.\n// w implements io.Writer.\nw
:= obj.NewWriter(ctx)\n// Write some text to obj. This will either create the
object or overwrite whatever is there already.\nif _, err := fmt.Fprintf(w, &quot;This
object contains text.\\n&quot;); err != nil {\n // TODO: Handle error.\n}\n//
Close, just like writing a file.\nif err := w.Close(); err != nil {\n // TODO:
Handle error.\n}\n\n// Read it back.\nr, err := obj.NewReader(ctx)\nif err !=
nil {\n // TODO: Handle error.\n}\ndefer r.Close()\nif _, err := io.Copy(os.Stdout,
r); err != nil {\n // TODO: Handle error.\n}\n// Prints &quot;This object contains
text.&quot;\n</code></pre>\n<p>Objects also have attributes, which you can fetch
with Attrs:</p>\n<pre><code>objAttrs, err := obj.Attrs(ctx)\nif err != nil {\n
\ // TODO: Handle error.\n}\nfmt.Printf(&quot;object %s has size %d and can
be read using %s\\n&quot;,\n objAttrs.Name, objAttrs.Size, objAttrs.MediaLink)\n</code></pre>\n<h3>Listing
objects</h3>\n<p>Listing objects in a bucket is done with the Bucket.Objects method:</p>\n<pre><code>query
:= &amp;storage.Query{Prefix: &quot;&quot;}\n\nvar names []string\nit := bkt.Objects(ctx,
query)\nfor {\n attrs, err := it.Next()\n if err == iterator.Done {\n break\n
\ }\n if err != nil {\n log.Fatal(err)\n }\n names = append(names,
attrs.Name)\n}\n</pre>\n<p>\nIf only a subset of object attributes is needed when
listing, specifying this\nsubset using Query.SetAttrSelection may speed up the
listing process:\n</p>\n<pre>query := &amp;storage.Query{Prefix: &#34;&#34;}\nquery.SetAttrSelection([]string{&#34;Name&#34;})\n\n//
... as before\n</pre>\n<h3 id=\"hdr-ACLs\">ACLs</h3>\n<p>\nBoth objects and buckets
have ACLs (Access Control Lists). An ACL is a list of\nACLRules, each of which
specifies the role of a user, group or project. ACLs\nare suitable for fine-grained
control, but you may prefer using IAM to control\naccess at the project level
(see\n<a href=\"https://cloud.google.com/storage/docs/access-control/iam\">https://cloud.google.com/storage/docs/access-control/iam</a>).\n</p>\n<p>\nTo
list the ACLs of a bucket or object, obtain an ACLHandle and call its List method:\n</p>\n<pre>acls,
attrs.Name)\n}\n</code></pre>\n<p>If only a subset of object attributes is needed
when listing, specifying this\nsubset using Query.SetAttrSelection may speed up
the listing process:</p>\n<pre><code>query := &amp;storage.Query{Prefix: &quot;&quot;}\nquery.SetAttrSelection([]string{&quot;Name&quot;})\n\n//
... as before\n</code></pre>\n<h3>ACLs</h3>\n<p>Both objects and buckets have
ACLs (Access Control Lists). An ACL is a list of\nACLRules, each of which specifies
the role of a user, group or project. ACLs\nare suitable for fine-grained control,
but you may prefer using IAM to control\naccess at the project level (see\nhttps://cloud.google.com/storage/docs/access-control/iam).</p>\n<p>To
list the ACLs of a bucket or object, obtain an ACLHandle and call its List method:</p>\n<pre><code>acls,
err := obj.ACL().List(ctx)\nif err != nil {\n // TODO: Handle error.\n}\nfor
_, rule := range acls {\n fmt.Printf(&#34;%s has role %s\\n&#34;, rule.Entity,
rule.Role)\n}\n</pre>\n<p>\nYou can also set and delete ACLs.\n</p>\n<h3 id=\"hdr-Conditions\">Conditions</h3>\n<p>\nEvery
_, rule := range acls {\n fmt.Printf(&quot;%s has role %s\\n&quot;, rule.Entity,
rule.Role)\n}\n</code></pre>\n<p>You can also set and delete ACLs.</p>\n<h3>Conditions</h3>\n<p>Every
object has a generation and a metageneration. The generation changes\nwhenever
the content changes, and the metageneration changes whenever the\nmetadata changes.
Conditions let you check these values before an operation;\nthe operation only
executes if the conditions match. You can use conditions to\nprevent race conditions
in read-modify-write operations.\n</p>\n<p>\nFor example, say you&#39;ve read
an object&#39;s metadata into objAttrs. Now\nyou want to write to that object,
but only if its contents haven&#39;t changed\nsince you read it. Here is how to
express that:\n</p>\n<pre>w = obj.If(storage.Conditions{GenerationMatch: objAttrs.Generation}).NewWriter(ctx)\n//
Proceed with writing as above.\n</pre>\n<h3 id=\"hdr-Signed_URLs\">Signed URLs</h3>\n<p>\nYou
can obtain a URL that lets anyone read or write an object for a limited time.\nYou
don&#39;t need to create a client to do this. See the documentation of\nSignedURL
for details.\n</p>\n<pre>url, err := storage.SignedURL(bucketName, &#34;shared-object&#34;,
opts)\nif err != nil {\n // TODO: Handle error.\n}\nfmt.Println(url)\n</pre>\n<h3
id=\"hdr-Post_Policy_V4_Signed_Request\">Post Policy V4 Signed Request</h3>\n<p>\nA
type of signed request that allows uploads through HTML forms directly to Cloud
Storage with\ntemporary permission. Conditions can be applied to restrict how
the HTML form is used and exercised\nby a user.\n</p>\n<p>\nFor more information,
please see <a href=\"https://cloud.google.com/storage/docs/xml-api/post-object\">https://cloud.google.com/storage/docs/xml-api/post-object</a>
as well\nas the documentation of GenerateSignedPostPolicyV4.\n</p>\n<pre>pv4,
in read-modify-write operations.</p>\n<p>For example, say you've read an object's
metadata into objAttrs. Now\nyou want to write to that object, but only if its
contents haven't changed\nsince you read it. Here is how to express that:</p>\n<pre><code>w
= obj.If(storage.Conditions{GenerationMatch: objAttrs.Generation}).NewWriter(ctx)\n//
Proceed with writing as above.\n</code></pre>\n<h3>Signed URLs</h3>\n<p>You can
obtain a URL that lets anyone read or write an object for a limited time.\nYou
don't need to create a client to do this. See the documentation of\nSignedURL
for details.</p>\n<pre><code>url, err := storage.SignedURL(bucketName, &quot;shared-object&quot;,
opts)\nif err != nil {\n // TODO: Handle error.\n}\nfmt.Println(url)\n</code></pre>\n<h3>Post
Policy V4 Signed Request</h3>\n<p>A type of signed request that allows uploads
through HTML forms directly to Cloud Storage with\ntemporary permission. Conditions
can be applied to restrict how the HTML form is used and exercised\nby a user.</p>\n<p>For
more information, please see https://cloud.google.com/storage/docs/xml-api/post-object
as well\nas the documentation of GenerateSignedPostPolicyV4.</p>\n<pre><code>pv4,
err := storage.GenerateSignedPostPolicyV4(bucketName, objectName, opts)\nif err
!= nil {\n // TODO: Handle error.\n}\nfmt.Printf(&#34;URL: %s\\nFields; %v\\n&#34;,
pv4.URL, pv4.Fields)\n</pre>\n<h3 id=\"hdr-Errors\">Errors</h3>\n<p>\nErrors returned
by this client are often of the type [`googleapi.Error`](<a href=\"https://godoc.org/google.golang.org/api/googleapi#Error\">https://godoc.org/google.golang.org/api/googleapi#Error</a>).\nThese
!= nil {\n // TODO: Handle error.\n}\nfmt.Printf(&quot;URL: %s\\nFields; %v\\n&quot;,
pv4.URL, pv4.Fields)\n</code></pre>\n<h3>Errors</h3>\n<p>Errors returned by this
client are often of the type <a href=\"https://godoc.org/google.golang.org/api/googleapi#Error\"><code>googleapi.Error</code></a>.\nThese
errors can be introspected for more information by type asserting to the richer
`googleapi.Error` type. For example:\n</p>\n<pre>if e, ok := err.(*googleapi.Error);
ok {\n\t if e.Code == 409 { ... }\n}\n</pre>\n"
<code>googleapi.Error</code> type. For example:</p>\n<pre><code>if e, ok := err.(*googleapi.Error);
ok {\n\t if e.Code == 409 { ... }\n}\n</code></pre>\n"
type: package
langs:
- go
Expand Down
6 changes: 4 additions & 2 deletions internal/kokoro/vet.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,8 @@ golint ./... 2>&1 | (
grep -v "a blank import should be only in a main or test package" |
grep -v "method ExecuteSql should be ExecuteSQL" |
grep -vE "spanner/spansql/(sql|types).go:.*should have comment" |
grep -vE "\.pb\.go:"
grep -vE "\.pb\.go:" |
grep -v "third_party/go/doc"
) |
tee /dev/stderr | (! read)

Expand All @@ -100,7 +101,8 @@ staticcheck -go 1.11 ./... 2>&1 | (
grep -v bigtable/reader.go |
grep -v internal/btree/btree.go |
grep -v container/apiv1/mock_test.go |
grep -v third_party/pkgsite/synopsis.go
grep -v third_party/pkgsite/synopsis.go |
grep -v third_party/go/doc
) |
tee /dev/stderr | (! read)

Expand Down
7 changes: 7 additions & 0 deletions third_party/go/doc/Makefile
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
# Copyright 2009 The Go Authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.

# Script to test heading detection heuristic
headscan: headscan.go
go build headscan.go
Loading

0 comments on commit 56d5d0a

Please sign in to comment.