Parse new-style "Nested Schema" sections in markdown docs (#345)

* WIP on parsing newstyle md docs

* Inching closer on parsing param descriptions

* Hack new-style parser into the old parser

* Warn instead of panic

* Cleanup parse_markdown, add doc headers

* Test and clean up a few oddities

* Add tests for nested map parsing

* Complete one TODO

* Also parse top-level Schema

* Handle top-level and nested together

* Fixup description parser

* Skip some more redundant things

* Fix request_max_bytes parsing as request*max*bytes

* Parse top-levels as Attributes not Arguments

* Try to normalize windows line endings to unix before parsing tests

* Fix

* Add CHANGELOG entry

CHANGELOG.md

@@ -3,6 +3,9 @@ CHANGELOG
 
 ## HEAD (Unreleased)
 
+- Parse new-style "Nested Schema" sections in markdown docs
+  [#345](https://github.com/pulumi/pulumi-terraform-bridge/pull/345)
+
 - Adjust input extraction in `Read` s.t. it returns a safer set of values.
   [#330](https://github.com/pulumi/pulumi-terraform-bridge/pull/330)
 
@@ -55,25 +58,25 @@ CHANGELOG
   [#231](https://github.com/pulumi/pulumi-terraform-bridge/pull/231)
 
 - Updating regular expressions for unordered lists in markdown
-  [#234](https://github.com/pulumi/pulumi-terraform-bridge/pull/234) 
-  
+  [#234](https://github.com/pulumi/pulumi-terraform-bridge/pull/234)
+
 - Clean codegen directories before generating new code.
   [#236](https://github.com/pulumi/pulumi-terraform-bridge/pull/236)
 
 - Convert examples using the current schema.
   [#224](https://github.com/pulumi/pulumi-terraform-bridge/pull/224)
-  
+
 - Add check against empty assets.
   [#223](https://github.com/pulumi/pulumi-terraform-bridge/pull/223)
 
 - Add support for generating Go examples.
   [#194](https://github.com/pulumi/pulumi-terraform-bridge/pull/218)
-  
+
 - Add JSON path links to properties when generating doc comments and deprecation messages for
   a Pulumi schema.
   [#202](https://github.com/pulumi/pulumi-terraform-bridge/pull/202)
 
-- Update README generation for python. 
+- Update README generation for python.
   [#217](https://github.com/pulumi/pulumi-terraform-bridge/pull/217)
 
 - Use the schema-based code generator for NodeJS.
@@ -99,7 +102,7 @@ CHANGELOG
 - Add support for generating a new-style Go SDK.
   [#101](https://github.com/pulumi/pulumi-terraform-bridge/pull/101)
 
-- Add support for PEP 561 Python type specs. 
+- Add support for PEP 561 Python type specs.
   [100](https://github.com/pulumi/pulumi-terraform-bridge/pull/100)
 
 - Add support for generating a .NET SDK.

go.mod

@@ -36,6 +36,7 @@ require (
 	github.com/pulumi/pulumi/pkg/v3 v3.0.0
 	github.com/pulumi/pulumi/sdk/v3 v3.0.0
 	github.com/pulumi/terraform-diff-reader v0.0.0-20201211191010-ad4715e9285e
+	github.com/russross/blackfriday/v2 v2.1.0
 	github.com/spf13/afero v1.2.2
 	github.com/spf13/cobra v1.0.0
 	github.com/stretchr/testify v1.6.1

go.sum

@@ -157,7 +157,6 @@ github.com/aws/aws-sdk-go v1.15.78/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3A
 github.com/aws/aws-sdk-go v1.19.39/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.23.20/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.25.3/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
-github.com/aws/aws-sdk-go v1.36.1 h1:rDgSL20giXXu48Ycx6Qa4vWaNTVTltUl6vA73ObCSVk=
 github.com/aws/aws-sdk-go v1.36.1/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
 github.com/aws/aws-sdk-go v1.37.31 h1:eK7hgg1H4xivwopAbnzfQ7ZBbDb9cEkGDivd9rUMnJs=
 github.com/aws/aws-sdk-go v1.37.31/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
@@ -689,22 +688,8 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzr
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
-github.com/pulumi/pulumi/pkg/v2 v2.22.1-0.20210310211618-1f16423ede4c h1:ozWoZFocQfHmPahclxia6CGhP3eZ73XK6xznr4J5/cU=
-github.com/pulumi/pulumi/pkg/v2 v2.22.1-0.20210310211618-1f16423ede4c/go.mod h1:cghYgU9E3gorp3W0gIZshlUib4LtCh36XBTAzLQO7qI=
-github.com/pulumi/pulumi/pkg/v3 v3.0.0-20210323100828-ae9253ac9b76 h1:zCWwJe+6kPDcuotzoW/eR1r31DJm9EGfnNOOlIsAwns=
-github.com/pulumi/pulumi/pkg/v3 v3.0.0-20210323100828-ae9253ac9b76/go.mod h1:eMwJviOF/gL0fubZg6bW5CgleNqjNg90UE/Ww0MPtqg=
-github.com/pulumi/pulumi/pkg/v3 v3.0.0-20210324220902-b543e235f01d h1:1avqGOvRbUcsTMbAjhIdlpSOX23gdLXuBBq0YoHa5I4=
-github.com/pulumi/pulumi/pkg/v3 v3.0.0-20210324220902-b543e235f01d/go.mod h1:eMwJviOF/gL0fubZg6bW5CgleNqjNg90UE/Ww0MPtqg=
-github.com/pulumi/pulumi/pkg/v3 v3.0.0-beta.2 h1:9ZrTeR2CKNmyYjx4GprltxHGMRguno4shKRWedWBRvo=
-github.com/pulumi/pulumi/pkg/v3 v3.0.0-beta.2/go.mod h1:JCl+sZFEEeDQF1CLAYkxmQv7jxGsNVR7Sn2+gXTu/ag=
-github.com/pulumi/pulumi/pkg/v3 v3.0.0-rc.1 h1:Wqtq1v+Q3Jpx8cxDh24iEWvsZ6y6672maA5Kw9pBGU4=
-github.com/pulumi/pulumi/pkg/v3 v3.0.0-rc.1/go.mod h1:Ollz7eH01VTWzhhzwJ7jp01nS1wD4CEgpZltrwymrfs=
 github.com/pulumi/pulumi/pkg/v3 v3.0.0 h1:Wj920eXTkNydNXHX7YQESO2vSuW33X+n1BS4Uywstzs=
 github.com/pulumi/pulumi/pkg/v3 v3.0.0/go.mod h1:XsgC0PpPdjzZAj9YyEMvtU0ei542zOGxS7M1EzFmH84=
-github.com/pulumi/pulumi/sdk/v3 v3.0.0-beta.2 h1:uw3nUFEtygkBTSVjL+kvE12pS8imafKcALPewEFVv1c=
-github.com/pulumi/pulumi/sdk/v3 v3.0.0-beta.2/go.mod h1:GBHyQ7awNQSRmiKp/p8kIKrGrMOZeA/k2czoM/GOqds=
-github.com/pulumi/pulumi/sdk/v3 v3.0.0-rc.1 h1:RJcy2x1UJOsWJOUzDO7yC98yEd5w9lR4shaQD0dhJkE=
-github.com/pulumi/pulumi/sdk/v3 v3.0.0-rc.1/go.mod h1:GBHyQ7awNQSRmiKp/p8kIKrGrMOZeA/k2czoM/GOqds=
 github.com/pulumi/pulumi/sdk/v3 v3.0.0 h1:zkragE05t1Rco/ymfqMU4UXdEmKMmOH0SXhUKQxujxQ=
 github.com/pulumi/pulumi/sdk/v3 v3.0.0/go.mod h1:GBHyQ7awNQSRmiKp/p8kIKrGrMOZeA/k2czoM/GOqds=
 github.com/pulumi/terraform-diff-reader v0.0.0-20201211191010-ad4715e9285e h1:Dik4Qe/+xguB8JagPyXNlbOnRiXGmq/PSPQTGunYnTk=
@@ -717,8 +702,9 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af h1:gu+uRPtBe88sK
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v2.1.0+incompatible h1:j1Wcmh8OrK4Q7GXY+V7SVSY8nUWQxHW5TkBe7YUl+2s=
 github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=

pkg/tfgen/docs.go

@@ -47,6 +47,23 @@ type argumentDocs struct {
 	isNested bool
 }
 
+// Included for testing convenience.
+func (ad argumentDocs) MarshalJSON() ([]byte, error) {
+	j, err := json.Marshal(struct {
+		Description string
+		Arguments   map[string]string
+		IsNested    bool
+	}{
+		Description: ad.description,
+		Arguments:   ad.arguments,
+		IsNested:    ad.isNested,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return j, nil
+}
+
 // entityDocs represents the documentation for a resource or datasource as extracted from TF markdown.
 type entityDocs struct {
 	// Description is the description of the resource
@@ -83,6 +100,20 @@ type entityDocs struct {
 	Import string
 }
 
+func (ed *entityDocs) getOrCreateArgumentDocs(argumentName string) (*argumentDocs, bool) {
+	if ed.Arguments == nil {
+		ed.Arguments = make(map[string]*argumentDocs)
+	}
+	var created bool
+	args, has := ed.Arguments[argumentName]
+	if !has {
+		args = &argumentDocs{arguments: make(map[string]string)}
+		ed.Arguments[argumentName] = args
+		created = true
+	}
+	return args, created
+}
+
 // DocKind indicates what kind of entity's documentation is being requested.
 type DocKind string
 
@@ -398,6 +429,9 @@ func (p *tfMarkdownParser) parse() (entityDocs, error) {
 	// Replace any Windows-style newlines.
 	markdown := strings.Replace(p.markdown, "\r\n", "\n", -1)
 
+	// Replace redundant comment.
+	markdown = strings.Replace(markdown, "<!-- schema generated by tfplugindocs -->", "", -1)
+
 	// Split the sections by H2 topics in the Markdown file.
 	sections := splitGroupLines(markdown, "## ")
 
@@ -524,6 +558,8 @@ func (p *tfMarkdownParser) reformatExamples(sections [][]string) [][]string {
 	return result
 }
 
+var nestedSchemaPattern *regexp.Regexp = regexp.MustCompile("^[#]+ Nested Schema for [`]([^`]+)[`]")
+
 func (p *tfMarkdownParser) parseSection(section []string) error {
 	// Extract the header name, since this will drive how we process the content.
 	if len(section) == 0 {
@@ -538,6 +574,7 @@ func (p *tfMarkdownParser) parseSection(section []string) error {
 	}
 
 	sectionKind := sectionOther
+
 	switch header {
 	case "Timeout", "Timeouts", "User Project Override", "User Project Overrides":
 		p.g.debug("Ignoring doc section [%v] for [%v]", header, p.rawname)
@@ -554,6 +591,9 @@ func (p *tfMarkdownParser) parseSection(section []string) error {
 		sectionKind = sectionImports
 	case "---":
 		sectionKind = sectionFrontMatter
+	case "Schema":
+		p.parseSchemaWithNestedSections(section)
+		return nil
 	}
 
 	// Now split the sections by H3 topics. This is done because we'll ignore sub-sections with code
@@ -620,6 +660,17 @@ func getFooterLinks(markdown string) map[string]string {
 	return links
 }
 
+func (p *tfMarkdownParser) parseSchemaWithNestedSections(subsection []string) {
+	topLevelSchema, err := parseTopLevelSchema(parseNode(strings.Join(subsection, "\n")), nil)
+	if err != nil {
+		panic(err)
+	}
+	if topLevelSchema == nil {
+		panic("Failed to parse top-level Schema section")
+	}
+	parseTopLevelSchemaIntoDocs(&p.ret, topLevelSchema, p.g.warn)
+}
+
 func (p *tfMarkdownParser) parseArgReferenceSection(subsection []string) {
 	var lastMatch, nested string
 	for _, line := range subsection {
@@ -708,6 +759,11 @@ func (p *tfMarkdownParser) parseImports(subsection []string) {
 			continue
 		}
 
+		// Skip another redundant comment
+		if strings.Contains(section, "Import is supported using the following syntax") {
+			continue
+		}
+
 		// There are multiple variations of codeblocks for import syntax
 		section = strings.Replace(section, "```shell", "", -1)
 		section = strings.Replace(section, "```sh", "", -1)
@@ -1121,6 +1177,7 @@ func cleanupDoc(name string, g *Generator, info tfbridge.ResourceOrDataSourceInf
 		newargs[k] = &argumentDocs{
 			description: cleanedText,
 			arguments:   make(map[string]string, len(v.arguments)),
+			isNested:    v.isNested,
 		}
 
 		// Clean nested arguments (if any)