From c005d6ebab4071233847c4e2701ac7de038bb5dd Mon Sep 17 00:00:00 2001 From: erezrokah Date: Wed, 11 Jan 2023 13:16:44 +0200 Subject: [PATCH 1/2] feat(docs): Sort tables in docs --- .../TestGeneratePluginDocs-JSON-__tables.json | 106 +++++++++++------- .../TestGeneratePluginDocs-Markdown-README.md | 5 +- ...ocs-Markdown-relation_relation_table_a.md} | 2 +- ...Docs-Markdown-relation_relation_table_b.md | 19 ++++ ...eratePluginDocs-Markdown-relation_table.md | 3 +- plugins/source/docs.go | 12 ++ plugins/source/docs_test.go | 14 ++- 7 files changed, 116 insertions(+), 45 deletions(-) rename plugins/source/.snapshots/{TestGeneratePluginDocs-Markdown-relation_relation_table.md => TestGeneratePluginDocs-Markdown-relation_relation_table_a.md} (91%) create mode 100644 plugins/source/.snapshots/TestGeneratePluginDocs-Markdown-relation_relation_table_b.md diff --git a/plugins/source/.snapshots/TestGeneratePluginDocs-JSON-__tables.json b/plugins/source/.snapshots/TestGeneratePluginDocs-JSON-__tables.json index 9406aa69bd..f6e258c151 100644 --- a/plugins/source/.snapshots/TestGeneratePluginDocs-JSON-__tables.json +++ b/plugins/source/.snapshots/TestGeneratePluginDocs-JSON-__tables.json @@ -1,4 +1,42 @@ [ + { + "name": "incremental_table", + "description": "Description for incremental table", + "columns": [ + { + "name": "_cq_source_name", + "type": "TypeString" + }, + { + "name": "_cq_sync_time", + "type": "TypeTimestamp" + }, + { + "name": "_cq_id", + "type": "TypeUUID" + }, + { + "name": "_cq_parent_id", + "type": "TypeUUID" + }, + { + "name": "int_col", + "type": "TypeInt" + }, + { + "name": "id_col", + "type": "TypeInt", + "is_primary_key": true, + "is_incremental_key": true + }, + { + "name": "id_col2", + "type": "TypeInt", + "is_incremental_key": true + } + ], + "relations": [] + }, { "name": "test_table", "description": "Description for test table", @@ -63,7 +101,35 @@ ], "relations": [ { - "name": "relation_relation_table", + "name": "relation_relation_table_a", + "description": "Description for relational table's relation", + "columns": [ + { + "name": "_cq_source_name", + "type": "TypeString" + }, + { + "name": "_cq_sync_time", + "type": "TypeTimestamp" + }, + { + "name": "_cq_id", + "type": "TypeUUID", + "is_primary_key": true + }, + { + "name": "_cq_parent_id", + "type": "TypeUUID" + }, + { + "name": "string_col", + "type": "TypeString" + } + ], + "relations": [] + }, + { + "name": "relation_relation_table_b", "description": "Description for relational table's relation", "columns": [ { @@ -121,43 +187,5 @@ "relations": [] } ] - }, - { - "name": "incremental_table", - "description": "Description for incremental table", - "columns": [ - { - "name": "_cq_source_name", - "type": "TypeString" - }, - { - "name": "_cq_sync_time", - "type": "TypeTimestamp" - }, - { - "name": "_cq_id", - "type": "TypeUUID" - }, - { - "name": "_cq_parent_id", - "type": "TypeUUID" - }, - { - "name": "int_col", - "type": "TypeInt" - }, - { - "name": "id_col", - "type": "TypeInt", - "is_primary_key": true, - "is_incremental_key": true - }, - { - "name": "id_col2", - "type": "TypeInt", - "is_incremental_key": true - } - ], - "relations": [] } ] diff --git a/plugins/source/.snapshots/TestGeneratePluginDocs-Markdown-README.md b/plugins/source/.snapshots/TestGeneratePluginDocs-Markdown-README.md index 0c717cf7d9..9480a0598a 100644 --- a/plugins/source/.snapshots/TestGeneratePluginDocs-Markdown-README.md +++ b/plugins/source/.snapshots/TestGeneratePluginDocs-Markdown-README.md @@ -2,8 +2,9 @@ ## Tables +- [incremental_table](incremental_table.md) (Incremental) - [test_table](test_table.md) - [relation_table](relation_table.md) - - [relation_relation_table](relation_relation_table.md) + - [relation_relation_table_a](relation_relation_table_a.md) + - [relation_relation_table_b](relation_relation_table_b.md) - [relation_table2](relation_table2.md) -- [incremental_table](incremental_table.md) (Incremental) diff --git a/plugins/source/.snapshots/TestGeneratePluginDocs-Markdown-relation_relation_table.md b/plugins/source/.snapshots/TestGeneratePluginDocs-Markdown-relation_relation_table_a.md similarity index 91% rename from plugins/source/.snapshots/TestGeneratePluginDocs-Markdown-relation_relation_table.md rename to plugins/source/.snapshots/TestGeneratePluginDocs-Markdown-relation_relation_table_a.md index 658e747322..648230a87b 100644 --- a/plugins/source/.snapshots/TestGeneratePluginDocs-Markdown-relation_relation_table.md +++ b/plugins/source/.snapshots/TestGeneratePluginDocs-Markdown-relation_relation_table_a.md @@ -1,4 +1,4 @@ -# Table: relation_relation_table +# Table: relation_relation_table_a Description for relational table's relation diff --git a/plugins/source/.snapshots/TestGeneratePluginDocs-Markdown-relation_relation_table_b.md b/plugins/source/.snapshots/TestGeneratePluginDocs-Markdown-relation_relation_table_b.md new file mode 100644 index 0000000000..c50f3eb44d --- /dev/null +++ b/plugins/source/.snapshots/TestGeneratePluginDocs-Markdown-relation_relation_table_b.md @@ -0,0 +1,19 @@ +# Table: relation_relation_table_b + +Description for relational table's relation + +The primary key for this table is **_cq_id**. + +## Relations + +This table depends on [relation_table](relation_table.md). + +## Columns + +| Name | Type | +| ------------- | ------------- | +|_cq_source_name|String| +|_cq_sync_time|Timestamp| +|_cq_id (PK)|UUID| +|_cq_parent_id|UUID| +|string_col|String| diff --git a/plugins/source/.snapshots/TestGeneratePluginDocs-Markdown-relation_table.md b/plugins/source/.snapshots/TestGeneratePluginDocs-Markdown-relation_table.md index 10561c7bfe..f944640aef 100644 --- a/plugins/source/.snapshots/TestGeneratePluginDocs-Markdown-relation_table.md +++ b/plugins/source/.snapshots/TestGeneratePluginDocs-Markdown-relation_table.md @@ -9,7 +9,8 @@ The primary key for this table is **_cq_id**. This table depends on [test_table](test_table.md). The following tables depend on relation_table: - - [relation_relation_table](relation_relation_table.md) + - [relation_relation_table_a](relation_relation_table_a.md) + - [relation_relation_table_b](relation_relation_table_b.md) ## Columns diff --git a/plugins/source/docs.go b/plugins/source/docs.go index 1a2da92bf0..739ed559f2 100644 --- a/plugins/source/docs.go +++ b/plugins/source/docs.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "regexp" + "sort" "strings" "text/template" @@ -21,6 +22,16 @@ var templatesFS embed.FS var reMatchNewlines = regexp.MustCompile(`\n{3,}`) var reMatchHeaders = regexp.MustCompile(`(#{1,6}.+)\n+`) +func sortTables(tables schema.Tables) { + sort.SliceStable(tables, func(i, j int) bool { + return tables[i].Name < tables[j].Name + }) + + for _, table := range tables { + sortTables(table.Relations) + } +} + // GeneratePluginDocs creates table documentation for the source plugin based on its list of tables func (p *Plugin) GeneratePluginDocs(dir, format string) error { if err := os.MkdirAll(dir, os.ModePerm); err != nil { @@ -28,6 +39,7 @@ func (p *Plugin) GeneratePluginDocs(dir, format string) error { } destination.SetDestinationManagedCqColumns(p.Tables()) + sortTables(p.Tables()) switch format { case "markdown": diff --git a/plugins/source/docs_test.go b/plugins/source/docs_test.go index b7ac907a1b..040ba6b808 100644 --- a/plugins/source/docs_test.go +++ b/plugins/source/docs_test.go @@ -44,7 +44,17 @@ var testTables = []*schema.Table{ }, Relations: []*schema.Table{ { - Name: "relation_relation_table", + Name: "relation_relation_table_b", + Description: "Description for relational table's relation", + Columns: []schema.Column{ + { + Name: "string_col", + Type: schema.TypeString, + }, + }, + }, + { + Name: "relation_relation_table_a", Description: "Description for relational table's relation", Columns: []schema.Column{ { @@ -101,7 +111,7 @@ func TestGeneratePluginDocs(t *testing.T) { t.Fatalf("unexpected error calling GeneratePluginDocs: %v", err) } - expectFiles := []string{"test_table.md", "relation_table.md", "relation_relation_table.md", "incremental_table.md", "README.md"} + expectFiles := []string{"test_table.md", "relation_table.md", "relation_relation_table_a.md", "relation_relation_table_b.md", "incremental_table.md", "README.md"} for _, exp := range expectFiles { t.Run(exp, func(t *testing.T) { output := path.Join(tmpdir, exp) From 07c8b6e7ab0c3d269a621f3f430f723d3448d084 Mon Sep 17 00:00:00 2001 From: erezrokah Date: Wed, 11 Jan 2023 14:40:17 +0200 Subject: [PATCH 2/2] refactor: Copy tables before sort --- plugins/source/docs.go | 31 ++++++++++++------- plugins/source/templates/all_tables.md.go.tpl | 2 +- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/plugins/source/docs.go b/plugins/source/docs.go index 739ed559f2..8d71dfff21 100644 --- a/plugins/source/docs.go +++ b/plugins/source/docs.go @@ -32,6 +32,11 @@ func sortTables(tables schema.Tables) { } } +type templateData struct { + PluginName string + Tables schema.Tables +} + // GeneratePluginDocs creates table documentation for the source plugin based on its list of tables func (p *Plugin) GeneratePluginDocs(dir, format string) error { if err := os.MkdirAll(dir, os.ModePerm); err != nil { @@ -39,13 +44,17 @@ func (p *Plugin) GeneratePluginDocs(dir, format string) error { } destination.SetDestinationManagedCqColumns(p.Tables()) - sortTables(p.Tables()) + sortedTables := make(schema.Tables, 0, len(p.Tables())) + for _, t := range p.Tables() { + sortedTables = append(sortedTables, t.Copy(nil)) + } + sortTables(sortedTables) switch format { case "markdown": - return p.renderTablesAsMarkdown(dir) + return renderTablesAsMarkdown(dir, p.name, sortedTables) case "json": - return p.renderTablesAsJSON(dir) + return renderTablesAsJSON(dir, sortedTables) default: return fmt.Errorf("unsupported format: %v", format) } @@ -65,9 +74,9 @@ type jsonColumn struct { IsIncrementalKey bool `json:"is_incremental_key,omitempty"` } -func (p *Plugin) renderTablesAsJSON(dir string) error { - tables := p.jsonifyTables(p.Tables()) - b, err := json.MarshalIndent(tables, "", " ") +func renderTablesAsJSON(dir string, tables schema.Tables) error { + jsonTables := jsonifyTables(tables) + b, err := json.MarshalIndent(jsonTables, "", " ") if err != nil { return fmt.Errorf("failed to marshal tables as json: %v", err) } @@ -75,7 +84,7 @@ func (p *Plugin) renderTablesAsJSON(dir string) error { return os.WriteFile(outputPath, b, 0644) } -func (p *Plugin) jsonifyTables(tables schema.Tables) []jsonTable { +func jsonifyTables(tables schema.Tables) []jsonTable { jsonTables := make([]jsonTable, len(tables)) for i, table := range tables { jsonColumns := make([]jsonColumn, len(table.Columns)) @@ -91,14 +100,14 @@ func (p *Plugin) jsonifyTables(tables schema.Tables) []jsonTable { Name: table.Name, Description: table.Description, Columns: jsonColumns, - Relations: p.jsonifyTables(table.Relations), + Relations: jsonifyTables(table.Relations), } } return jsonTables } -func (p *Plugin) renderTablesAsMarkdown(dir string) error { - for _, table := range p.Tables() { +func renderTablesAsMarkdown(dir string, pluginName string, tables schema.Tables) error { + for _, table := range tables { if err := renderAllTables(table, dir); err != nil { return err } @@ -111,7 +120,7 @@ func (p *Plugin) renderTablesAsMarkdown(dir string) error { } var b bytes.Buffer - if err := t.Execute(&b, p); err != nil { + if err := t.Execute(&b, templateData{PluginName: pluginName, Tables: tables}); err != nil { return fmt.Errorf("failed to execute template: %v", err) } content := formatMarkdown(b.String()) diff --git a/plugins/source/templates/all_tables.md.go.tpl b/plugins/source/templates/all_tables.md.go.tpl index 618559dcb4..008afb66fd 100644 --- a/plugins/source/templates/all_tables.md.go.tpl +++ b/plugins/source/templates/all_tables.md.go.tpl @@ -1,4 +1,4 @@ -# Source Plugin: {{.Name}} +# Source Plugin: {{.PluginName}} ## Tables {{- range $table := $.Tables }} {{- template "all_tables_entry.md.go.tpl" $table}}