feat(analyzer): Analyze queries using a running PostgreSQL database (#2805)

* feat(analyzer): Analyze queries using a running PostgreSQL database

94 of the open issues on sqlc are related to the analyzer. There are
many cases where the current analyzer produces false positives or
false negatives.

sqlx and some other projects have proven that it's possible to extract
query metadata from a running database.

This approach is a bit different, in that the database analysis is
layered on top of the existing query analyzer. We use the new analysis
to provide better type information and support a wider set of cases when
the existing analyzer fails.

* test(analyzer): Update endtoend tests for new analyzer

Add a `contexts` key to exec.json to opt certain tests into or out of
database-backed analysis.

Fix many incorrect test cases that didn't run against an actual
database.

Author: kyleconroy

examples/authors/sqlc.yaml

@@ -7,6 +7,8 @@ sql:
   engine: postgresql
   database:
     managed: true
+  analyzer:
+    database: false
   rules:
   - sqlc/db-prepare
   - postgresql-query-too-costly

examples/booktest/sqlc.json

@@ -13,6 +13,9 @@
       "database": {
         "managed": true
       },
+      "analyzer": {
+        "database": false
+      },
       "rules": [
         "sqlc/db-prepare"
       ]

examples/jets/sqlc.json

@@ -13,6 +13,9 @@
       "database": {
         "managed": true
       },
+      "analyzer": {
+        "database": false
+      },
       "rules": [
         "sqlc/db-prepare"
       ]

examples/ondeck/sqlc.json

@@ -13,6 +13,9 @@
       "database": {
         "managed": true
       },
+      "analyzer": {
+        "database": false
+      },
       "rules": [
         "sqlc/db-prepare"
       ],
@@ -21,7 +24,7 @@
       "emit_interface": true
     },
     {
-      "path": "mysql",
+    "path": "mysql",
       "name": "ondeck",
       "schema": "mysql/schema",
       "queries": "mysql/query",

go.mod

@@ -29,6 +29,7 @@ require (
 
 require (
 	github.com/benbjohnson/clock v1.3.5 // indirect
+	github.com/jackc/puddle/v2 v2.2.1 // indirect
 	github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect

go.sum

@@ -100,7 +100,10 @@ github.com/jackc/pgx/v5 v5.4.3/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSlj
 github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jackc/puddle v1.3.0 h1:eHK/5clGOatcjX3oWGBO/MpxpbHzSwud5EWTSCI+MX0=
 github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
+github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=

internal/analyzer/analyzer.go

@@ -0,0 +1,46 @@
+package analyzer
+
+import (
+	"context"
+
+	"github.com/sqlc-dev/sqlc/internal/sql/ast"
+	"github.com/sqlc-dev/sqlc/internal/sql/named"
+)
+
+type Column struct {
+	Name         string
+	OriginalName string
+	DataType     string
+	NotNull      bool
+	Unsigned     bool
+	IsArray      bool
+	ArrayDims    int
+	Comment      string
+	Length       *int
+	IsNamedParam bool
+	IsFuncCall   bool
+
+	// XXX: Figure out what PostgreSQL calls `foo.id`
+	Scope      string
+	Table      *ast.TableName
+	TableAlias string
+	Type       *ast.TypeName
+	EmbedTable *ast.TableName
+
+	IsSqlcSlice bool // is this sqlc.slice()
+}
+
+type Parameter struct {
+	Number int
+	Column *Column
+}
+
+type Analysis struct {
+	Columns []Column
+	Params  []Parameter
+}
+
+type Analyzer interface {
+	Analyze(context.Context, ast.Node, string, []string, *named.ParamSet) (*Analysis, error)
+	Close(context.Context) error
+}

internal/cmd/cmd.go

@@ -197,7 +197,10 @@ var genCmd = &cobra.Command{
 		defer trace.StartRegion(cmd.Context(), "generate").End()
 		stderr := cmd.ErrOrStderr()
 		dir, name := getConfigPath(stderr, cmd.Flag("file"))
-		output, err := Generate(cmd.Context(), ParseEnv(cmd), dir, name, stderr)
+		output, err := Generate(cmd.Context(), dir, name, &Options{
+			Env:    ParseEnv(cmd),
+			Stderr: stderr,
+		})
 		if err != nil {
 			os.Exit(1)
 		}
@@ -219,7 +222,11 @@ var uploadCmd = &cobra.Command{
 	RunE: func(cmd *cobra.Command, args []string) error {
 		stderr := cmd.ErrOrStderr()
 		dir, name := getConfigPath(stderr, cmd.Flag("file"))
-		if err := createPkg(cmd.Context(), ParseEnv(cmd), dir, name, stderr); err != nil {
+		opts := &Options{
+			Env:    ParseEnv(cmd),
+			Stderr: stderr,
+		}
+		if err := createPkg(cmd.Context(), dir, name, opts); err != nil {
 			fmt.Fprintf(stderr, "error uploading: %s\n", err)
 			os.Exit(1)
 		}
@@ -234,7 +241,11 @@ var checkCmd = &cobra.Command{
 		defer trace.StartRegion(cmd.Context(), "compile").End()
 		stderr := cmd.ErrOrStderr()
 		dir, name := getConfigPath(stderr, cmd.Flag("file"))
-		if _, err := Generate(cmd.Context(), ParseEnv(cmd), dir, name, stderr); err != nil {
+		_, err := Generate(cmd.Context(), dir, name, &Options{
+			Env:    ParseEnv(cmd),
+			Stderr: stderr,
+		})
+		if err != nil {
 			os.Exit(1)
 		}
 		return nil
@@ -277,7 +288,11 @@ var diffCmd = &cobra.Command{
 		defer trace.StartRegion(cmd.Context(), "diff").End()
 		stderr := cmd.ErrOrStderr()
 		dir, name := getConfigPath(stderr, cmd.Flag("file"))
-		if err := Diff(cmd.Context(), ParseEnv(cmd), dir, name, stderr); err != nil {
+		opts := &Options{
+			Env:    ParseEnv(cmd),
+			Stderr: stderr,
+		}
+		if err := Diff(cmd.Context(), dir, name, opts); err != nil {
 			os.Exit(1)
 		}
 		return nil

internal/cmd/diff.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"io"
 	"os"
 	"runtime/trace"
 	"sort"
@@ -13,8 +12,9 @@ import (
 	"github.com/cubicdaiya/gonp"
 )
 
-func Diff(ctx context.Context, e Env, dir, name string, stderr io.Writer) error {
-	output, err := Generate(ctx, e, dir, name, stderr)
+func Diff(ctx context.Context, dir, name string, opts *Options) error {
+	stderr := opts.Stderr
+	output, err := Generate(ctx, dir, name, opts)
 	if err != nil {
 		return err
 	}

internal/cmd/generate.go

@@ -139,8 +139,11 @@ func readConfig(stderr io.Writer, dir, filename string) (string, *config.Config,
 	return configPath, &conf, nil
 }
 
-func Generate(ctx context.Context, e Env, dir, filename string, stderr io.Writer) (map[string]string, error) {
-	configPath, conf, err := readConfig(stderr, dir, filename)
+func Generate(ctx context.Context, dir, filename string, o *Options) (map[string]string, error) {
+	e := o.Env
+	stderr := o.Stderr
+
+	configPath, conf, err := o.ReadConfig(dir, filename)
 	if err != nil {
 		return nil, err
 	}
@@ -343,7 +346,12 @@ func remoteGenerate(ctx context.Context, configPath string, conf *config.Config,
 
 func parse(ctx context.Context, name, dir string, sql config.SQL, combo config.CombinedSettings, parserOpts opts.Parser, stderr io.Writer) (*compiler.Result, bool) {
 	defer trace.StartRegion(ctx, "parse").End()
-	c := compiler.NewCompiler(sql, combo)
+	c, err := compiler.NewCompiler(sql, combo)
+	defer c.Close(ctx)
+	if err != nil {
+		fmt.Fprintf(stderr, "error creating compiler: %s\n", err)
+		return nil, true
+	}
 	if err := c.ParseCatalog(sql.Schema); err != nil {
 		fmt.Fprintf(stderr, "# package %s\n", name)
 		if parserErr, ok := err.(*multierr.Error); ok {

internal/cmd/options.go

@@ -0,0 +1,24 @@
+package cmd
+
+import (
+	"io"
+
+	"github.com/sqlc-dev/sqlc/internal/config"
+)
+
+type Options struct {
+	Env          Env
+	Stderr       io.Writer
+	MutateConfig func(*config.Config)
+}
+
+func (o *Options) ReadConfig(dir, filename string) (string, *config.Config, error) {
+	path, conf, err := readConfig(o.Stderr, dir, filename)
+	if err != nil {
+		return path, conf, err
+	}
+	if o.MutateConfig != nil {
+		o.MutateConfig(conf)
+	}
+	return path, conf, nil
+}

internal/cmd/package.go

@@ -2,13 +2,14 @@ package cmd
 
 import (
 	"context"
-	"io"
 	"os"
 
 	"github.com/sqlc-dev/sqlc/internal/bundler"
 )
 
-func createPkg(ctx context.Context, e Env, dir, filename string, stderr io.Writer) error {
+func createPkg(ctx context.Context, dir, filename string, opts *Options) error {
+	e := opts.Env
+	stderr := opts.Stderr
 	configPath, conf, err := readConfig(stderr, dir, filename)
 	if err != nil {
 		return err
@@ -17,7 +18,7 @@ func createPkg(ctx context.Context, e Env, dir, filename string, stderr io.Write
 	if err := up.Validate(); err != nil {
 		return err
 	}
-	output, err := Generate(ctx, e, dir, filename, stderr)
+	output, err := Generate(ctx, dir, filename, opts)
 	if err != nil {
 		os.Exit(1)
 	}

internal/cmd/vet.go

@@ -47,8 +47,12 @@ func NewCmdVet() *cobra.Command {
 		RunE: func(cmd *cobra.Command, args []string) error {
 			defer trace.StartRegion(cmd.Context(), "vet").End()
 			stderr := cmd.ErrOrStderr()
+			opts := &Options{
+				Env:    ParseEnv(cmd),
+				Stderr: stderr,
+			}
 			dir, name := getConfigPath(stderr, cmd.Flag("file"))
-			if err := Vet(cmd.Context(), ParseEnv(cmd), dir, name, stderr); err != nil {
+			if err := Vet(cmd.Context(), dir, name, opts); err != nil {
 				if !errors.Is(err, ErrFailedChecks) {
 					fmt.Fprintf(stderr, "%s\n", err)
 				}
@@ -59,7 +63,9 @@ func NewCmdVet() *cobra.Command {
 	}
 }
 
-func Vet(ctx context.Context, e Env, dir, filename string, stderr io.Writer) error {
+func Vet(ctx context.Context, dir, filename string, opts *Options) error {
+	e := opts.Env
+	stderr := opts.Stderr
 	configPath, conf, err := readConfig(stderr, dir, filename)
 	if err != nil {
 		return err

internal/codegen/golang/result.go

@@ -249,9 +249,7 @@ func buildQueries(req *plugin.CodeGenRequest, options *opts, structs []Struct) (
 		if len(query.Columns) == 1 && query.Columns[0].EmbedTable == nil {
 			c := query.Columns[0]
 			name := columnName(c, 0)
-			if c.IsFuncCall {
-				name = strings.Replace(name, "$", "_", -1)
-			}
+			name = strings.Replace(name, "$", "_", -1)
 			gq.Ret = QueryValue{
 				Name:      name,
 				DBName:    name,