feat(indexer): postgres schema creation + CI config

.github/dependabot.yml

@@ -106,6 +106,24 @@ updates:
     labels:
       - "A:automerge"
       - dependencies
+  - package-ecosystem: gomod
+    directory: "/indexer/postgres"
+    schedule:
+      interval: weekly
+      day: wednesday
+      time: "01:53"
+    labels:
+      - "A:automerge"
+      - dependencies
+  - package-ecosystem: gomod
+    directory: "/indexer/postgres/tests"
+    schedule:
+      interval: weekly
+      day: wednesday
+      time: "01:53"
+    labels:
+      - "A:automerge"
+      - dependencies
   - package-ecosystem: gomod
     directory: "/schema"
     schedule:

.github/pr_labeler.yml

@@ -24,6 +24,8 @@
   - orm/**/*
 "C:schema":
   - schema/**/*
+"C:indexer/postgres":
+  - indexer/postgres/**/*
 "C:x/accounts":
   - x/accounts/**/*
 "C:x/auth":

.github/workflows/test.yml

@@ -487,6 +487,42 @@ jobs:
         with:
           projectBaseDir: schema/
 
+  test-indexer-postgres:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: "1.22"
+          cache: true
+          cache-dependency-path: indexer/postgres/tests/go.sum
+      - uses: technote-space/get-diff-action@v6.1.2
+        id: git_diff
+        with:
+          PATTERNS: |
+            indexer/postgres/**/*.go
+            indexer/postgres/go.mod
+            indexer/postgres/go.sum
+            indexer/postgres/tests/go.mod
+            indexer/postgres/tests/go.sum
+      - name: tests
+        if: env.GIT_DIFF
+        run: |
+          cd indexer/postgres
+          go test -mod=readonly -timeout 30m -coverprofile=cov.out -covermode=atomic ./...
+          cd tests
+          go test -mod=readonly -timeout 30m -coverprofile=cov.out -covermode=atomic -coverpkg=cosmossdk.io/indexer/postgres ./...
+          cd ..
+          go run github.com/dylandreimerink/gocovmerge/cmd/gocovmerge@latest cov.out tests/cov.out > coverage.out
+      - name: sonarcloud
+        if: ${{ env.GIT_DIFF && !github.event.pull_request.draft && env.SONAR_TOKEN != null }}
+        uses: SonarSource/sonarcloud-github-action@master
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+        with:
+          projectBaseDir: indexer/postgres/
+
   test-simapp:
     runs-on: ubuntu-latest
     steps:

go.work.example

@@ -9,6 +9,7 @@ use (
 	./core/testing
 	./depinject
 	./errors
+	./indexer/postgres
 	./log
 	./math
 	./orm

indexer/postgres/CHANGELOG.md

@@ -0,0 +1,37 @@
+<!--
+Guiding Principles:
+
+Changelogs are for humans, not machines.
+There should be an entry for every single version.
+The same types of changes should be grouped.
+Versions and sections should be linkable.
+The latest version comes first.
+The release date of each version is displayed.
+Mention whether you follow Semantic Versioning.
+
+Usage:
+
+Change log entries are to be added to the Unreleased section under the
+appropriate stanza (see below). Each entry should ideally include a tag and
+the Github issue reference in the following format:
+
+* (<tag>) \#<issue-number> message
+
+The issue numbers will later be link-ified during the release process so you do
+not have to worry about including a link manually, but you can if you wish.
+
+Types of changes (Stanzas):
+
+"Features" for new features.
+"Improvements" for changes in existing functionality.
+"Deprecated" for soon-to-be removed features.
+"Bug Fixes" for any bug fixes.
+"Client Breaking" for breaking Protobuf, gRPC and REST routes used by end-users.
+"CLI Breaking" for breaking CLI commands.
+"API Breaking" for breaking exported APIs used by developers building on SDK.
+Ref: https://keepachangelog.com/en/1.0.0/
+-->
+
+# Changelog
+
+## [Unreleased]

indexer/postgres/README.md

@@ -0,0 +1,41 @@
+# PostgreSQL Indexer
+
+The PostgreSQL indexer can fully index the current state for all modules that implement `cosmossdk.io/schema.HasModuleCodec`.
+implement `cosmossdk.io/schema.HasModuleCodec`.
+
+## Table, Column and Enum Naming
+
+`ObjectType`s names are converted to table names prefixed with the module name and an underscore. i.e. the `ObjectType` `foo` in module `bar` will be stored in a table named `bar_foo`.
+
+Column names are identical to field names. All identifiers are quoted with double quotes so that they are case-sensitive and won't clash with any reserved names. 
+
+Like, table names, enum types are prefixed with the module name and an underscore.
+
+## Schema Type Mapping
+
+The mapping of `cosmossdk.io/schema` `Kind`s to PostgreSQL types is as follows:
+
+| Kind | PostgreSQL Type            | Notes                                                                                                                                                                           |
+|---------------------|----------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `StringKind`        | `TEXT`                     |                                                                                                                                                                                 |
+| `BoolKind`          | `BOOLEAN`                  |                                                                                                                                                                                 |
+| `BytesKind`         | `BYTEA`                    |                                                                                                                                                                                 |
+| `Int8Kind`          | `SMALLINT`                 |                                                                                                                                                                                 |
+| `Int16Kind`         | `SMALLINT`                 |                                                                                                                                                                                 |
+| `Int32Kind`         | `INTEGER`                  |                                                                                                                                                                                 |
+| `Int64Kind`         | `BIGINT`                   |                                                                                                                                                                                 |
+| `Uint8Kind`         | `SMALLINT`                 |                                                                                                                                                                                 |
+| `Uint16Kind`        | `INTEGER`                  |                                                                                                                                                                                 |
+| `Uint32Kind`        | `BIGINT`                   |                                                                                                                                                                                 |
+| `Uint64Kind`        | `NUMERIC`                  |                                                                                                                                                                                 |
+| `Float32Kind`       | `REAL`                     |                                                                                                                                                                                 |
+| `Float64Kind`       | `DOUBLE PRECISION`         |                                                                                                                                                                                 |
+| `IntegerStringKind` | `NUMERIC`                  |                                                                                                                                                                                 |
+| `DecimalStringKind` | `NUMERIC`                  |                                                                                                                                                                                 |
+| `JSONKind`          | `JSONB`                    |                                                                                                                                                                                 |
+| `Bech32AddressKind` | `TEXT`                     | addresses are converted to strings with the specified address prefix                                                                                                            |
+| `TimeKind`          | `BIGINT` and `TIMESTAMPTZ` | time types are stored as two columns, one with the `_nanos` suffix with full nanoseconds precision, and another as a `TIMESTAMPTZ` generated column with microsecond precision |
+| `DurationKind`      | `BIGINT`                   | durations are stored as a single column in nanoseconds                                                                                                                          |
+| `EnumKind` | `<module_name>_<enum_name>` | a custom enum type is created for each module prefixed with the module name it pertains to                                                                                     |
+
+

indexer/postgres/base_sql.go

@@ -0,0 +1,8 @@
+package postgres
+
+// BaseSQL is the base SQL that is always included in the schema.
+const BaseSQL = `
+CREATE OR REPLACE FUNCTION nanos_to_timestamptz(nanos bigint) RETURNS timestamptz AS $$
+    SELECT to_timestamp(nanos / 1000000000) + (nanos / 1000000000) * INTERVAL '1 microsecond'
+$$ LANGUAGE SQL IMMUTABLE;
+`

indexer/postgres/column.go

@@ -0,0 +1,120 @@
+package postgres
+
+import (
+	"fmt"
+	"io"
+
+	"cosmossdk.io/schema"
+)
+
+// createColumnDefinition writes a column definition within a CREATE TABLE statement for the field.
+func (tm *ObjectIndexer) createColumnDefinition(writer io.Writer, field schema.Field) error {
+	_, err := fmt.Fprintf(writer, "%q ", field.Name)
+	if err != nil {
+		return err
+	}
+
+	simple := simpleColumnType(field.Kind)
+	if simple != "" {
+		_, err = fmt.Fprintf(writer, "%s", simple)
+		if err != nil {
+			return err
+		}
+
+		return writeNullability(writer, field.Nullable)
+	} else {
+		switch field.Kind {
+		case schema.EnumKind:
+			_, err = fmt.Fprintf(writer, "%q", enumTypeName(tm.moduleName, field.EnumDefinition))
+			if err != nil {
+				return err
+			}
+		case schema.TimeKind:
+			// for time fields, we generate two columns:
+			// - one with nanoseconds precision for lossless storage, suffixed with _nanos
+			// - one as a timestamptz (microsecond precision) for ease of use, that is GENERATED
+			nanosColName := fmt.Sprintf("%s_nanos", field.Name)
+			_, err = fmt.Fprintf(writer, "TIMESTAMPTZ GENERATED ALWAYS AS (nanos_to_timestamptz(%q)) STORED,\n\t", nanosColName)
+			if err != nil {
+				return err
+			}
+
+			_, err = fmt.Fprintf(writer, `%q BIGINT`, nanosColName)
+			if err != nil {
+				return err
+			}
+		default:
+			return fmt.Errorf("unexpected kind: %v, this should have been handled earlier", field.Kind)
+		}
+
+		return writeNullability(writer, field.Nullable)
+	}
+}
+
+// writeNullability writes column nullability.
+func writeNullability(writer io.Writer, nullable bool) error {
+	if nullable {
+		_, err := fmt.Fprintf(writer, " NULL,\n\t")
+		return err
+	} else {
+		_, err := fmt.Fprintf(writer, " NOT NULL,\n\t")
+		return err
+	}
+}
+
+// simpleColumnType returns the postgres column type for the kind for simple types.
+func simpleColumnType(kind schema.Kind) string {
+	//nolint:goconst // adding constants for these postgres type names would impede readability
+	switch kind {
+	case schema.StringKind:
+		return "TEXT"
+	case schema.BoolKind:
+		return "BOOLEAN"
+	case schema.BytesKind:
+		return "BYTEA"
+	case schema.Int8Kind:
+		return "SMALLINT"
+	case schema.Int16Kind:
+		return "SMALLINT"
+	case schema.Int32Kind:
+		return "INTEGER"
+	case schema.Int64Kind:
+		return "BIGINT"
+	case schema.Uint8Kind:
+		return "SMALLINT"
+	case schema.Uint16Kind:
+		return "INTEGER"
+	case schema.Uint32Kind:
+		return "BIGINT"
+	case schema.Uint64Kind:
+		return "NUMERIC"
+	case schema.IntegerStringKind:
+		return "NUMERIC"
+	case schema.DecimalStringKind:
+		return "NUMERIC"
+	case schema.Float32Kind:
+		return "REAL"
+	case schema.Float64Kind:
+		return "DOUBLE PRECISION"
+	case schema.JSONKind:
+		return "JSONB"
+	case schema.DurationKind:
+		return "BIGINT"
+	case schema.Bech32AddressKind:
+		return "TEXT"
+	default:
+		return ""
+	}
+}
+
+// updatableColumnName is the name of the insertable/updatable column name for the field.
+// This is the field name in most cases, except for time columns which are stored as nanos
+// and then converted to timestamp generated columns.
+func (tm *ObjectIndexer) updatableColumnName(field schema.Field) (name string, err error) {
+	name = field.Name
+	if field.Kind == schema.TimeKind {
+		name = fmt.Sprintf("%s_nanos", name)
+	}
+	name = fmt.Sprintf("%q", name)
+	return
+}

indexer/postgres/conn.go

@@ -0,0 +1,14 @@
+package postgres
+
+import (
+	"context"
+	"database/sql"
+)
+
+// DBConn is an interface that abstracts the *sql.DB, *sql.Tx and *sql.Conn types.
+type DBConn interface {
+	ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error)
+	PrepareContext(ctx context.Context, query string) (*sql.Stmt, error)
+	QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error)
+	QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row
+}