From d8c3a2abafbe90f5d19bf206ea92d991e13f7ee4 Mon Sep 17 00:00:00 2001 From: Florent Messa Date: Mon, 17 May 2021 10:36:33 +0200 Subject: [PATCH] fix: add compatibility with latest loukoum --- dialect/postgres/postgres.go | 49 +++++++++++++++++++----------------- etl/engine.go | 2 +- etl/extractor.go | 4 +-- go.mod | 4 +-- go.sum | 13 ++++++---- 5 files changed, 39 insertions(+), 33 deletions(-) diff --git a/dialect/postgres/postgres.go b/dialect/postgres/postgres.go index eb72a4d..769156b 100644 --- a/dialect/postgres/postgres.go +++ b/dialect/postgres/postgres.go @@ -105,9 +105,9 @@ func (d *PGDialect) ReferenceKeys(ctx context.Context, tableName string) (dialec builder := lk.Select( "conname", - "c2.relname AS table", - "(SELECT attname FROM pg_attribute WHERE attrelid = r.conrelid AND ARRAY[attnum] <@ r.conkey) AS column", - ).From("pg_constraint r, pg_class c, pg_class c2"). + lk.Raw("c2.relname AS table"), + lk.Raw("(SELECT attname FROM pg_attribute WHERE attrelid = r.conrelid AND ARRAY[attnum] <@ r.conkey) AS column"), + ).From(lk.Raw("pg_constraint r, pg_class c, pg_class c2")). Where(lk.Condition("r.confrelid").Equal(oid)). And(lk.Raw("c.oid = r.confrelid")). And(lk.Raw("c2.oid = r.conrelid")). @@ -140,8 +140,8 @@ func (d *PGDialect) ForeignKeys(ctx context.Context, tableName string) (dialect. return nil, err } - builder := lk.Select("r.conname", "pg_catalog.pg_get_constraintdef(r.oid, true) AS condef"). - From("pg_catalog.pg_constraint r, pg_namespace n, pg_class c"). + builder := lk.Select("r.conname", lk.Raw("pg_catalog.pg_get_constraintdef(r.oid, true) AS condef")). + From(lk.Raw("pg_catalog.pg_constraint r, pg_namespace n, pg_class c")). Where(lk.Condition("r.conrelid").Equal(oid)). And(lk.Raw("r.contype = 'f'")). And(lk.Raw("c.oid = confrelid")). @@ -183,7 +183,7 @@ func (d *PGDialect) PrimaryKeyConstraint(ctx context.Context, tableName string) } builder := lk.Select("conname"). - From("pg_catalog.pg_constraint r"). + From(lk.Table("pg_catalog.pg_constraint").As("r")). Where(lk.Condition("r.conrelid").Equal(oid)). And(lk.Raw("r.contype = 'p'")). OrderBy(lk.Order("1")) @@ -203,8 +203,11 @@ func (d *PGDialect) PrimaryKeys(ctx context.Context, tableName string) ([]dialec return nil, err } - builder := lk.Select("pg_attribute.attname AS name", "format_type(pg_attribute.atttypid, pg_attribute.atttypmod) AS data_type"). - From("pg_index, pg_class, pg_attribute, pg_namespace"). + builder := lk.Select( + lk.Raw("pg_attribute.attname AS name"), + lk.Raw("format_type(pg_attribute.atttypid, pg_attribute.atttypmod) AS data_type"), + ). + From(lk.Raw("pg_index, pg_class, pg_attribute, pg_namespace")). Where(lk.Condition("pg_class.oid").Equal(oid)). And(lk.Raw("indrelid = pg_class.oid")). And(lk.Raw("nspname = 'public'")). @@ -239,21 +242,21 @@ func (d *PGDialect) PrimaryKeys(ctx context.Context, tableName string) ([]dialec // Columns returns sorted columns with types of a table. func (d *PGDialect) Columns(ctx context.Context, tableName string) ([]dialect.Column, error) { builder := lk.Select( - "a.attname AS column_name", - "pg_catalog.format_type(a.atttypid, a.atttypmod) AS data_type", - `( + lk.Raw("a.attname AS column_name"), + lk.Raw("pg_catalog.format_type(a.atttypid, a.atttypmod) AS data_type"), + lk.Raw(`( SELECT pg_catalog.pg_get_expr(d.adbin, d.adrelid) FROM pg_catalog.pg_attrdef d WHERE d.adrelid = a.attrelid AND d.adnum = a.attnum AND a.atthasdef - ) AS default`, - "a.attnotnull AS is_nullable", - "c.relname AS table_name", - "a.attnum as ordinal_position", + ) AS default`), + lk.Raw("a.attnotnull AS is_nullable"), + lk.Raw("c.relname AS table_name"), + lk.Raw("a.attnum as ordinal_position"), ). - From("pg_catalog.pg_attribute a"). - Join("pg_catalog.pg_class c", lk.On("a.attrelid", "c.oid"), lk.LeftJoin). - Join("pg_catalog.pg_description pgd", lk.On("pgd.objoid = a.attrelid AND pgd.objsubid", "a.attnum"), lk.LeftJoin). + From(lk.Table("pg_catalog.pg_attribute").As("a")). + Join(lk.Table("pg_catalog.pg_class").As("c"), lk.On("a.attrelid", "c.oid"), lk.LeftJoin). + Join(lk.Table("pg_catalog.pg_description").As("pgd"), lk.AndOn(lk.On("pgd.objoid", "a.attrelid"), lk.On("pgd.objsubid", "a.attnum")), lk.LeftJoin). Where(lk.Condition("a.attnum").GreaterThan(0)). And(lk.Condition("a.attisdropped").Equal(false)). OrderBy(lk.Order("a.attnum")) @@ -330,8 +333,8 @@ func (d *PGDialect) Table(ctx context.Context, tableName string) (dialect.Table, // Tables returns all the tables from the database. func (d *PGDialect) Tables(ctx context.Context) (dialect.Tables, error) { builder := lk.Select("c.relname"). - From("pg_catalog.pg_class c"). - Join("pg_namespace n", lk.On("n.oid", "c.relnamespace")). + From(lk.Table("pg_catalog.pg_class").As("c")). + Join(lk.Table("pg_namespace").As("n"), lk.On("n.oid", "c.relnamespace")). Where(lk.Raw("relkind = 'r'")). And(lk.Condition("n.nspname").Equal("public")). Comment("tables") @@ -426,9 +429,9 @@ func (d *PGDialect) query(ctx context.Context, query string, args ...interface{} func (d *PGDialect) getTableOID(ctx context.Context, tableName string) (int64, error) { builder := lk.Select("c.oid"). - From("pg_catalog.pg_class c"). - Join(lk.Table("pg_catalog.pg_namespace n"), lk.On("n.oid", "c.relnamespace"), lk.LeftJoin). - Where(lk.Condition("pg_catalog.pg_table_is_visible(c.oid)")). + From(lk.Table("pg_catalog.pg_class").As("c")). + Join(lk.Table("pg_catalog.pg_namespace").As("n"), lk.On("n.oid", "c.relnamespace"), lk.LeftJoin). + Where(lk.Raw("pg_catalog.pg_table_is_visible(c.oid)")). And(lk.Condition("c.relname").Equal(tableName)). And(lk.Raw("c.relkind IN ('r', 'v', 'm', 'f', 'p')")). Comment("table oid") diff --git a/etl/engine.go b/etl/engine.go index 03b9d32..b305c59 100644 --- a/etl/engine.go +++ b/etl/engine.go @@ -116,7 +116,7 @@ func (e *Engine) Extract(ctx context.Context, outputPath, query string) error { for i := range e.config.Extra { tableName := e.config.Extra[i].TableName - query, _ := lk.Select("*"). + query, _ := lk.Select(lk.Raw("*")). From(tableName).Query() cache, err = extractor.Handle(ctx, e.schema[tableName], query) if err != nil { diff --git a/etl/extractor.go b/etl/extractor.go index 39a5d66..25a0860 100644 --- a/etl/extractor.go +++ b/etl/extractor.go @@ -52,7 +52,7 @@ func (e *extractor) handleReferenceKeys(ctx context.Context, depth int, table di value := row[primaryKey.Name] referenceKey := referenceKeys[i] - query, args := lk.Select("*"). + query, args := lk.Select(lk.Raw("*")). From(referenceKey.Table.Name). Where(lk.Condition(referenceKey.ColumnName).Equal(value)). Query() @@ -100,7 +100,7 @@ func (e *extractor) handleRow(ctx context.Context, depth int, table dialect.Tabl } if foreignKey, ok := foreignKeys[k]; ok { - query, args := lk.Select("*"). + query, args := lk.Select(lk.Raw("*")). From(foreignKey.ReferencedTable.Name). Where(lk.Condition(foreignKey.ReferencedColumnName).Equal(v)). Query() diff --git a/go.mod b/go.mod index 09084b2..1d6afe7 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ require ( github.com/georgysavva/scany v0.2.7 github.com/jackc/pgtype v1.6.1 github.com/jackc/pgx/v4 v4.9.2 - github.com/stretchr/testify v1.5.1 - github.com/ulule/loukoum/v3 v3.3.0 + github.com/stretchr/testify v1.7.0 + github.com/ulule/loukoum/v3 v3.5.1-0.20210517081636-4790f61dc7e9 go.uber.org/zap v1.10.0 golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9 // indirect golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 diff --git a/go.sum b/go.sum index 789ab07..a811762 100644 --- a/go.sum +++ b/go.sum @@ -99,8 +99,9 @@ github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.4.0 h1:TmtCFbH+Aw0AixwyttznSMQDgbR5Yed/Gg6S8Funrhc= github.com/lib/pq v1.4.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -134,10 +135,11 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/ulule/loukoum/v3 v3.3.0 h1:4YFRMd3dM0buB2ND13bCsEaFcSVDGvvprupTyXf4SsU= -github.com/ulule/loukoum/v3 v3.3.0/go.mod h1:NFdYIhh/wucEHL/lJYvui79DZwRh5kHXWr9pziou1fo= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/ulule/loukoum/v3 v3.5.1-0.20210517081636-4790f61dc7e9 h1:5L6ExfajpYLtYioBcBDQeASaTwBPdtMpzjaJd2y91qI= +github.com/ulule/loukoum/v3 v3.5.1-0.20210517081636-4790f61dc7e9/go.mod h1:V7wI/bEqfFGwksUKTEEC5Rmi7dVsffxqsR2S9z6Z2HA= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -212,8 +214,9 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3 h1:fvjTMHxHEw/mxHbtzPi3JCcKXQRAnQTBRo6YCJSVHKI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= syreclabs.com/go/faker v1.2.3 h1:HPrWtnHazIf0/bVuPZJLFrtHlBHk10hS0SB+mV8v6R4=