From da55fb3bac2f6da95cccfdf6b9c921a36c439af4 Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Tue, 4 Apr 2023 16:29:18 +0100 Subject: [PATCH 01/25] Move to Arrow (WIP) --- csv/read.go | 38 ++---- csv/write.go | 29 ++--- csv/write_read_test.go | 59 ++++----- go.mod | 7 +- go.sum | 7 +- internal/cqarrow/arrow.go | 218 -------------------------------- internal/cqarrow/arrow_test.go | 89 ------------- internal/cqarrow/inet.go | 183 --------------------------- internal/cqarrow/inet_test.go | 64 ---------- internal/cqarrow/json.go | 178 -------------------------- internal/cqarrow/json_test.go | 53 -------- internal/cqarrow/mac.go | 190 ---------------------------- internal/cqarrow/mac_test.go | 64 ---------- internal/cqarrow/uuid.go | 222 --------------------------------- internal/cqarrow/uuid_test.go | 56 --------- json/read.go | 24 ++-- json/write.go | 20 ++- json/write_read_test.go | 128 ++++--------------- parquet/read.go | 12 +- read.go | 3 +- 20 files changed, 114 insertions(+), 1530 deletions(-) delete mode 100644 internal/cqarrow/arrow.go delete mode 100644 internal/cqarrow/arrow_test.go delete mode 100644 internal/cqarrow/inet.go delete mode 100644 internal/cqarrow/inet_test.go delete mode 100644 internal/cqarrow/json.go delete mode 100644 internal/cqarrow/json_test.go delete mode 100644 internal/cqarrow/mac.go delete mode 100644 internal/cqarrow/mac_test.go delete mode 100644 internal/cqarrow/uuid.go delete mode 100644 internal/cqarrow/uuid_test.go diff --git a/csv/read.go b/csv/read.go index a431b5df..ad607960 100644 --- a/csv/read.go +++ b/csv/read.go @@ -1,43 +1,29 @@ package csv import ( - "encoding/csv" - "errors" "fmt" "io" + "github.com/apache/arrow/go/v12/arrow" + "github.com/apache/arrow/go/v12/arrow/csv" "github.com/cloudquery/plugin-sdk/schema" ) -func (cl *Client) Read(r io.Reader, table *schema.Table, sourceName string, res chan<- []any) error { - reader := csv.NewReader(r) - reader.Comma = cl.Delimiter +func (cl *Client) Read(r io.Reader, table *schema.Table, _ string, res chan<- arrow.Record) error { + arrowSchema := table.ToArrowSchema() + + reader := csv.NewReader(r, arrowSchema, csv.WithComma(cl.Delimiter), csv.WithHeader(cl.IncludeHeaders), csv.WithNullReader(true, "")) sourceNameIndex := table.Columns.Index(schema.CqSourceNameColumn.Name) if sourceNameIndex == -1 { return fmt.Errorf("could not find column %s in table %s", schema.CqSourceNameColumn.Name, table.Name) } - if cl.IncludeHeaders { - _, err := reader.Read() - if err != nil { - return err - } - } - for { - record, err := reader.Read() - if err != nil { - if errors.Is(err, io.EOF) { - break - } - return err - } - if record[sourceNameIndex] != sourceName { - continue - } - values := make([]any, len(record)) - for i, v := range record { - values[i] = v + for reader.Next() { + if reader.Err() != nil { + return reader.Err() } - res <- values + rec := reader.Record() + rec.Retain() + res <- rec } return nil } diff --git a/csv/write.go b/csv/write.go index 1261483d..4b30b0a1 100644 --- a/csv/write.go +++ b/csv/write.go @@ -5,30 +5,21 @@ import ( "fmt" "io" - "github.com/apache/arrow/go/arrow/memory" + "github.com/apache/arrow/go/v12/arrow" "github.com/apache/arrow/go/v12/arrow/csv" - "github.com/cloudquery/filetypes/internal/cqarrow" "github.com/cloudquery/plugin-sdk/schema" ) -func (cl *Client) WriteTableBatch(w io.Writer, table *schema.Table, resources [][]any) error { - arrowSchema := cqarrow.CQSchemaToArrow(table) - cqTypes := make([]schema.CQTypes, len(resources)) - for i := range resources { - cqTypes[i] = make(schema.CQTypes, len(resources[i])) - for j := range resources[i] { - cqTypes[i][j] = resources[i][j].(schema.CQType) - } - } - record := cqarrow.CQTypesToRecord(memory.DefaultAllocator, cqTypes, arrowSchema) - defer record.Release() - +func (cl *Client) WriteTableBatch(w io.Writer, table *schema.Table, records []arrow.Record) error { + arrowSchema := table.ToArrowSchema() writer := csv.NewWriter(w, arrowSchema, csv.WithComma(cl.Delimiter), csv.WithHeader(cl.IncludeHeaders), csv.WithNullWriter("")) - if err := writer.Write(record); err != nil { - return fmt.Errorf("failed to write record to csv: %w", err) - } - if err := writer.Flush(); err != nil { - return fmt.Errorf("failed to flush csv writer: %w", err) + for _, record := range records { + if err := writer.Write(record); err != nil { + return fmt.Errorf("failed to write record to csv: %w", err) + } + if err := writer.Flush(); err != nil { + return fmt.Errorf("failed to flush csv writer: %w", err) + } } return nil } diff --git a/csv/write_read_test.go b/csv/write_read_test.go index 7a5e2c6c..ea1fcf78 100644 --- a/csv/write_read_test.go +++ b/csv/write_read_test.go @@ -1,12 +1,15 @@ package csv import ( - "bufio" "bytes" "testing" + "time" - "github.com/cloudquery/plugin-sdk/schema" + "github.com/apache/arrow/go/v12/arrow" + "github.com/apache/arrow/go/v12/arrow/memory" + "github.com/cloudquery/plugin-sdk/plugins/destination" "github.com/cloudquery/plugin-sdk/testdata" + "github.com/google/uuid" ) func TestWriteRead(t *testing.T) { @@ -18,58 +21,58 @@ func TestWriteRead(t *testing.T) { {name: "default", outputCount: 1}, {name: "with_headers", options: []Options{WithHeader()}, outputCount: 1}, {name: "with_delimiter", options: []Options{WithDelimiter('\t')}, outputCount: 1}, - {name: "with_delimter_headers", options: []Options{WithDelimiter('\t'), WithHeader()}, outputCount: 1}, + {name: "with_delimiter_headers", options: []Options{WithDelimiter('\t'), WithHeader()}, outputCount: 1}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { var b bytes.Buffer table := testdata.TestTable("test") - cqtypes := testdata.GenTestData(table) - if err := cqtypes[0].Set("test-source"); err != nil { - t.Fatal(err) + sch := table.ToArrowSchema() + sourceName := "test-source" + syncTime := time.Now().UTC().Round(1 * time.Second) + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + opts := testdata.GenTestDataOptions{ + SourceName: sourceName, + SyncTime: syncTime, + MaxRows: 1, + StableUUID: uuid.Nil, } - writer := bufio.NewWriter(&b) - reader := bufio.NewReader(&b) - transformer := &schema.DefaultTransformer{} - transformedValues := schema.TransformWithTransformer(transformer, cqtypes) - client, err := NewClient(tc.options...) + records := testdata.GenTestData(mem, sch, opts) + defer func() { + for _, r := range records { + r.Release() + } + }() + cl, err := NewClient(tc.options...) if err != nil { t.Fatal(err) } - - if err := client.WriteTableBatch(writer, table, [][]any{transformedValues}); err != nil { + if err := cl.WriteTableBatch(&b, table, records); err != nil { t.Fatal(err) } - writer.Flush() - ch := make(chan []any) + ch := make(chan arrow.Record) var readErr error go func() { - readErr = client.Read(reader, table, "test-source", ch) + readErr = cl.Read(&b, table, "test-source", ch) close(ch) }() totalCount := 0 - reverseTransformer := &ReverseTransformer{} - for row := range ch { - if client.IncludeHeaders && totalCount == 0 { - totalCount++ - continue - } - gotCqtypes, err := reverseTransformer.ReverseTransformValues(table, row) - if err != nil { - t.Fatal(err) - } - if diff := cqtypes.Diff(gotCqtypes); diff != "" { + for got := range ch { + if diff := destination.RecordDiff(records[totalCount], got); diff != "" { + got.Release() t.Fatalf("got diff: %s", diff) } + got.Release() totalCount++ } if readErr != nil { t.Fatal(readErr) } if totalCount != tc.outputCount { - t.Fatalf("expected %d row, got %d", tc.outputCount, totalCount) + t.Fatalf("got %d row(s), want %d", totalCount, tc.outputCount) } }) } diff --git a/go.mod b/go.mod index e6d26c90..d9fbc41b 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,8 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect ) +replace github.com/cloudquery/plugin-sdk v1.44.1 => ../plugin-sdk + require ( github.com/ghodss/yaml v1.0.0 // indirect github.com/google/uuid v1.3.0 @@ -18,20 +20,19 @@ require ( github.com/mattn/go-isatty v0.0.18 // indirect; indirect // indirect github.com/rs/zerolog v1.29.0 // indirect; indirect // indirect github.com/thoas/go-funk v0.9.3 // indirect; indirect // indirect - golang.org/x/sync v0.1.0 // indirect golang.org/x/sys v0.6.0 // indirect; indirect // indirect // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) require ( github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516 - github.com/apache/arrow/go/v12 v12.0.0-20230331222054-7e19111f2f81 + github.com/apache/arrow/go/v12 v12.0.0-20230401212133-adf33cc43010 github.com/goccy/go-json v0.10.2 github.com/google/go-cmp v0.5.9 github.com/xitongsys/parquet-go v1.6.2 ) -replace github.com/apache/arrow/go/v12 => github.com/cloudquery/arrow/go/v12 v12.0.0-20230317130341-c648117570af +replace github.com/apache/arrow/go/v12 => github.com/cloudquery/arrow/go/v12 v12.0.0-20230403113355-afc433992be2 require golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect diff --git a/go.sum b/go.sum index 26d1b28a..18655b4a 100644 --- a/go.sum +++ b/go.sum @@ -153,10 +153,8 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudquery/arrow/go/v12 v12.0.0-20230317130341-c648117570af h1:iK2UwRTmBl9+I41tASIttizlmiY7dH9KmKt7iOiwyOc= -github.com/cloudquery/arrow/go/v12 v12.0.0-20230317130341-c648117570af/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg= -github.com/cloudquery/plugin-sdk v1.44.1 h1:dprTbGbITkvU7XlBx2a7I2XLCHNF0UwFSxRTa/goeIw= -github.com/cloudquery/plugin-sdk v1.44.1/go.mod h1:CIv+fgm6siZhReOuMGU/OCBPNRLNY4At2RkC31LRaS0= +github.com/cloudquery/arrow/go/v12 v12.0.0-20230403113355-afc433992be2 h1:ifVrKVJXhP8hQvjiUjAXicRYv+zWzRg7B640gDvEWq4= +github.com/cloudquery/arrow/go/v12 v12.0.0-20230403113355-afc433992be2/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -676,7 +674,6 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/internal/cqarrow/arrow.go b/internal/cqarrow/arrow.go deleted file mode 100644 index 763e9b24..00000000 --- a/internal/cqarrow/arrow.go +++ /dev/null @@ -1,218 +0,0 @@ -package cqarrow - -import ( - "github.com/apache/arrow/go/v12/arrow" - "github.com/apache/arrow/go/v12/arrow/array" - "github.com/apache/arrow/go/v12/arrow/memory" - "github.com/cloudquery/plugin-sdk/schema" - "github.com/goccy/go-json" -) - -const ( - MetadataPrimaryKey = "cq:extension:primary_key" - MetadataPrimaryKeyTrue = "true" -) - -func CQColumnToArrowField(col *schema.Column) arrow.Field { - var typ arrow.DataType - metadata := make(map[string]string) - - switch col.Type { - case schema.TypeBool: - typ = arrow.FixedWidthTypes.Boolean - case schema.TypeInt: - typ = arrow.PrimitiveTypes.Int64 - case schema.TypeFloat: - typ = arrow.PrimitiveTypes.Float64 - case schema.TypeUUID: - typ = NewUUIDType() - case schema.TypeString: - typ = arrow.BinaryTypes.String - case schema.TypeByteArray: - typ = arrow.BinaryTypes.Binary - case schema.TypeStringArray: - typ = arrow.ListOf(arrow.BinaryTypes.String) - case schema.TypeIntArray: - typ = arrow.ListOf(arrow.PrimitiveTypes.Int64) - case schema.TypeTimestamp: - typ = arrow.FixedWidthTypes.Timestamp_us - case schema.TypeJSON: - typ = NewJSONType() - case schema.TypeUUIDArray: - typ = arrow.ListOf(NewUUIDType()) - case schema.TypeInet: - typ = NewInetType() - case schema.TypeInetArray: - typ = arrow.ListOf(NewInetType()) - case schema.TypeCIDR: - typ = NewInetType() - case schema.TypeCIDRArray: - typ = arrow.ListOf(NewInetType()) - case schema.TypeMacAddr: - typ = NewMacType() - case schema.TypeMacAddrArray: - typ = arrow.ListOf(NewMacType()) - default: - panic("unknown type " + typ.Name()) - } - if col.CreationOptions.PrimaryKey { - metadata[MetadataPrimaryKey] = MetadataPrimaryKeyTrue - } - return arrow.Field{ - Name: col.Name, - Type: typ, - Nullable: !col.CreationOptions.NotNull, - Metadata: arrow.MetadataFrom(metadata), - } -} - -func CQSchemaToArrow(table *schema.Table) *arrow.Schema { - fields := make([]arrow.Field, 0, len(table.Columns)) - for _, col := range table.Columns { - fields = append(fields, CQColumnToArrowField(&col)) - } - return arrow.NewSchema(fields, nil) -} - -func CQTypesToRecord(mem memory.Allocator, c []schema.CQTypes, arrowSchema *arrow.Schema) arrow.Record { - bldr := array.NewRecordBuilder(mem, arrowSchema) - fields := bldr.Fields() - for i := range fields { - for j := range c { - switch c[j][i].Type() { - case schema.TypeBool: - if c[j][i].(*schema.Bool).Status == schema.Present { - bldr.Field(i).(*array.BooleanBuilder).Append(c[j][i].(*schema.Bool).Bool) - } else { - bldr.Field(i).(*array.BooleanBuilder).AppendNull() - } - case schema.TypeInt: - if c[j][i].(*schema.Int8).Status == schema.Present { - bldr.Field(i).(*array.Int64Builder).Append(c[j][i].(*schema.Int8).Int) - } else { - bldr.Field(i).(*array.Int64Builder).AppendNull() - } - case schema.TypeFloat: - if c[j][i].(*schema.Float8).Status == schema.Present { - bldr.Field(i).(*array.Float64Builder).Append(c[j][i].(*schema.Float8).Float) - } else { - bldr.Field(i).(*array.Float64Builder).AppendNull() - } - case schema.TypeString: - if c[j][i].(*schema.Text).Status == schema.Present { - bldr.Field(i).(*array.StringBuilder).Append(c[j][i].(*schema.Text).Str) - } else { - bldr.Field(i).(*array.StringBuilder).AppendNull() - } - case schema.TypeByteArray: - if c[j][i].(*schema.Bytea).Status == schema.Present { - bldr.Field(i).(*array.BinaryBuilder).Append(c[j][i].(*schema.Bytea).Bytes) - } else { - bldr.Field(i).(*array.BinaryBuilder).AppendNull() - } - case schema.TypeStringArray: - if c[j][i].(*schema.TextArray).Status == schema.Present { - listBldr := bldr.Field(i).(*array.ListBuilder) - listBldr.Append(true) - for _, str := range c[j][i].(*schema.TextArray).Elements { - listBldr.ValueBuilder().(*array.StringBuilder).Append(str.Str) - } - } else { - bldr.Field(i).(*array.ListBuilder).AppendNull() - } - case schema.TypeIntArray: - if c[j][i].(*schema.Int8Array).Status == schema.Present { - listBldr := bldr.Field(i).(*array.ListBuilder) - listBldr.Append(true) - for _, e := range c[j][i].(*schema.Int8Array).Elements { - listBldr.ValueBuilder().(*array.Int64Builder).Append(e.Int) - } - } else { - bldr.Field(i).(*array.ListBuilder).AppendNull() - } - case schema.TypeTimestamp: - if c[j][i].(*schema.Timestamptz).Status == schema.Present { - bldr.Field(i).(*array.TimestampBuilder).Append(arrow.Timestamp(c[j][i].(*schema.Timestamptz).Time.UnixMicro())) - } else { - bldr.Field(i).(*array.TimestampBuilder).AppendNull() - } - case schema.TypeJSON: - if c[j][i].(*schema.JSON).Status == schema.Present { - var d any - if err := json.Unmarshal(c[j][i].(*schema.JSON).Bytes, &d); err != nil { - panic(err) - } - bldr.Field(i).(*JSONBuilder).Append(d) - } else { - bldr.Field(i).(*JSONBuilder).AppendNull() - } - case schema.TypeUUID: - if c[j][i].(*schema.UUID).Status == schema.Present { - bldr.Field(i).(*UUIDBuilder).Append(c[j][i].(*schema.UUID).Bytes) - } else { - bldr.Field(i).(*UUIDBuilder).AppendNull() - } - case schema.TypeUUIDArray: - if c[j][i].(*schema.UUIDArray).Status == schema.Present { - listBldr := bldr.Field(i).(*array.ListBuilder) - listBldr.Append(true) - for _, e := range c[j][i].(*schema.UUIDArray).Elements { - listBldr.ValueBuilder().(*UUIDBuilder).Append(e.Bytes) - } - } else { - bldr.Field(i).(*array.ListBuilder).AppendNull() - } - case schema.TypeInet: - if c[j][i].(*schema.Inet).Status == schema.Present { - bldr.Field(i).(*InetBuilder).Append(*c[j][i].(*schema.Inet).IPNet) - } else { - bldr.Field(i).(*InetBuilder).AppendNull() - } - case schema.TypeInetArray: - if c[j][i].(*schema.InetArray).Status == schema.Present { - listBldr := bldr.Field(i).(*array.ListBuilder) - listBldr.Append(true) - for _, e := range c[j][i].(*schema.InetArray).Elements { - listBldr.ValueBuilder().(*InetBuilder).Append(*e.IPNet) - } - } else { - bldr.Field(i).(*array.ListBuilder).AppendNull() - } - case schema.TypeCIDR: - if c[j][i].(*schema.CIDR).Status == schema.Present { - bldr.Field(i).(*InetBuilder).Append(*c[j][i].(*schema.CIDR).IPNet) - } else { - bldr.Field(i).(*InetBuilder).AppendNull() - } - case schema.TypeCIDRArray: - if c[j][i].(*schema.CIDRArray).Status == schema.Present { - listBldr := bldr.Field(i).(*array.ListBuilder) - listBldr.Append(true) - for _, e := range c[j][i].(*schema.CIDRArray).Elements { - listBldr.ValueBuilder().(*InetBuilder).Append(*e.IPNet) - } - } else { - bldr.Field(i).(*array.ListBuilder).AppendNull() - } - case schema.TypeMacAddr: - if c[j][i].(*schema.Macaddr).Status == schema.Present { - bldr.Field(i).(*MacBuilder).Append(c[j][i].(*schema.Macaddr).Addr) - } else { - bldr.Field(i).(*MacBuilder).AppendNull() - } - case schema.TypeMacAddrArray: - if c[j][i].(*schema.MacaddrArray).Status == schema.Present { - listBldr := bldr.Field(i).(*array.ListBuilder) - listBldr.Append(true) - for _, e := range c[j][i].(*schema.MacaddrArray).Elements { - listBldr.ValueBuilder().(*MacBuilder).Append(e.Addr) - } - } else { - bldr.Field(i).(*array.ListBuilder).AppendNull() - } - } - } - } - - return bldr.NewRecord() -} diff --git a/internal/cqarrow/arrow_test.go b/internal/cqarrow/arrow_test.go deleted file mode 100644 index d628a1a9..00000000 --- a/internal/cqarrow/arrow_test.go +++ /dev/null @@ -1,89 +0,0 @@ -package cqarrow - -import ( - "encoding/json" - "fmt" - "testing" - - "github.com/apache/arrow/go/v12/arrow" - "github.com/apache/arrow/go/v12/arrow/memory" - "github.com/cloudquery/plugin-sdk/schema" - "github.com/cloudquery/plugin-sdk/testdata" - "github.com/google/go-cmp/cmp" -) - -func DiffSchema(sc, o *arrow.Schema) (string, bool) { - switch { - case sc == o: - return "", true - case sc == nil || o == nil: - return "one of the objects is nil", false - case len(sc.Fields()) != len(o.Fields()): - return "diff len", false - case sc.Endianness() != o.Endianness(): - return "Diff endianness", false - } - - for i := range sc.Fields() { - f1 := sc.Fields()[i] - o1 := o.Fields()[i] - if !f1.Equal(o1) { - return fmt.Sprintf("diff at %d", i), false - } - } - return "", true -} - -func TestCQSchemaToArrow(t *testing.T) { - expecetdSchema := arrow.NewSchema([]arrow.Field{ - {Name: "_cq_id", Type: NewUUIDType()}, - {Name: "_cq_parent_id", Type: NewUUIDType(), Nullable: true}, - {Name: "bool", Type: arrow.FixedWidthTypes.Boolean, Nullable: true}, - {Name: "int", Type: arrow.PrimitiveTypes.Int64, Nullable: true}, - {Name: "float", Type: arrow.PrimitiveTypes.Float64, Nullable: true}, - {Name: "uuid", Type: NewUUIDType(), Nullable: true, - Metadata: arrow.MetadataFrom(map[string]string{ - MetadataPrimaryKey: MetadataPrimaryKeyTrue, - })}, - {Name: "text", Type: arrow.BinaryTypes.String, Nullable: true}, - {Name: "text_with_null", Type: arrow.BinaryTypes.String, Nullable: true}, - {Name: "bytea", Type: arrow.BinaryTypes.Binary, Nullable: true}, - {Name: "text_array", Type: arrow.ListOf(arrow.BinaryTypes.String), Nullable: true}, - {Name: "text_array_with_null", Type: arrow.ListOf(arrow.BinaryTypes.String), Nullable: true}, - {Name: "int_array", Type: arrow.ListOf(arrow.PrimitiveTypes.Int64), Nullable: true}, - {Name: "timestamp", Type: arrow.FixedWidthTypes.Timestamp_us, Nullable: true}, - {Name: "json", Type: NewJSONType(), Nullable: true}, - {Name: "uuid_array", Type: arrow.ListOf(NewUUIDType()), Nullable: true}, - {Name: "inet", Type: NewInetType(), Nullable: true}, - {Name: "inet_array", Type: arrow.ListOf(NewInetType()), Nullable: true}, - {Name: "cidr", Type: NewInetType(), Nullable: true}, - {Name: "cidr_array", Type: arrow.ListOf(NewInetType()), Nullable: true}, - {Name: "macaddr", Type: NewMacType(), Nullable: true}, - {Name: "macaddr_array", Type: arrow.ListOf(NewMacType()), Nullable: true}, - }, nil) - - testTable := testdata.TestSourceTable("test_table") - arrowSchema := CQSchemaToArrow(testTable) - if diff := cmp.Diff(arrowSchema.String(), expecetdSchema.String()); diff != "" { - t.Errorf(diff) - } - // if diff, _ := DiffSchema(arrowSchema, expecetdSchema); diff != "" { - // t.Errorf(diff) - // } - if !arrowSchema.Equal(expecetdSchema) { - t.Errorf("got:\n%v\nwant:\n%v\n", arrowSchema, expecetdSchema) - } -} - -func TestCQTypesToRecord(t *testing.T) { - testTable := testdata.TestSourceTable("test_table") - testCqTypes := testdata.GenTestData(testTable) - arrowSchema := CQSchemaToArrow(testTable) - mem := memory.NewGoAllocator() - record := CQTypesToRecord(mem, []schema.CQTypes{testCqTypes}, arrowSchema) - str, err := json.MarshalIndent(record, "", " ") - if err != nil { - t.Error(err) - } - t.Log(string(str)) -} diff --git a/internal/cqarrow/inet.go b/internal/cqarrow/inet.go deleted file mode 100644 index 8b29390d..00000000 --- a/internal/cqarrow/inet.go +++ /dev/null @@ -1,183 +0,0 @@ -package cqarrow - -import ( - "bytes" - "fmt" - "net" - "reflect" - "strings" - - "github.com/apache/arrow/go/v12/arrow" - "github.com/apache/arrow/go/v12/arrow/array" - "github.com/goccy/go-json" -) - -type InetBuilder struct { - *array.ExtensionBuilder -} - -func NewInetBuilder(bldr *array.ExtensionBuilder) *InetBuilder { - b := &InetBuilder{ - ExtensionBuilder: bldr, - } - return b -} - -func (b *InetBuilder) Append(v net.IPNet) { - b.ExtensionBuilder.Builder.(*array.StringBuilder).Append(v.String()) -} - -func (b *InetBuilder) UnsafeAppend(v net.IPNet) { - b.ExtensionBuilder.Builder.(*array.StringBuilder).UnsafeAppend([]byte(v.String())) -} - -func (b *InetBuilder) AppendValues(v []net.IPNet, valid []bool) { - data := make([]string, len(v)) - for i, v := range v { - data[i] = v.String() - } - b.ExtensionBuilder.Builder.(*array.StringBuilder).AppendValues(data, valid) -} - -func (b *InetBuilder) UnmarshalOne(dec *json.Decoder) error { - t, err := dec.Token() - if err != nil { - return err - } - - var val net.IPNet - switch v := t.(type) { - case string: - _, data, err := net.ParseCIDR(v) - if err != nil { - return err - } - val = *data - case []byte: - _, data, err := net.ParseCIDR(string(v)) - if err != nil { - return err - } - val = *data - case nil: - b.AppendNull() - return nil - default: - return &json.UnmarshalTypeError{ - Value: fmt.Sprint(t), - Type: reflect.TypeOf([]byte{}), - Offset: dec.InputOffset(), - Struct: "String", - } - } - - b.Append(val) - return nil -} - -func (b *InetBuilder) Unmarshal(dec *json.Decoder) error { - for dec.More() { - if err := b.UnmarshalOne(dec); err != nil { - return err - } - } - return nil -} - -func (b *InetBuilder) UnmarshalJSON(data []byte) error { - dec := json.NewDecoder(bytes.NewReader(data)) - t, err := dec.Token() - if err != nil { - return err - } - - if delim, ok := t.(json.Delim); !ok || delim != '[' { - return fmt.Errorf("fixed size binary builder must unpack from json array, found %s", delim) - } - - return b.Unmarshal(dec) -} - -// InetArray is a simple array which is a FixedSizeBinary(16) -type InetArray struct { - array.ExtensionArrayBase -} - -func (a InetArray) String() string { - arr := a.Storage().(*array.String) - o := new(strings.Builder) - o.WriteString("[") - for i := 0; i < arr.Len(); i++ { - if i > 0 { - o.WriteString(" ") - } - switch { - case a.IsNull(i): - o.WriteString("(null)") - default: - fmt.Fprintf(o, "\"%s\"", arr.Value(i)) - } - } - o.WriteString("]") - return o.String() -} - -func (a *InetArray) GetOneForMarshal(i int) any { - arr := a.Storage().(*array.String) - if a.IsValid(i) { - _, ipnet, err := net.ParseCIDR(arr.Value(i)) - if err != nil { - panic(fmt.Errorf("invalid ip+net: %w", err)) - } - return ipnet.String() - } - return nil -} - -// InetType is a simple extension type that represents a StringType -// to be used for representing IP Addresses and CIDRs -type InetType struct { - arrow.ExtensionBase -} - -// NewInetType is a convenience function to create an instance of InetType -// with the correct storage type -func NewInetType() *InetType { - return &InetType{ - ExtensionBase: arrow.ExtensionBase{ - Storage: &arrow.StringType{}}} -} - -func (InetType) ArrayType() reflect.Type { - return reflect.TypeOf(InetArray{}) -} - -func (InetType) ExtensionName() string { - return "inet" -} - -// Serialize returns "inet-serialized" for testing proper metadata passing -func (InetType) Serialize() string { - return "inet-serialized" -} - -// Deserialize expects storageType to be StringType and the data to be -// "inet-serialized" in order to correctly create a InetType for testing deserialize. -func (InetType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) { - if data != "inet-serialized" { - return nil, fmt.Errorf("type identifier did not match: '%s'", data) - } - if !arrow.TypeEqual(storageType, &arrow.StringType{}) { - return nil, fmt.Errorf("invalid storage type for InetType: %s", storageType.Name()) - } - return NewInetType(), nil -} - -// InetType are equal if both are named "inet" -func (u InetType) ExtensionEquals(other arrow.ExtensionType) bool { - return u.ExtensionName() == other.ExtensionName() -} - -func (InetType) NewBuilder(bldr *array.ExtensionBuilder) array.Builder { - return NewInetBuilder(bldr) -} diff --git a/internal/cqarrow/inet_test.go b/internal/cqarrow/inet_test.go deleted file mode 100644 index 8f573c8d..00000000 --- a/internal/cqarrow/inet_test.go +++ /dev/null @@ -1,64 +0,0 @@ -package cqarrow - -import ( - "net" - "testing" - - "github.com/apache/arrow/go/v12/arrow/array" - "github.com/apache/arrow/go/v12/arrow/memory" - "github.com/stretchr/testify/require" -) - -func mustParseInet(s string) net.IPNet { - _, ipnet, err := net.ParseCIDR(s) - if err != nil { - panic(err) - } - return *ipnet -} - -func TestInetBuilder(t *testing.T) { - mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) - defer mem.AssertSize(t, 0) - - b := NewInetBuilder(array.NewExtensionBuilder(mem, NewInetType())) - - b.Append(mustParseInet("192.168.0.0/24")) - b.AppendNull() - b.Append(mustParseInet("192.168.0.0/25")) - b.AppendNull() - - require.Equal(t, 4, b.Len(), "unexpected Len()") - require.Equal(t, 2, b.NullN(), "unexpected NullN()") - - values := []net.IPNet{ - mustParseInet("192.168.0.0/26"), - mustParseInet("192.168.0.0/27"), - } - b.AppendValues(values, []bool{true, true}) - - require.Equal(t, 6, b.Len(), "unexpected Len()") - - a := b.NewArray() - - // check state of builder after NewInetBuilder - require.Zero(t, b.Len(), "unexpected ArrayBuilder.Len(), did not reset state") - require.Zero(t, b.Cap(), "unexpected ArrayBuilder.Cap(), did not reset state") - require.Zero(t, b.NullN(), "unexpected ArrayBuilder.NullN(), did not reset state") - - require.Equal(t, `["192.168.0.0/24" (null) "192.168.0.0/25" (null) "192.168.0.0/26" "192.168.0.0/27"]`, a.String()) - st, err := a.MarshalJSON() - require.NoError(t, err) - - b.Release() - a.Release() - - b = NewInetBuilder(array.NewExtensionBuilder(mem, NewInetType())) - err = b.UnmarshalJSON(st) - require.NoError(t, err) - - a = b.NewArray() - require.Equal(t, `["192.168.0.0/24" (null) "192.168.0.0/25" (null) "192.168.0.0/26" "192.168.0.0/27"]`, a.String()) - b.Release() - a.Release() -} diff --git a/internal/cqarrow/json.go b/internal/cqarrow/json.go deleted file mode 100644 index 8a0146c3..00000000 --- a/internal/cqarrow/json.go +++ /dev/null @@ -1,178 +0,0 @@ -package cqarrow - -import ( - "fmt" - "reflect" - "strings" - - "github.com/goccy/go-json" - - "github.com/apache/arrow/go/v12/arrow" - "github.com/apache/arrow/go/v12/arrow/array" -) - -type JSONBuilder struct { - *array.ExtensionBuilder -} - -func NewJSONBuilder(bldr *array.ExtensionBuilder) *JSONBuilder { - b := &JSONBuilder{ - ExtensionBuilder: bldr, - } - return b -} - -func (b *JSONBuilder) Append(v any) { - if v == nil { - b.AppendNull() - return - } - bytes, err := json.Marshal(v) - if err != nil { - panic(err) - } - b.ExtensionBuilder.Builder.(*array.BinaryBuilder).Append(bytes) -} - -func (b *JSONBuilder) UnsafeAppend(v any) { - bytes, err := json.Marshal(v) - if err != nil { - panic(err) - } - b.ExtensionBuilder.Builder.(*array.BinaryBuilder).UnsafeAppend(bytes) -} - -func (b *JSONBuilder) AppendValues(v []any, valid []bool) { - data := make([][]byte, len(v)) - for i := range v { - bytes, err := json.Marshal(v[i]) - if err != nil { - panic(err) - } - data[i] = bytes - } - b.ExtensionBuilder.Builder.(*array.BinaryBuilder).AppendValues(data, valid) -} - -func (b *JSONBuilder) UnmarshalJSON(data []byte) error { - var a []any - if err := json.Unmarshal(data, &a); err != nil { - return err - } - valid := make([]bool, len(a)) - for i := range a { - valid[i] = a[i] != nil - } - b.AppendValues(a, valid) - return nil -} - -// JSONArray is a simple array which is a Binary -type JSONArray struct { - array.ExtensionArrayBase -} - -func (a JSONArray) String() string { - arr := a.Storage().(*array.Binary) - o := new(strings.Builder) - o.WriteString("[") - for i := 0; i < arr.Len(); i++ { - if i > 0 { - o.WriteString(" ") - } - switch { - case a.IsNull(i): - o.WriteString("(null)") - default: - fmt.Fprintf(o, "\"%s\"", arr.Value(i)) - } - } - o.WriteString("]") - return o.String() -} - -func (a *JSONArray) MarshalJSON() ([]byte, error) { - arr := a.Storage().(*array.Binary) - vals := make([]any, a.Len()) - for i := 0; i < a.Len(); i++ { - if a.IsValid(i) { - err := json.Unmarshal(arr.Value(i), &vals[i]) - if err != nil { - panic(fmt.Errorf("invalid json: %w", err)) - } - } else { - vals[i] = nil - } - } - return json.Marshal(vals) -} - -func (a *JSONArray) GetOneForMarshal(i int) any { - arr := a.Storage().(*array.Binary) - if a.IsValid(i) { - var data any - err := json.Unmarshal(arr.Value(i), &data) - if err != nil { - panic(fmt.Errorf("invalid json: %w", err)) - } - return data - } - return nil -} - -// JSONType is a simple extension type that represents a BinaryType -// to be used for representing JSONs -type JSONType struct { - arrow.ExtensionBase -} - -// NewJSONType is a convenience function to create an instance of JSONType -// with the correct storage type -func NewJSONType() *JSONType { - return &JSONType{ - ExtensionBase: arrow.ExtensionBase{ - Storage: &arrow.BinaryType{}}} -} - -// ArrayType returns TypeOf(JSONType) for constructing JSON arrays -func (JSONType) ArrayType() reflect.Type { - return reflect.TypeOf(JSONArray{}) -} - -func (JSONType) ExtensionName() string { - return "json" -} - -func (e JSONType) String() string { - return fmt.Sprintf("extension_type", e.Storage) -} - -func (e JSONType) MarshalJSON() ([]byte, error) { - return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil -} - -// Serialize returns "json-serialized" for testing proper metadata passing -func (JSONType) Serialize() string { - return "json-serialized" -} - -// Deserialize expects storageType to be BinaryBuilder and the data to be -// "json-serialized" in order to correctly create a JSONType for testing deserialize. -func (JSONType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) { - if data != "json-serialized" { - return nil, fmt.Errorf("type identifier did not match: '%s'", data) - } - if !arrow.TypeEqual(storageType, &arrow.BinaryType{}) { - return nil, fmt.Errorf("invalid storage type for JSONType: %s", storageType.Name()) - } - return NewJSONType(), nil -} - -// ExtensionEquals returns true if both extensions have the same name -func (e JSONType) ExtensionEquals(other arrow.ExtensionType) bool { - return e.ExtensionName() == other.ExtensionName() -} - -func (JSONType) NewBuilder(bldr *array.ExtensionBuilder) array.Builder { - return NewJSONBuilder(bldr) -} diff --git a/internal/cqarrow/json_test.go b/internal/cqarrow/json_test.go deleted file mode 100644 index e98c1f03..00000000 --- a/internal/cqarrow/json_test.go +++ /dev/null @@ -1,53 +0,0 @@ -package cqarrow - -import ( - "testing" - - "github.com/apache/arrow/go/v12/arrow/array" - "github.com/apache/arrow/go/v12/arrow/memory" - "github.com/stretchr/testify/require" -) - -func TestJSONBuilder(t *testing.T) { - mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) - defer mem.AssertSize(t, 0) - - b := NewJSONBuilder(array.NewExtensionBuilder(mem, NewJSONType())) - b.Append(map[string]any{"a": 1, "b": 2}) - b.AppendNull() - b.Append(map[string]any{"c": 3, "d": 4}) - b.AppendNull() - - require.Equal(t, 4, b.Len(), "unexpected Len()") - require.Equal(t, 2, b.NullN(), "unexpected NullN()") - - values := []any{ - map[string]any{"e": 5, "f": 6}, - map[string]any{"g": 7, "h": 8}, - } - b.AppendValues(values, []bool{true, true}) - - require.Equal(t, 6, b.Len(), "unexpected Len()") - - a := b.NewArray() - - // check state of builder after NewJSONBuilder - require.Zero(t, b.Len(), "unexpected ArrayBuilder.Len(), NewJSONBuilder did not reset state") - require.Zero(t, b.Cap(), "unexpected ArrayBuilder.Cap(), NewJSONBuilder did not reset state") - require.Zero(t, b.NullN(), "unexpected ArrayBuilder.NullN(), NewJSONBuilder did not reset state") - require.Equal(t, `["{"a":1,"b":2}" (null) "{"c":3,"d":4}" (null) "{"e":5,"f":6}" "{"g":7,"h":8}"]`, a.String()) - st, err := a.MarshalJSON() - require.NoError(t, err) - - b.Release() - a.Release() - - b = NewJSONBuilder(array.NewExtensionBuilder(mem, NewJSONType())) - err = b.UnmarshalJSON(st) - require.NoError(t, err) - - a = b.NewArray() - require.Equal(t, `["{"a":1,"b":2}" (null) "{"c":3,"d":4}" (null) "{"e":5,"f":6}" "{"g":7,"h":8}"]`, a.String()) - b.Release() - a.Release() -} diff --git a/internal/cqarrow/mac.go b/internal/cqarrow/mac.go deleted file mode 100644 index 0b2671a3..00000000 --- a/internal/cqarrow/mac.go +++ /dev/null @@ -1,190 +0,0 @@ -package cqarrow - -import ( - "bytes" - "fmt" - "net" - "reflect" - "strings" - - "github.com/apache/arrow/go/v12/arrow" - "github.com/apache/arrow/go/v12/arrow/array" - "github.com/goccy/go-json" -) - -type MacBuilder struct { - *array.ExtensionBuilder -} - -func NewMacBuilder(bldr *array.ExtensionBuilder) *MacBuilder { - b := &MacBuilder{ - ExtensionBuilder: bldr, - } - return b -} - -func (b *MacBuilder) Append(v net.HardwareAddr) { - b.ExtensionBuilder.Builder.(*array.BinaryBuilder).Append(v[:]) -} - -func (b *MacBuilder) UnsafeAppend(v net.HardwareAddr) { - b.ExtensionBuilder.Builder.(*array.BinaryBuilder).UnsafeAppend(v[:]) -} - -func (b *MacBuilder) AppendValues(v []net.HardwareAddr, valid []bool) { - data := make([][]byte, len(v)) - for i, v := range v { - data[i] = v - } - b.ExtensionBuilder.Builder.(*array.BinaryBuilder).AppendValues(data, valid) -} - -func (b *MacBuilder) UnmarshalOne(dec *json.Decoder) error { - t, err := dec.Token() - if err != nil { - return err - } - - var val net.HardwareAddr - switch v := t.(type) { - case string: - data, err := net.ParseMAC(v) - if err != nil { - return err - } - val = data - case []byte: - val = net.HardwareAddr(v) - case nil: - b.AppendNull() - return nil - default: - return &json.UnmarshalTypeError{ - Value: fmt.Sprint(t), - Type: reflect.TypeOf([]byte{}), - Offset: dec.InputOffset(), - Struct: "Binary", - } - } - - b.Append(val) - return nil -} - -func (b *MacBuilder) Unmarshal(dec *json.Decoder) error { - for dec.More() { - if err := b.UnmarshalOne(dec); err != nil { - return err - } - } - return nil -} - -func (b *MacBuilder) UnmarshalJSON(data []byte) error { - dec := json.NewDecoder(bytes.NewReader(data)) - t, err := dec.Token() - if err != nil { - return err - } - - if delim, ok := t.(json.Delim); !ok || delim != '[' { - return fmt.Errorf("fixed size binary builder must unpack from json array, found %s", delim) - } - - return b.Unmarshal(dec) -} - -// MacArray is a simple array which is a wrapper around a BinaryArray -type MacArray struct { - array.ExtensionArrayBase -} - -func (a MacArray) String() string { - arr := a.Storage().(*array.Binary) - o := new(strings.Builder) - o.WriteString("[") - for i := 0; i < arr.Len(); i++ { - if i > 0 { - o.WriteString(" ") - } - switch { - case a.IsNull(i): - o.WriteString("(null)") - default: - mac := net.HardwareAddr(arr.Value(i)) - fmt.Fprintf(o, "\"%s\"", mac) - } - } - o.WriteString("]") - return o.String() -} - -func (a *MacArray) MarshalJSON() ([]byte, error) { - arr := a.Storage().(*array.Binary) - vals := make([]any, a.Len()) - for i := 0; i < a.Len(); i++ { - if a.IsValid(i) { - vals[i] = net.HardwareAddr(arr.Value(i)).String() - } else { - vals[i] = nil - } - } - return json.Marshal(vals) -} - -func (a *MacArray) GetOneForMarshal(i int) any { - arr := a.Storage().(*array.Binary) - if a.IsValid(i) { - return net.HardwareAddr(arr.Value(i)).String() - } - return nil -} - -// MacType is a simple extension type that represents a BinaryType -// to be used for representing mac addresses. -type MacType struct { - arrow.ExtensionBase -} - -// NewMacType is a convenience function to create an instance of MacType -// with the correct storage type -func NewMacType() *MacType { - return &MacType{ - ExtensionBase: arrow.ExtensionBase{ - Storage: &arrow.BinaryType{}}} -} - -// ArrayType returns TypeOf(MacArray) for constructing mac arrays -func (MacType) ArrayType() reflect.Type { - return reflect.TypeOf(MacArray{}) -} - -func (MacType) ExtensionName() string { - return "mac" -} - -// Serialize returns "mac-serialized" for testing proper metadata passing -func (MacType) Serialize() string { - return "mac-serialized" -} - -// Deserialize expects storageType to be FixedSizeBinaryType{ByteWidth: 16} and the data to be -// "mac-serialized" in order to correctly create a MacType for testing deserialize. -func (MacType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) { - if data != "mac-serialized" { - return nil, fmt.Errorf("type identifier did not match: '%s'", data) - } - if !arrow.TypeEqual(storageType, &arrow.BinaryType{}) { - return nil, fmt.Errorf("invalid storage type for MacType: %s", storageType.Name()) - } - return NewInetType(), nil -} - -// MacTypes are equal if both are named "mac" -func (u MacType) ExtensionEquals(other arrow.ExtensionType) bool { - return u.ExtensionName() == other.ExtensionName() -} - -func (MacType) NewBuilder(bldr *array.ExtensionBuilder) array.Builder { - return NewMacBuilder(bldr) -} diff --git a/internal/cqarrow/mac_test.go b/internal/cqarrow/mac_test.go deleted file mode 100644 index 58c70c7f..00000000 --- a/internal/cqarrow/mac_test.go +++ /dev/null @@ -1,64 +0,0 @@ -package cqarrow - -import ( - "net" - "testing" - - "github.com/apache/arrow/go/v12/arrow/array" - "github.com/apache/arrow/go/v12/arrow/memory" - "github.com/stretchr/testify/require" -) - -func mustParseMac(s string) net.HardwareAddr { - mac, err := net.ParseMAC(s) - if err != nil { - panic(err) - } - return mac -} - -func TestMacBuilder(t *testing.T) { - mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) - defer mem.AssertSize(t, 0) - - b := NewMacBuilder(array.NewExtensionBuilder(mem, NewMacType())) - - b.Append(mustParseMac("00:00:00:00:00:01")) - b.AppendNull() - b.Append(mustParseMac("00:00:00:00:00:02")) - b.AppendNull() - - require.Equal(t, 4, b.Len(), "unexpected Len()") - require.Equal(t, 2, b.NullN(), "unexpected NullN()") - - values := []net.HardwareAddr{ - mustParseMac("00:00:00:00:00:03"), - mustParseMac("00:00:00:00:00:04"), - } - b.AppendValues(values, []bool{true, true}) - - require.Equal(t, 6, b.Len(), "unexpected Len()") - - a := b.NewArray() - - // check state of builder after NewArray - require.Zero(t, b.Len(), "unexpected ArrayBuilder.Len(), did not reset state") - require.Zero(t, b.Cap(), "unexpected ArrayBuilder.Cap(), did not reset state") - require.Zero(t, b.NullN(), "unexpected ArrayBuilder.NullN(), did not reset state") - - require.Equal(t, `["00:00:00:00:00:01" (null) "00:00:00:00:00:02" (null) "00:00:00:00:00:03" "00:00:00:00:00:04"]`, a.String()) - st, err := a.MarshalJSON() - require.NoError(t, err) - - b.Release() - a.Release() - - b = NewMacBuilder(array.NewExtensionBuilder(mem, NewMacType())) - err = b.UnmarshalJSON(st) - require.NoError(t, err) - - a = b.NewArray() - require.Equal(t, `["00:00:00:00:00:01" (null) "00:00:00:00:00:02" (null) "00:00:00:00:00:03" "00:00:00:00:00:04"]`, a.String()) - b.Release() - a.Release() -} diff --git a/internal/cqarrow/uuid.go b/internal/cqarrow/uuid.go deleted file mode 100644 index 5b1dc980..00000000 --- a/internal/cqarrow/uuid.go +++ /dev/null @@ -1,222 +0,0 @@ -package cqarrow - -import ( - "bytes" - "fmt" - "reflect" - "strings" - - "github.com/goccy/go-json" - - "github.com/apache/arrow/go/v12/arrow" - "github.com/apache/arrow/go/v12/arrow/array" - "github.com/google/uuid" -) - -type UUIDBuilder struct { - *array.ExtensionBuilder -} - -func NewUUIDBuilder(bldr *array.ExtensionBuilder) *UUIDBuilder { - b := &UUIDBuilder{ - ExtensionBuilder: bldr, - } - return b -} - -func (b *UUIDBuilder) Append(v uuid.UUID) { - b.ExtensionBuilder.Builder.(*array.FixedSizeBinaryBuilder).Append(v[:]) -} - -func (b *UUIDBuilder) UnsafeAppend(v uuid.UUID) { - b.ExtensionBuilder.Builder.(*array.FixedSizeBinaryBuilder).UnsafeAppend(v[:]) -} - -func (b *UUIDBuilder) AppendValues(v []uuid.UUID, valid []bool) { - data := make([][]byte, len(v)) - for i := range v { - data[i] = v[i][:] - } - b.ExtensionBuilder.Builder.(*array.FixedSizeBinaryBuilder).AppendValues(data, valid) -} - -func (b *UUIDBuilder) UnmarshalOne(dec *json.Decoder) error { - t, err := dec.Token() - if err != nil { - return err - } - - var val uuid.UUID - switch v := t.(type) { - case string: - data, err := uuid.Parse(v) - if err != nil { - return err - } - val = data - case []byte: - data, err := uuid.ParseBytes(v) - if err != nil { - return err - } - val = data - case nil: - b.AppendNull() - return nil - default: - return &json.UnmarshalTypeError{ - Value: fmt.Sprint(t), - Type: reflect.TypeOf([]byte{}), - Offset: dec.InputOffset(), - Struct: fmt.Sprintf("FixedSizeBinary[%d]", 16), - } - } - - if len(val) != 16 { - return &json.UnmarshalTypeError{ - Value: fmt.Sprint(val), - Type: reflect.TypeOf([]byte{}), - Offset: dec.InputOffset(), - Struct: fmt.Sprintf("FixedSizeBinary[%d]", 16), - } - } - b.Append(val) - return nil -} - -func (b *UUIDBuilder) Unmarshal(dec *json.Decoder) error { - for dec.More() { - if err := b.UnmarshalOne(dec); err != nil { - return err - } - } - return nil -} - -func (b *UUIDBuilder) UnmarshalJSON(data []byte) error { - dec := json.NewDecoder(bytes.NewReader(data)) - t, err := dec.Token() - if err != nil { - return err - } - - if delim, ok := t.(json.Delim); !ok || delim != '[' { - return fmt.Errorf("uuid builder must unpack from json array, found %s", delim) - } - - return b.Unmarshal(dec) -} - -// UUIDArray is a simple array which is a FixedSizeBinary(16) -type UUIDArray struct { - array.ExtensionArrayBase -} - -func (a UUIDArray) String() string { - arr := a.Storage().(*array.FixedSizeBinary) - o := new(strings.Builder) - o.WriteString("[") - for i := 0; i < arr.Len(); i++ { - if i > 0 { - o.WriteString(" ") - } - switch { - case a.IsNull(i): - o.WriteString("(null)") - default: - uuidStr, err := uuid.FromBytes(arr.Value(i)) - if err != nil { - panic(fmt.Errorf("invalid uuid: %w", err)) - } - fmt.Fprintf(o, "%q", uuidStr) - } - } - o.WriteString("]") - return o.String() -} - -func (a *UUIDArray) MarshalJSON() ([]byte, error) { - arr := a.Storage().(*array.FixedSizeBinary) - vals := make([]any, a.Len()) - for i := 0; i < a.Len(); i++ { - if a.IsValid(i) { - uuidStr, err := uuid.FromBytes(arr.Value(i)) - if err != nil { - panic(fmt.Errorf("invalid uuid: %w", err)) - } - vals[i] = uuidStr.String() - } else { - vals[i] = nil - } - } - return json.Marshal(vals) -} - -func (a *UUIDArray) GetOneForMarshal(i int) any { - arr := a.Storage().(*array.FixedSizeBinary) - if a.IsValid(i) { - uuidObj, err := uuid.FromBytes(arr.Value(i)) - if err != nil { - panic(fmt.Errorf("invalid uuid: %w", err)) - } - return uuidObj - } - return nil -} - -// UUIDType is a simple extension type that represents a FixedSizeBinary(16) -// to be used for representing UUIDs -type UUIDType struct { - arrow.ExtensionBase -} - -// NewUUIDType is a convenience function to create an instance of UuidType -// with the correct storage type -func NewUUIDType() *UUIDType { - return &UUIDType{ - ExtensionBase: arrow.ExtensionBase{ - Storage: &arrow.FixedSizeBinaryType{ByteWidth: 16}}} -} - -// ArrayType returns TypeOf(UuidArray) for constructing uuid arrays -func (UUIDType) ArrayType() reflect.Type { - return reflect.TypeOf(UUIDArray{}) -} - -func (UUIDType) ExtensionName() string { - return "uuid" -} - -func (e UUIDType) String() string { - return fmt.Sprintf("extension_type", e.Storage) -} - -func (e UUIDType) MarshalJSON() ([]byte, error) { - return []byte(fmt.Sprintf(`{"name":"%s","metadata":%s}`, e.ExtensionName(), e.Serialize())), nil -} - -// Serialize returns "uuid-serialized" for testing proper metadata passing -func (UUIDType) Serialize() string { - return "uuid-serialized" -} - -// Deserialize expects storageType to be FixedSizeBinaryType{ByteWidth: 16} and the data to be -// "uuid-serialized" in order to correctly create a UuidType for testing deserialize. -func (UUIDType) Deserialize(storageType arrow.DataType, data string) (arrow.ExtensionType, error) { - if data != "uuid-serialized" { - return nil, fmt.Errorf("type identifier did not match: '%s'", data) - } - if !arrow.TypeEqual(storageType, &arrow.FixedSizeBinaryType{ByteWidth: 16}) { - return nil, fmt.Errorf("invalid storage type for UuidType: %s", storageType.Name()) - } - return NewUUIDType(), nil -} - -// UuidTypes are equal if both are named "uuid" -func (e UUIDType) ExtensionEquals(other arrow.ExtensionType) bool { - return e.ExtensionName() == other.ExtensionName() -} - -func (UUIDType) NewBuilder(bldr *array.ExtensionBuilder) array.Builder { - return NewUUIDBuilder(bldr) -} diff --git a/internal/cqarrow/uuid_test.go b/internal/cqarrow/uuid_test.go deleted file mode 100644 index 830bb17c..00000000 --- a/internal/cqarrow/uuid_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package cqarrow - -import ( - "testing" - - "github.com/apache/arrow/go/v12/arrow/array" - "github.com/apache/arrow/go/v12/arrow/memory" - "github.com/google/uuid" - "github.com/stretchr/testify/require" -) - -func TestUUIDBuilder(t *testing.T) { - mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) - defer mem.AssertSize(t, 0) - - b := NewUUIDBuilder(array.NewExtensionBuilder(mem, NewUUIDType())) - - b.Append(uuid.MustParse("00000000-0000-0000-0000-000000000001")) - b.AppendNull() - b.Append(uuid.MustParse("00000000-0000-0000-0000-000000000002")) - b.AppendNull() - - require.Equal(t, 4, b.Len(), "unexpected Len()") - require.Equal(t, 2, b.NullN(), "unexpected NullN()") - - values := []uuid.UUID{ - uuid.MustParse("00000000-0000-0000-0000-000000000003"), - uuid.MustParse("00000000-0000-0000-0000-000000000004"), - } - b.AppendValues(values, []bool{true, true}) - - require.Equal(t, 6, b.Len(), "unexpected Len()") - - a := b.NewArray() - - // check state of builder after NewUUIDBuilder - require.Zero(t, b.Len(), "unexpected ArrayBuilder.Len(), NewUUIDBuilder did not reset state") - require.Zero(t, b.Cap(), "unexpected ArrayBuilder.Cap(), NewUUIDBuilder did not reset state") - require.Zero(t, b.NullN(), "unexpected ArrayBuilder.NullN(), NewUUIDBuilder did not reset state") - - require.Equal(t, `["00000000-0000-0000-0000-000000000001" (null) "00000000-0000-0000-0000-000000000002" (null) "00000000-0000-0000-0000-000000000003" "00000000-0000-0000-0000-000000000004"]`, a.String()) - st, err := a.MarshalJSON() - require.NoError(t, err) - - b.Release() - a.Release() - - b = NewUUIDBuilder(array.NewExtensionBuilder(mem, NewUUIDType())) - err = b.UnmarshalJSON(st) - require.NoError(t, err) - - a = b.NewArray() - require.Equal(t, `["00000000-0000-0000-0000-000000000001" (null) "00000000-0000-0000-0000-000000000002" (null) "00000000-0000-0000-0000-000000000003" "00000000-0000-0000-0000-000000000004"]`, a.String()) - b.Release() - a.Release() -} diff --git a/json/read.go b/json/read.go index 9fcbe811..d5840192 100644 --- a/json/read.go +++ b/json/read.go @@ -2,36 +2,38 @@ package json import ( "bufio" - "encoding/json" "fmt" "io" + "github.com/apache/arrow/go/v12/arrow" + "github.com/apache/arrow/go/v12/arrow/array" + "github.com/apache/arrow/go/v12/arrow/memory" "github.com/cloudquery/plugin-sdk/schema" ) const maxJSONSize = 1024 * 1024 * 20 -func (*Client) Read(f io.Reader, table *schema.Table, sourceName string, res chan<- []any) error { +func (*Client) Read(f io.Reader, table *schema.Table, sourceName string, res chan<- arrow.Record) error { sourceNameIndex := table.Columns.Index(schema.CqSourceNameColumn.Name) if sourceNameIndex == -1 { return fmt.Errorf("could not find column %s in table %s", schema.CqSourceNameColumn.Name, table.Name) } scanner := bufio.NewScanner(f) scanner.Buffer(make([]byte, maxJSONSize), maxJSONSize) + arrowSchema := table.ToArrowSchema() + rb := array.NewRecordBuilder(memory.DefaultAllocator, arrowSchema) + defer rb.Release() for scanner.Scan() { - jsonObj := make(map[string]any, len(table.Columns)) - row := scanner.Bytes() - if err := json.Unmarshal(row, &jsonObj); err != nil { + b := scanner.Bytes() + err := rb.UnmarshalJSON(b) + if err != nil { return err } - if jsonObj[schema.CqSourceNameColumn.Name] != sourceName { + r := rb.NewRecord() + if r.Column(sourceNameIndex).(*array.String).Value(0) != sourceName { continue } - jsonArr := make([]any, len(table.Columns)) - for i, col := range table.Columns { - jsonArr[i] = jsonObj[col.Name] - } - res <- jsonArr + res <- r } return scanner.Err() diff --git a/json/write.go b/json/write.go index eda2121a..c1a294f0 100644 --- a/json/write.go +++ b/json/write.go @@ -4,24 +4,22 @@ import ( "encoding/json" "io" + "github.com/apache/arrow/go/v12/arrow" "github.com/apache/arrow/go/v12/arrow/array" - "github.com/apache/arrow/go/v12/arrow/memory" - "github.com/cloudquery/filetypes/internal/cqarrow" "github.com/cloudquery/plugin-sdk/schema" ) -func (*Client) WriteTableBatch(w io.Writer, table *schema.Table, resources [][]any) error { - arrowSchema := cqarrow.CQSchemaToArrow(table) - cqTypes := make([]schema.CQTypes, len(resources)) - for i := range resources { - cqTypes[i] = make(schema.CQTypes, len(resources[i])) - for j := range resources[i] { - cqTypes[i][j] = resources[i][j].(schema.CQType) +func (c *Client) WriteTableBatch(w io.Writer, _ *schema.Table, records []arrow.Record) error { + for _, r := range records { + err := c.writeTableBatch(w, r) + if err != nil { + return err } } - record := cqarrow.CQTypesToRecord(memory.DefaultAllocator, cqTypes, arrowSchema) - defer record.Release() + return nil +} +func (c *Client) writeTableBatch(w io.Writer, record arrow.Record) error { arr := array.RecordToStructArray(record) defer arr.Release() enc := json.NewEncoder(w) diff --git a/json/write_read_test.go b/json/write_read_test.go index 701a22e2..570e9928 100644 --- a/json/write_read_test.go +++ b/json/write_read_test.go @@ -3,132 +3,50 @@ package json import ( "bytes" "testing" + "time" - "github.com/cloudquery/plugin-sdk/schema" + "github.com/apache/arrow/go/v12/arrow" + "github.com/apache/arrow/go/v12/arrow/memory" + "github.com/cloudquery/plugin-sdk/plugins/destination" "github.com/cloudquery/plugin-sdk/testdata" ) -func GenTestSourceTable(name string) *schema.Table { - return &schema.Table{ - Name: name, - Description: "Test table", - Columns: schema.ColumnList{ - schema.CqSourceNameColumn, - schema.CqIDColumn, - schema.CqParentIDColumn, - { - Name: "bool", - Type: schema.TypeBool, - }, - { - Name: "int", - Type: schema.TypeInt, - }, - { - Name: "float", - Type: schema.TypeFloat, - }, - { - Name: "uuid", - Type: schema.TypeUUID, - CreationOptions: schema.ColumnCreationOptions{PrimaryKey: true}, - }, - { - Name: "text", - Type: schema.TypeString, - }, - { - Name: "text_with_null", - Type: schema.TypeString, - }, - { - Name: "bytea", - Type: schema.TypeByteArray, - }, - { - Name: "text_array", - Type: schema.TypeStringArray, - }, - { - Name: "text_array_with_null", - Type: schema.TypeStringArray, - }, - { - Name: "int_array", - Type: schema.TypeIntArray, - }, - { - Name: "timestamp", - Type: schema.TypeTimestamp, - }, - { - Name: "json", - Type: schema.TypeJSON, - }, - { - Name: "uuid_array", - Type: schema.TypeUUIDArray, - }, - { - Name: "inet", - Type: schema.TypeInet, - }, - { - Name: "inet_array", - Type: schema.TypeInetArray, - }, - { - Name: "cidr", - Type: schema.TypeCIDR, - }, - { - Name: "cidr_array", - Type: schema.TypeCIDRArray, - }, - { - Name: "macaddr", - Type: schema.TypeMacAddr, - }, - { - Name: "macaddr_array", - Type: schema.TypeMacAddrArray, - }, - }, - } -} - func TestWriteRead(t *testing.T) { var b bytes.Buffer table := testdata.TestTable("test") - // table := GenTestSourceTable("test") - cqtypes := testdata.GenTestData(table) - if err := cqtypes[0].Set("test-source"); err != nil { - t.Fatal(err) + sch := table.ToArrowSchema() + sourceName := "test-source" + syncTime := time.Now().UTC().Round(1 * time.Second) + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + opts := testdata.GenTestDataOptions{ + SourceName: sourceName, + SyncTime: syncTime, + MaxRows: 1, } - transformer := &schema.DefaultTransformer{} - transformedValues := schema.TransformWithTransformer(transformer, cqtypes) + records := testdata.GenTestData(mem, sch, opts) + defer func() { + for _, r := range records { + r.Release() + } + }() cl, err := NewClient() if err != nil { t.Fatal(err) } - if err := cl.WriteTableBatch(&b, table, [][]any{transformedValues}); err != nil { + if err := cl.WriteTableBatch(&b, table, records); err != nil { t.Fatal(err) } - ch := make(chan []any) + ch := make(chan arrow.Record) var readErr error go func() { readErr = cl.Read(&b, table, "test-source", ch) close(ch) }() totalCount := 0 - reverseTransformer := &ReverseTransformer{} - for resource := range ch { - gotCqtypes, err := reverseTransformer.ReverseTransformValues(table, resource) - if err != nil { - t.Fatal(err) - } - if diff := cqtypes.Diff(gotCqtypes); diff != "" { + for got := range ch { + if diff := destination.RecordDiff(records[totalCount], got); diff != "" { t.Fatalf("got diff: %s", diff) } totalCount++ diff --git a/parquet/read.go b/parquet/read.go index 2f1de992..aae60cea 100644 --- a/parquet/read.go +++ b/parquet/read.go @@ -5,11 +5,14 @@ import ( "fmt" "io" + "github.com/apache/arrow/go/v12/arrow" + "github.com/apache/arrow/go/v12/arrow/array" + "github.com/apache/arrow/go/v12/arrow/memory" "github.com/cloudquery/plugin-sdk/schema" "github.com/xitongsys/parquet-go/reader" ) -func (*Client) Read(f io.Reader, table *schema.Table, sourceName string, res chan<- []any) error { +func (*Client) Read(f io.Reader, table *schema.Table, sourceName string, res chan<- arrow.Record) error { sourceNameIndex := int64(table.Columns.Index(schema.CqSourceNameColumn.Name)) if sourceNameIndex == -1 { return fmt.Errorf("could not find column %s in table %s", schema.CqSourceNameColumn.Name, table.Name) @@ -27,22 +30,23 @@ func (*Client) Read(f io.Reader, table *schema.Table, sourceName string, res cha } defer r.ReadStop() + arrowSchema := table.ToArrowSchema() for row := int64(0); row < r.GetNumRows(); row++ { - record := make([]any, len(table.Columns)) + rb := array.NewRecordBuilder(memory.DefaultAllocator, arrowSchema) for col := 0; col < len(table.Columns); col++ { vals, _, _, err := r.ReadColumnByIndex(int64(col), 1) if err != nil { return err } if len(vals) == 1 { - record[col] = vals[0] + rb.Field(col) = vals[0] } else { record[col] = vals } } if record[sourceNameIndex] == sourceName { - res <- record + res <- schema.CQTypesToRecord(memory.DefaultAllocator, record, arrowSchema) } } diff --git a/read.go b/read.go index c30311fe..282c3cca 100644 --- a/read.go +++ b/read.go @@ -3,10 +3,11 @@ package filetypes import ( "io" + "github.com/apache/arrow/go/v12/arrow" "github.com/cloudquery/plugin-sdk/schema" ) -func (cl *Client) Read(r io.Reader, table *schema.Table, sourceName string, res chan<- []any) error { +func (cl *Client) Read(r io.Reader, table *schema.Table, sourceName string, res chan<- arrow.Record) error { switch cl.spec.Format { case FormatTypeCSV: if err := cl.csv.Read(r, table, sourceName, res); err != nil { From c4d5ad6c41316645d73782b0be87c6eeb4498be4 Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Thu, 6 Apr 2023 11:43:33 +0100 Subject: [PATCH 02/25] Parquet --- go.mod | 12 ++++++-- go.sum | 14 ++++++++- json/read.go | 4 +-- parquet/read.go | 59 +++++++++++++++++++------------------- parquet/write.go | 27 +++++++++++------ parquet/write_read_test.go | 50 +++++++++++++++++++++----------- read.go | 10 +++---- 7 files changed, 110 insertions(+), 66 deletions(-) diff --git a/go.mod b/go.mod index d9fbc41b..6e3ebdad 100644 --- a/go.mod +++ b/go.mod @@ -25,10 +25,7 @@ require ( ) require ( - github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516 github.com/apache/arrow/go/v12 v12.0.0-20230401212133-adf33cc43010 - github.com/goccy/go-json v0.10.2 - github.com/google/go-cmp v0.5.9 github.com/xitongsys/parquet-go v1.6.2 ) @@ -37,8 +34,12 @@ replace github.com/apache/arrow/go/v12 => github.com/cloudquery/arrow/go/v12 v12 require golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect require ( + github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect github.com/andybalholm/brotli v1.0.5 // indirect + github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516 // indirect github.com/apache/thrift v0.18.1 // indirect + github.com/goccy/go-json v0.10.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/flatbuffers v2.0.8+incompatible // indirect github.com/klauspost/asmfmt v1.3.2 // indirect @@ -48,8 +49,13 @@ require ( github.com/xitongsys/parquet-go-source v0.0.0-20230312005205-fbbcdea5f512 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect golang.org/x/mod v0.9.0 // indirect + golang.org/x/net v0.8.0 // indirect + golang.org/x/sync v0.1.0 // indirect golang.org/x/tools v0.7.0 // indirect golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect + google.golang.org/genproto v0.0.0-20230227214838-9b19f0bdc514 // indirect + google.golang.org/grpc v1.54.0 // indirect + google.golang.org/protobuf v1.30.0 // indirect ) require ( diff --git a/go.sum b/go.sum index 18655b4a..8df6e264 100644 --- a/go.sum +++ b/go.sum @@ -107,6 +107,7 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/GoogleCloudPlatform/cloudsql-proxy v1.29.0/go.mod h1:spvB9eLJH9dutlbPSRmHvSXXHOwGRyeXh1jVdquA2G8= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= +github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= @@ -260,6 +261,8 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= @@ -283,7 +286,6 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-replayers/grpcreplay v1.1.0/go.mod h1:qzAvJ8/wi57zq7gWqaE6AwLM6miiXUQwP1S+I9icmhk= github.com/google/go-replayers/httpreplay v1.1.1/go.mod h1:gN9GeLIs7l6NUoVaSSnv2RiqK1NiwAmD0MrKeC9IIks= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -478,6 +480,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -642,6 +645,8 @@ golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220401154927-543a649e0bdd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -674,6 +679,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -970,6 +976,8 @@ google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2 google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= google.golang.org/genproto v0.0.0-20220401170504-314d38edb7de/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20230227214838-9b19f0bdc514 h1:rtNKfB++wz5mtDY2t5C8TXlU5y52ojSu7tZo0z7u8eQ= +google.golang.org/genproto v0.0.0-20230227214838-9b19f0bdc514/go.mod h1:TvhZT5f700eVlTNwND1xoEZQeWTB2RY/65kplwl/bFA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -998,6 +1006,8 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= +google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1013,6 +1023,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/json/read.go b/json/read.go index d5840192..18652757 100644 --- a/json/read.go +++ b/json/read.go @@ -13,12 +13,12 @@ import ( const maxJSONSize = 1024 * 1024 * 20 -func (*Client) Read(f io.Reader, table *schema.Table, sourceName string, res chan<- arrow.Record) error { +func (*Client) Read(r io.Reader, table *schema.Table, sourceName string, res chan<- arrow.Record) error { sourceNameIndex := table.Columns.Index(schema.CqSourceNameColumn.Name) if sourceNameIndex == -1 { return fmt.Errorf("could not find column %s in table %s", schema.CqSourceNameColumn.Name, table.Name) } - scanner := bufio.NewScanner(f) + scanner := bufio.NewScanner(r) scanner.Buffer(make([]byte, maxJSONSize), maxJSONSize) arrowSchema := table.ToArrowSchema() rb := array.NewRecordBuilder(memory.DefaultAllocator, arrowSchema) diff --git a/parquet/read.go b/parquet/read.go index aae60cea..7c40147a 100644 --- a/parquet/read.go +++ b/parquet/read.go @@ -1,53 +1,54 @@ package parquet import ( - "bytes" + "context" "fmt" "io" "github.com/apache/arrow/go/v12/arrow" - "github.com/apache/arrow/go/v12/arrow/array" "github.com/apache/arrow/go/v12/arrow/memory" + "github.com/apache/arrow/go/v12/parquet" + "github.com/apache/arrow/go/v12/parquet/file" + "github.com/apache/arrow/go/v12/parquet/pqarrow" "github.com/cloudquery/plugin-sdk/schema" - "github.com/xitongsys/parquet-go/reader" ) -func (*Client) Read(f io.Reader, table *schema.Table, sourceName string, res chan<- arrow.Record) error { +type ReaderAtSeeker interface { + io.Reader + io.ReaderAt + io.Seeker +} + +func (*Client) Read(f ReaderAtSeeker, table *schema.Table, sourceName string, res chan<- arrow.Record) error { sourceNameIndex := int64(table.Columns.Index(schema.CqSourceNameColumn.Name)) if sourceNameIndex == -1 { return fmt.Errorf("could not find column %s in table %s", schema.CqSourceNameColumn.Name, table.Name) } - buf := &bytes.Buffer{} - if _, err := io.Copy(buf, f); err != nil { - return err + mem := memory.DefaultAllocator + ctx := context.Background() + props := &parquet.ReaderProperties{ + BufferSize: 0, + FileDecryptProps: nil, + BufferedStreamEnabled: false, } - - s := makeSchema(table.Name, table.Columns) - r, err := reader.NewParquetReader(newPQReader(buf.Bytes()), s, 2) + rdr, err := file.NewParquetReader(f, file.WithReadProps(props)) if err != nil { - return fmt.Errorf("can't create parquet reader: %w", err) + return err } - defer r.ReadStop() - - arrowSchema := table.ToArrowSchema() - for row := int64(0); row < r.GetNumRows(); row++ { - rb := array.NewRecordBuilder(memory.DefaultAllocator, arrowSchema) - for col := 0; col < len(table.Columns); col++ { - vals, _, _, err := r.ReadColumnByIndex(int64(col), 1) - if err != nil { - return err - } - if len(vals) == 1 { - rb.Field(col) = vals[0] - } else { - record[col] = vals - } - } + arrProps := pqarrow.ArrowReadProperties{ + Parallel: false, + BatchSize: 0, + } + fr, err := pqarrow.NewFileReader(rdr, arrProps, mem) - if record[sourceNameIndex] == sourceName { - res <- schema.CQTypesToRecord(memory.DefaultAllocator, record, arrowSchema) + rr, err := fr.GetRecordReader(ctx, nil, nil) + for rr.Next() { + rec, err := rr.Read() + if err != nil { + return err } + res <- rec } return nil diff --git a/parquet/write.go b/parquet/write.go index 7666c112..8e649420 100644 --- a/parquet/write.go +++ b/parquet/write.go @@ -1,16 +1,19 @@ package parquet import ( + "bytes" "encoding/json" "fmt" "io" + "github.com/apache/arrow/go/v12/arrow" "github.com/cloudquery/plugin-sdk/schema" "github.com/xitongsys/parquet-go/parquet" "github.com/xitongsys/parquet-go/writer" ) -func (*Client) WriteTableBatch(w io.Writer, table *schema.Table, resources [][]any) error { +func (*Client) WriteTableBatch(w io.Writer, table *schema.Table, records []arrow.Record) error { + // TODO: use arrow's parquet writer pw, err := writer.NewJSONWriterFromWriter(makeSchema(table.Name, table.Columns), w, 2) if err != nil { return fmt.Errorf("can't create parquet writer: %w", err) @@ -19,15 +22,21 @@ func (*Client) WriteTableBatch(w io.Writer, table *schema.Table, resources [][]a pw.RowGroupSize = 128 * 1024 * 1024 // 128M pw.CompressionType = parquet.CompressionCodec_SNAPPY - for i := range resources { - rec := make(map[string]any, len(table.Columns)) - for j := range table.Columns { - rec[table.Columns[j].Name] = resources[i][j] - } - b, _ := json.Marshal(rec) - if err := pw.Write(b); err != nil { - return err + for _, rec := range records { + for r := int64(0); r < rec.NumRows(); r++ { + m := rec.NewSlice(r, r+1) + b, err := json.Marshal(m) + if err != nil { + return fmt.Errorf("can't marshal record: %w", err) + } + // hacky, but we don't have a better way to get json for a single row + b = bytes.TrimPrefix(b, []byte("[")) + b = bytes.TrimSuffix(b, []byte("]")) + if err := pw.Write(b); err != nil { + return err + } } + pw.Flush(true) } return pw.WriteStop() diff --git a/parquet/write_read_test.go b/parquet/write_read_test.go index 0b00e0d2..b43398d0 100644 --- a/parquet/write_read_test.go +++ b/parquet/write_read_test.go @@ -3,22 +3,38 @@ package parquet import ( "bufio" "bytes" + "io" "testing" + "time" - "github.com/cloudquery/plugin-sdk/schema" + "github.com/apache/arrow/go/v12/arrow" + "github.com/apache/arrow/go/v12/arrow/memory" + "github.com/cloudquery/plugin-sdk/plugins/destination" "github.com/cloudquery/plugin-sdk/testdata" ) func TestWriteRead(t *testing.T) { var b bytes.Buffer table := testdata.TestTable("test") - cqtypes := testdata.GenTestData(table) - if err := cqtypes[0].Set("test-source"); err != nil { - t.Fatal(err) + sch := table.ToArrowSchema() + sourceName := "test-source" + syncTime := time.Now().UTC().Round(1 * time.Second) + // TODO: use checked allocator here; can't right now because there + // are memory leaks in the arrow parquet reader implementation :( + // mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + // defer mem.AssertSize(t, 0) + mem := memory.NewGoAllocator() + opts := testdata.GenTestDataOptions{ + SourceName: sourceName, + SyncTime: syncTime, + MaxRows: 1, } - transformer := &Transformer{} - transformedValues := schema.TransformWithTransformer(transformer, cqtypes) - + records := testdata.GenTestData(mem, sch, opts) + defer func() { + for _, r := range records { + r.Release() + } + }() writer := bufio.NewWriter(&b) reader := bufio.NewReader(&b) @@ -26,25 +42,25 @@ func TestWriteRead(t *testing.T) { if err != nil { t.Fatal(err) } - if err := cl.WriteTableBatch(writer, table, [][]any{transformedValues}); err != nil { + if err := cl.WriteTableBatch(writer, table, records); err != nil { t.Fatal(err) } writer.Flush() - ch := make(chan []any) + rawBytes, err := io.ReadAll(reader) + if err != nil { + t.Fatal(err) + } + byteReader := bytes.NewReader(rawBytes) + ch := make(chan arrow.Record) var readErr error go func() { - readErr = cl.Read(reader, table, "test-source", ch) + readErr = cl.Read(byteReader, table, "test-source", ch) close(ch) }() totalCount := 0 - reverseTransformer := &ReverseTransformer{} - for resource := range ch { - gotCqtypes, err := reverseTransformer.ReverseTransformValues(table, resource) - if err != nil { - t.Fatal(err) - } - if diff := cqtypes.Diff(gotCqtypes); diff != "" { + for got := range ch { + if diff := destination.RecordDiff(records[totalCount], got); diff != "" { t.Fatalf("got diff: %s", diff) } totalCount++ diff --git a/read.go b/read.go index 282c3cca..dbce1c80 100644 --- a/read.go +++ b/read.go @@ -1,24 +1,24 @@ package filetypes import ( - "io" + "os" "github.com/apache/arrow/go/v12/arrow" "github.com/cloudquery/plugin-sdk/schema" ) -func (cl *Client) Read(r io.Reader, table *schema.Table, sourceName string, res chan<- arrow.Record) error { +func (cl *Client) Read(f *os.File, table *schema.Table, sourceName string, res chan<- arrow.Record) error { switch cl.spec.Format { case FormatTypeCSV: - if err := cl.csv.Read(r, table, sourceName, res); err != nil { + if err := cl.csv.Read(f, table, sourceName, res); err != nil { return err } case FormatTypeJSON: - if err := cl.json.Read(r, table, sourceName, res); err != nil { + if err := cl.json.Read(f, table, sourceName, res); err != nil { return err } case FormatTypeParquet: - if err := cl.parquet.Read(r, table, sourceName, res); err != nil { + if err := cl.parquet.Read(f, table, sourceName, res); err != nil { return err } default: From bc2a916432505422f51776c0befaaa8f6e560dab Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Fri, 7 Apr 2023 09:40:14 +0100 Subject: [PATCH 03/25] Update --- go.mod | 5 +- go.sum | 150 +++++++++++++++++++++++++++++++++++-- json/write_read_test.go | 29 +++++++ parquet/pqreader.go | 41 ---------- parquet/read.go | 51 ++++++++++--- parquet/schema.go | 91 ---------------------- parquet/write.go | 73 +++++++++++------- parquet/write_read_test.go | 5 +- 8 files changed, 263 insertions(+), 182 deletions(-) delete mode 100644 parquet/pqreader.go delete mode 100644 parquet/schema.go diff --git a/go.mod b/go.mod index 6e3ebdad..53e0d144 100644 --- a/go.mod +++ b/go.mod @@ -26,10 +26,9 @@ require ( require ( github.com/apache/arrow/go/v12 v12.0.0-20230401212133-adf33cc43010 - github.com/xitongsys/parquet-go v1.6.2 ) -replace github.com/apache/arrow/go/v12 => github.com/cloudquery/arrow/go/v12 v12.0.0-20230403113355-afc433992be2 +replace github.com/apache/arrow/go/v12 => ../arrow/go require golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect @@ -39,6 +38,7 @@ require ( github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516 // indirect github.com/apache/thrift v0.18.1 // indirect github.com/goccy/go-json v0.10.2 // indirect + github.com/goccy/go-reflect v1.2.0 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/flatbuffers v2.0.8+incompatible // indirect @@ -46,7 +46,6 @@ require ( github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect - github.com/xitongsys/parquet-go-source v0.0.0-20230312005205-fbbcdea5f512 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect golang.org/x/mod v0.9.0 // indirect golang.org/x/net v0.8.0 // indirect diff --git a/go.sum b/go.sum index 8df6e264..771dbd4b 100644 --- a/go.sum +++ b/go.sum @@ -67,6 +67,8 @@ contrib.go.opencensus.io/exporter/aws v0.0.0-20200617204711-c478e41e60e9/go.mod contrib.go.opencensus.io/exporter/stackdriver v0.13.10/go.mod h1:I5htMbyta491eUxufwwZPQdcKvvgzMB4O9ni41YnIM8= contrib.go.opencensus.io/integrations/ocsql v0.1.7/go.mod h1:8DsSdjz3F+APR+0z0WkU1aRorQCFfRxvqjUUPMbF3fE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= +git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc= github.com/Azure/azure-amqp-common-go/v3 v3.2.1/go.mod h1:O6X1iYHP7s2x7NjUKsXVhkwWrQhxrd+d8/3rRadj4CI= github.com/Azure/azure-amqp-common-go/v3 v3.2.2/go.mod h1:O6X1iYHP7s2x7NjUKsXVhkwWrQhxrd+d8/3rRadj4CI= github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= @@ -110,13 +112,18 @@ github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvK github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY= +github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516 h1:byKBBF2CKWBjjA4J1ZL2JXttJULvWSl50LegTyRZ728= github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516/go.mod h1:QNYViu/X0HXDHw7m3KXzWSVXIbfUvJqBFe6Gj8/pYA0= -github.com/apache/thrift v0.0.0-20181112125854-24918abba929/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.14.2/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= github.com/apache/thrift v0.18.1 h1:lNhK/1nqjbwbiOPDBPFJVKxgDEGSepKuTh6OLiXW8kg= github.com/apache/thrift v0.18.1/go.mod h1:rdQn/dCcDKEWjjylUeueum4vQEjG2v8v2PqriUnbr+I= github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= @@ -146,6 +153,8 @@ github.com/aws/aws-sdk-go-v2/service/sso v1.11.3/go.mod h1:7UQ/e69kU7LDPtY40OyoH github.com/aws/aws-sdk-go-v2/service/sts v1.16.3/go.mod h1:bfBj0iVmsUyUg4weDB4NxktD9rDGeKSVWnjTnwbx9b8= github.com/aws/smithy-go v1.11.2/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -163,6 +172,7 @@ github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XP github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/colinmarc/hdfs/v2 v2.1.1/go.mod h1:M3x+k8UKKmxtFu++uAZ0OtDU8jR3jnaZIAc6yK4Ue0c= @@ -181,7 +191,9 @@ github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQ github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -190,7 +202,10 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= @@ -202,12 +217,21 @@ github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= github.com/gin-gonic/gin v1.7.3/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= +github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks= +github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= +github.com/go-fonts/liberation v0.2.0/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= +github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmnUIzUY= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U= +github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpxhq9o2S/CELCSUxEWWAuoCUcVCQWv7G2OCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= +github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= @@ -219,8 +243,11 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= +github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-reflect v1.2.0 h1:O0T8rZCuNmGXewnATuKYnkL0xm6o8UNOJZd/gOkb9ms= +github.com/goccy/go-reflect v1.2.0/go.mod h1:n0oYZn8VcV2CkWTxi8B9QjkCoq6GTtCEdfmR66YhFtE= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= @@ -229,6 +256,7 @@ github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzw github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang-sql/sqlexp v0.0.0-20170517235910-f1bb20e5a188/go.mod h1:vXjM/+wXQnTPR4KqTKDgJukSZ6amVRtWMPEjE6sQoK8= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -263,7 +291,6 @@ github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -285,7 +312,9 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-replayers/grpcreplay v1.1.0/go.mod h1:qzAvJ8/wi57zq7gWqaE6AwLM6miiXUQwP1S+I9icmhk= github.com/google/go-replayers/httpreplay v1.1.1/go.mod h1:gN9GeLIs7l6NUoVaSSnv2RiqK1NiwAmD0MrKeC9IIks= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -380,10 +409,12 @@ github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= -github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= @@ -392,6 +423,7 @@ github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0P github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.1.0/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= @@ -399,6 +431,7 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -425,8 +458,10 @@ github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98= github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= @@ -445,7 +480,11 @@ github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3P github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/ncw/swift v1.0.52/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/pborman/getopt v0.0.0-20180729010549-6fdd0a2c7117/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= +github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= +github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= +github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= github.com/pierrec/lz4/v4 v4.1.8/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= @@ -458,8 +497,11 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= @@ -468,6 +510,8 @@ github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OK github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/rs/zerolog v1.29.0 h1:Zes4hju04hjbvkVkOhdl2HpZa+0PmVwigmo8XoORE5w= github.com/rs/zerolog v1.29.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= +github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= +github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= @@ -497,10 +541,6 @@ github.com/thoas/go-funk v0.9.3 h1:7+nAEx3kn5ZJcnDm2Bh23N2yOtweO14bi//dvRtgLpw= github.com/thoas/go-funk v0.9.3/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -github.com/xitongsys/parquet-go v1.5.1/go.mod h1:xUxwM8ELydxh4edHGegYq1pA8NnMKDx0K/GyB0o2bww= -github.com/xitongsys/parquet-go v1.6.2 h1:MhCaXii4eqceKPu9BwrjLqyK10oX9WF+xGhwvwbw7xM= -github.com/xitongsys/parquet-go v1.6.2/go.mod h1:IulAQyalCm0rPiZVNnCgm/PCL64X2tdSVGMQ/UeKqWA= -github.com/xitongsys/parquet-go-source v0.0.0-20190524061010-2b72cbee77d5/go.mod h1:xxCx7Wpym/3QCo6JhujJX51dzSXrwmb0oH6FQb39SEA= github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0/go.mod h1:HYhIKsdns7xz80OgkbgJYrtQY7FjHWHKH6cvN7+czGE= github.com/xitongsys/parquet-go-source v0.0.0-20230312005205-fbbcdea5f512 h1:ghvb6MV6j1i4i/CVeQyaNTM6Egiw65O9gYBGQseeLs0= github.com/xitongsys/parquet-go-source v0.0.0-20230312005205-fbbcdea5f512/go.mod h1:d87JD+hvj9dzdbZNd9zJb9ThSLQfoUNx50KQ6nuf9Wg= @@ -509,7 +549,10 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= @@ -558,20 +601,36 @@ golang.org/x/crypto v0.0.0-20211115234514-b4de73f9ece8/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220511200225-c6db032c6c88/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug= golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200430140353-33d19683fad8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200618115811-c13761719519/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20201208152932-35266b937fa6/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20210216034530-4410531fe030/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -596,6 +655,9 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -637,6 +699,7 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -645,6 +708,8 @@ golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220401154927-543a649e0bdd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -678,6 +743,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -722,6 +788,7 @@ golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210304124612-50617c2ba197/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -740,6 +807,7 @@ golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -756,11 +824,13 @@ golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -770,6 +840,7 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -777,8 +848,10 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220224211638-0e9765cccd65/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -794,6 +867,7 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -825,6 +899,7 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -834,6 +909,9 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -844,7 +922,15 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0= golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= +gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0= gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E= +gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= +gonum.org/v1/plot v0.9.0/go.mod h1:3Pcqqmp6RHvJI72kgb8fThyUnav364FOsdDo2aGW5lY= +gonum.org/v1/plot v0.10.1/go.mod h1:VZW5OlhkL1mysU9vaqNHnsy86inf6Ot+jB3r+BczCEo= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1006,6 +1092,7 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= @@ -1023,6 +1110,7 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1054,7 +1142,55 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= +lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= +modernc.org/cc/v3 v3.36.2/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= +modernc.org/cc/v3 v3.37.0/go.mod h1:vtL+3mdHx/wcj3iEGz84rQa8vEqR6XM84v5Lcvfph20= +modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0= +modernc.org/ccgo/v3 v3.0.0-20220428102840-41399a37e894/go.mod h1:eI31LL8EwEBKPpNpA4bU1/i+sKOwOrQy8D87zWUcRZc= +modernc.org/ccgo/v3 v3.0.0-20220430103911-bc99d88307be/go.mod h1:bwdAnOoaIt8Ax9YdWGjxWsdkPcZyRPHqrOvJxaKAKGw= +modernc.org/ccgo/v3 v3.0.0-20220904174949-82d86e1b6d56/go.mod h1:YSXjPL62P2AMSxBphRHPn7IkzhVHqkvOnRKAKh+W6ZI= +modernc.org/ccgo/v3 v3.16.4/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= +modernc.org/ccgo/v3 v3.16.6/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= +modernc.org/ccgo/v3 v3.16.8/go.mod h1:zNjwkizS+fIFDrDjIAgBSCLkWbJuHF+ar3QRn+Z9aws= +modernc.org/ccgo/v3 v3.16.9/go.mod h1:zNMzC9A9xeNUepy6KuZBbugn3c0Mc9TeiJO4lgvkJDo= +modernc.org/ccgo/v3 v3.16.13-0.20221017192402-261537637ce8/go.mod h1:fUB3Vn0nVPReA+7IG7yZDfjv1TMWjhQP8gCxrFAtL5g= +modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY= +modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= +modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= +modernc.org/libc v0.0.0-20220428101251-2d5f3daf273b/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= +modernc.org/libc v1.16.0/go.mod h1:N4LD6DBE9cf+Dzf9buBlzVJndKr/iJHG97vGLHYnb5A= +modernc.org/libc v1.16.1/go.mod h1:JjJE0eu4yeK7tab2n4S1w8tlWd9MxXLRzheaRnAKymU= +modernc.org/libc v1.16.17/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU= +modernc.org/libc v1.16.19/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= +modernc.org/libc v1.17.0/go.mod h1:XsgLldpP4aWlPlsjqKRdHPqCxCjISdHfM/yeWC5GyW0= +modernc.org/libc v1.17.4/go.mod h1:WNg2ZH56rDEwdropAJeZPQkXmDwh+JCA1s/htl6r2fA= +modernc.org/libc v1.18.0/go.mod h1:vj6zehR5bfc98ipowQOM2nIDUZnVew/wNC/2tOGS+q0= +modernc.org/libc v1.20.3/go.mod h1:ZRfIaEkgrYgZDl6pa4W39HgN5G/yDW+NRmNKZBDFrk0= +modernc.org/libc v1.21.4/go.mod h1:przBsL5RDOZajTVslkugzLBj1evTue36jEomFQOoYuI= +modernc.org/libc v1.22.2/go.mod h1:uvQavJ1pZ0hIoC/jfqNoMLURIMhKzINIWypNM17puug= +modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/memory v1.1.1/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= +modernc.org/memory v1.2.0/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= +modernc.org/memory v1.3.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/memory v1.4.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sqlite v1.18.2/go.mod h1:kvrTLEWgxUcHa2GfHBQtanR1H9ht3hTJNtKpzH9k1u0= +modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= +modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= +modernc.org/tcl v1.13.2/go.mod h1:7CLiGIPo1M8Rv1Mitpv5akc2+8fxUd2y2UzC/MfMzy0= +modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/z v1.5.1/go.mod h1:eWFB510QWW5Th9YGZT81s+LwvaAs3Q2yr4sP0rmLkv8= nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/json/write_read_test.go b/json/write_read_test.go index 570e9928..538d0c59 100644 --- a/json/write_read_test.go +++ b/json/write_read_test.go @@ -11,6 +11,35 @@ import ( "github.com/cloudquery/plugin-sdk/testdata" ) +func TestWrite(t *testing.T) { + var b bytes.Buffer + table := testdata.TestTable("test") + sch := table.ToArrowSchema() + sourceName := "test-source" + syncTime := time.Now().UTC().Round(1 * time.Second) + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) + opts := testdata.GenTestDataOptions{ + SourceName: sourceName, + SyncTime: syncTime, + MaxRows: 1, + } + records := testdata.GenTestData(mem, sch, opts) + defer func() { + for _, r := range records { + r.Release() + } + }() + cl, err := NewClient() + if err != nil { + t.Fatal(err) + } + if err := cl.WriteTableBatch(&b, table, records); err != nil { + t.Fatal(err) + } + t.Log(b.String()) +} + func TestWriteRead(t *testing.T) { var b bytes.Buffer table := testdata.TestTable("test") diff --git a/parquet/pqreader.go b/parquet/pqreader.go deleted file mode 100644 index a4019f0d..00000000 --- a/parquet/pqreader.go +++ /dev/null @@ -1,41 +0,0 @@ -package parquet - -import ( - "bytes" - "fmt" - - "github.com/xitongsys/parquet-go/source" -) - -type pqReader struct { - data []byte - - *bytes.Reader -} - -var _ source.ParquetFile = (*pqReader)(nil) - -func newPQReader(data []byte) *pqReader { - bu := make([]byte, len(data)) - copy(bu, data) - - return &pqReader{ - Reader: bytes.NewReader(bu), - data: bu, - } -} -func (pq *pqReader) Open(string) (source.ParquetFile, error) { - return newPQReader(pq.data), nil -} - -func (*pqReader) Close() error { - return nil -} - -func (*pqReader) Write([]byte) (n int, err error) { - return 0, fmt.Errorf("not implemented") -} - -func (*pqReader) Create(string) (source.ParquetFile, error) { - return nil, fmt.Errorf("not implemented") -} diff --git a/parquet/read.go b/parquet/read.go index 7c40147a..974375de 100644 --- a/parquet/read.go +++ b/parquet/read.go @@ -6,8 +6,9 @@ import ( "io" "github.com/apache/arrow/go/v12/arrow" + "github.com/apache/arrow/go/v12/arrow/array" + "github.com/apache/arrow/go/v12/arrow/compute" "github.com/apache/arrow/go/v12/arrow/memory" - "github.com/apache/arrow/go/v12/parquet" "github.com/apache/arrow/go/v12/parquet/file" "github.com/apache/arrow/go/v12/parquet/pqarrow" "github.com/cloudquery/plugin-sdk/schema" @@ -20,36 +21,62 @@ type ReaderAtSeeker interface { } func (*Client) Read(f ReaderAtSeeker, table *schema.Table, sourceName string, res chan<- arrow.Record) error { + arrowSchema := table.ToArrowSchema() sourceNameIndex := int64(table.Columns.Index(schema.CqSourceNameColumn.Name)) if sourceNameIndex == -1 { return fmt.Errorf("could not find column %s in table %s", schema.CqSourceNameColumn.Name, table.Name) } - mem := memory.DefaultAllocator ctx := context.Background() - props := &parquet.ReaderProperties{ - BufferSize: 0, - FileDecryptProps: nil, - BufferedStreamEnabled: false, - } - rdr, err := file.NewParquetReader(f, file.WithReadProps(props)) + rdr, err := file.NewParquetReader(f) if err != nil { return err } arrProps := pqarrow.ArrowReadProperties{ Parallel: false, - BatchSize: 0, + BatchSize: 1024, } fr, err := pqarrow.NewFileReader(rdr, arrProps, mem) - rr, err := fr.GetRecordReader(ctx, nil, nil) + if err != nil { + return err + } for rr.Next() { - rec, err := rr.Read() + rec := rr.Record() + castRec, err := castRecord(mem, rec, arrowSchema) if err != nil { return err } - res <- rec + res <- castRec + _, err = rr.Read() + if err == io.EOF { + break + } else if err != nil { + return err + } } + rr.Release() return nil } + +func castRecord(mem memory.Allocator, rec arrow.Record, arrowSchema *arrow.Schema) (arrow.Record, error) { + ctx := context.Background() + rb := array.NewRecordBuilder(mem, arrowSchema) + defer rb.Release() + for c := 0; c < int(rec.NumCols()); c++ { + arr, err := compute.CastToType(ctx, rec.Column(c), arrowSchema.Field(c).Type) + if err != nil { + return nil, fmt.Errorf("failed to cast col %v to %v: %w", rec.ColumnName(c), arrowSchema.Field(c).Type, err) + } + b, err := arr.MarshalJSON() + if err != nil { + return nil, fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) + } + err = rb.Field(c).UnmarshalJSON(b) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal col %v: %w", rec.ColumnName(c), err) + } + } + return rb.NewRecord(), nil +} diff --git a/parquet/schema.go b/parquet/schema.go deleted file mode 100644 index 22fae90b..00000000 --- a/parquet/schema.go +++ /dev/null @@ -1,91 +0,0 @@ -package parquet - -import ( - "encoding/json" - "strings" - - "github.com/cloudquery/plugin-sdk/schema" - pschema "github.com/xitongsys/parquet-go/schema" -) - -func makeSchema(tableName string, cols schema.ColumnList) string { - s := pschema.JSONSchemaItemType{ - Tag: `name=` + tableName + `_root, repetitiontype=REQUIRED`, - } - - for _, col := range cols { - var subFields []*pschema.JSONSchemaItemType - - tag := []string{`name=` + col.Name} - - switch col.Type { - case schema.TypeTimestamp: - tag = append(tag, "type=INT64", "convertedtype=TIMESTAMP_MILLIS") - case schema.TypeJSON, schema.TypeString, schema.TypeUUID, schema.TypeCIDR, schema.TypeInet, schema.TypeMacAddr: - tag = append(tag, "type=BYTE_ARRAY", "convertedtype=UTF8") - case schema.TypeFloat: - tag = append(tag, "type=DOUBLE") - case schema.TypeInt: - tag = append(tag, "type=INT64") - case schema.TypeByteArray: - tag = append(tag, "type=BYTE_ARRAY") - case schema.TypeBool: - tag = append(tag, "type=BOOLEAN") - case schema.TypeIntArray: - tag = append(tag, "type=LIST", "repetitiontype=OPTIONAL") - subFields = []*pschema.JSONSchemaItemType{ - { - Tag: "name=element, type=INT64, repetitiontype=OPTIONAL", - }, - } - case schema.TypeStringArray, schema.TypeUUIDArray, schema.TypeCIDRArray, schema.TypeInetArray, schema.TypeMacAddrArray: - tag = append(tag, "type=LIST", "repetitiontype=OPTIONAL") - subFields = []*pschema.JSONSchemaItemType{ - { - Tag: "name=element, type=BYTE_ARRAY, convertedtype=UTF8, repetitiontype=OPTIONAL", - }, - } - default: - panic("unhandled type: " + col.Type.String()) - } - - if !isArray(col.Type) { // array types are handled differently, see above - if col.CreationOptions.PrimaryKey || col.CreationOptions.IncrementalKey { - tag = append(tag, "repetitiontype=REQUIRED") - } else { - tag = append(tag, "repetitiontype=OPTIONAL") - } - } - - s.Fields = append(s.Fields, &pschema.JSONSchemaItemType{ - Tag: strings.Join(tag, ", "), - Fields: subFields, - }) - } - - b, _ := json.Marshal(s) - return string(b) -} - -func isArray(t schema.ValueType) bool { - return arrayElement(t) != schema.TypeInvalid -} - -func arrayElement(t schema.ValueType) schema.ValueType { - switch t { - case schema.TypeIntArray: - return schema.TypeInt - case schema.TypeStringArray: - return schema.TypeString - case schema.TypeUUIDArray: - return schema.TypeUUID - case schema.TypeCIDRArray: - return schema.TypeCIDR - case schema.TypeInetArray: - return schema.TypeInet - case schema.TypeMacAddrArray: - return schema.TypeMacAddr - default: - return schema.TypeInvalid - } -} diff --git a/parquet/write.go b/parquet/write.go index 8e649420..803cb52f 100644 --- a/parquet/write.go +++ b/parquet/write.go @@ -1,43 +1,62 @@ package parquet import ( - "bytes" - "encoding/json" - "fmt" "io" "github.com/apache/arrow/go/v12/arrow" + "github.com/apache/arrow/go/v12/parquet" + "github.com/apache/arrow/go/v12/parquet/pqarrow" "github.com/cloudquery/plugin-sdk/schema" - "github.com/xitongsys/parquet-go/parquet" - "github.com/xitongsys/parquet-go/writer" ) func (*Client) WriteTableBatch(w io.Writer, table *schema.Table, records []arrow.Record) error { - // TODO: use arrow's parquet writer - pw, err := writer.NewJSONWriterFromWriter(makeSchema(table.Name, table.Columns), w, 2) + props := parquet.NewWriterProperties() + arrprops := pqarrow.DefaultWriterProps() + fw, err := pqarrow.NewFileWriter(table.ToArrowSchema(), w, props, arrprops) if err != nil { - return fmt.Errorf("can't create parquet writer: %w", err) + return err } - - pw.RowGroupSize = 128 * 1024 * 1024 // 128M - pw.CompressionType = parquet.CompressionCodec_SNAPPY - for _, rec := range records { - for r := int64(0); r < rec.NumRows(); r++ { - m := rec.NewSlice(r, r+1) - b, err := json.Marshal(m) - if err != nil { - return fmt.Errorf("can't marshal record: %w", err) - } - // hacky, but we don't have a better way to get json for a single row - b = bytes.TrimPrefix(b, []byte("[")) - b = bytes.TrimSuffix(b, []byte("]")) - if err := pw.Write(b); err != nil { - return err - } + if err := fw.Write(rec); err != nil { + return err } - pw.Flush(true) } - - return pw.WriteStop() + return fw.Close() + //parquetSchema, err := makeSchema(table.Name, table.Columns) + //if err != nil { + // return fmt.Errorf("failed to create parquet schema: %w", err) + //} + //wf := writerfile.NewWriterFile(w) + //arrowSchema := table.ToArrowSchema() + //pw, err := writer.NewArrowWriter(arrowSchema, wf, 2) + //if err != nil { + // return fmt.Errorf("can't create parquet writer: %w", err) + //} + // + //pw, err := writer.NewJSONWriterFromWriter(parquetSchema, w, 2) + //if err != nil { + // return fmt.Errorf("can't create parquet writer: %w", err) + //} + // + //pw.RowGroupSize = 128 * 1024 * 1024 // 128M + //pw.CompressionType = parquet.CompressionCodec_SNAPPY + // + //for _, rec := range records { + // for r := int64(0); r < rec.NumRows(); r++ { + // m := rec.NewSlice(r, r+1) + // b, err := json.Marshal(m) + // if err != nil { + // return fmt.Errorf("can't marshal record: %w", err) + // } + // // hacky, but we don't have a better way to get json for a single row + // b = bytes.TrimPrefix(b, []byte("[")) + // b = bytes.TrimSuffix(b, []byte("]")) + // if err := pw.Write(b); err != nil { + // return err + // } + // } + // pw.Flush(true) + //} + // + //return pw.WriteStop() } diff --git a/parquet/write_read_test.go b/parquet/write_read_test.go index b43398d0..aec3cb21 100644 --- a/parquet/write_read_test.go +++ b/parquet/write_read_test.go @@ -45,7 +45,10 @@ func TestWriteRead(t *testing.T) { if err := cl.WriteTableBatch(writer, table, records); err != nil { t.Fatal(err) } - writer.Flush() + err = writer.Flush() + if err != nil { + t.Fatal(err) + } rawBytes, err := io.ReadAll(reader) if err != nil { From e9c8d668086586a0e36b861645808f0ebadf39c8 Mon Sep 17 00:00:00 2001 From: Yevgeny Pats <16490766+yevgenypats@users.noreply.github.com> Date: Sat, 8 Apr 2023 16:51:18 +0300 Subject: [PATCH 04/25] minor fix --- write.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/write.go b/write.go index 3cf7165c..9abb3dfd 100644 --- a/write.go +++ b/write.go @@ -3,10 +3,11 @@ package filetypes import ( "io" + "github.com/apache/arrow/go/v12/arrow" "github.com/cloudquery/plugin-sdk/schema" ) -func (cl *Client) WriteTableBatchFile(w io.Writer, table *schema.Table, resources [][]any) error { +func (cl *Client) WriteTableBatchFile(w io.Writer, table *schema.Table, resources []arrow.Record) error { switch cl.spec.Format { case FormatTypeCSV: if err := cl.csv.WriteTableBatch(w, table, resources); err != nil { From 3e9d94be8696589b0aaabd16a5f71362d878242e Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Tue, 11 Apr 2023 11:08:43 +0100 Subject: [PATCH 05/25] Parquet read/write test passing :tada: --- parquet/read.go | 100 ++++++++++++++++++++++++++----------- parquet/write_read_test.go | 87 +++++++++++++++++++++++++++++++- 2 files changed, 158 insertions(+), 29 deletions(-) diff --git a/parquet/read.go b/parquet/read.go index 974375de..4c7b1e92 100644 --- a/parquet/read.go +++ b/parquet/read.go @@ -6,8 +6,6 @@ import ( "io" "github.com/apache/arrow/go/v12/arrow" - "github.com/apache/arrow/go/v12/arrow/array" - "github.com/apache/arrow/go/v12/arrow/compute" "github.com/apache/arrow/go/v12/arrow/memory" "github.com/apache/arrow/go/v12/parquet/file" "github.com/apache/arrow/go/v12/parquet/pqarrow" @@ -21,7 +19,7 @@ type ReaderAtSeeker interface { } func (*Client) Read(f ReaderAtSeeker, table *schema.Table, sourceName string, res chan<- arrow.Record) error { - arrowSchema := table.ToArrowSchema() + // arrowSchema := table.ToArrowSchema() sourceNameIndex := int64(table.Columns.Index(schema.CqSourceNameColumn.Name)) if sourceNameIndex == -1 { return fmt.Errorf("could not find column %s in table %s", schema.CqSourceNameColumn.Name, table.Name) @@ -43,11 +41,12 @@ func (*Client) Read(f ReaderAtSeeker, table *schema.Table, sourceName string, re } for rr.Next() { rec := rr.Record() - castRec, err := castRecord(mem, rec, arrowSchema) - if err != nil { - return err - } - res <- castRec + //castRec, err := castRecord(mem, rec, arrowSchema) + //if err != nil { + // return err + //} + rec.Retain() + res <- rec _, err = rr.Read() if err == io.EOF { break @@ -60,23 +59,68 @@ func (*Client) Read(f ReaderAtSeeker, table *schema.Table, sourceName string, re return nil } -func castRecord(mem memory.Allocator, rec arrow.Record, arrowSchema *arrow.Schema) (arrow.Record, error) { - ctx := context.Background() - rb := array.NewRecordBuilder(mem, arrowSchema) - defer rb.Release() - for c := 0; c < int(rec.NumCols()); c++ { - arr, err := compute.CastToType(ctx, rec.Column(c), arrowSchema.Field(c).Type) - if err != nil { - return nil, fmt.Errorf("failed to cast col %v to %v: %w", rec.ColumnName(c), arrowSchema.Field(c).Type, err) - } - b, err := arr.MarshalJSON() - if err != nil { - return nil, fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) - } - err = rb.Field(c).UnmarshalJSON(b) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal col %v: %w", rec.ColumnName(c), err) - } - } - return rb.NewRecord(), nil -} +// +//func castRecord(mem memory.Allocator, rec arrow.Record, arrowSchema *arrow.Schema) (arrow.Record, error) { +// ctx := context.Background() +// rb := array.NewRecordBuilder(mem, arrowSchema) +// defer rb.Release() +// for c := 0; c < int(rec.NumCols()); c++ { +// tp := arrowSchema.Field(c).Type +// if arrowSchema.Field(c).Type.ID() == arrow.EXTENSION { +// tp = arrowSchema.Field(c).Type.(arrow.ExtensionType).StorageType() +// } +// arr, err := compute.CastToType(ctx, rec.Column(c), tp) +// if err != nil { +// return nil, fmt.Errorf("failed to cast col %v to %v: %w", rec.ColumnName(c), arrowSchema.Field(c).Type, err) +// } +// // Ideally these cases should be handled by Arrow itself. +// if arrowSchema.Field(c).Type.ID() == arrow.EXTENSION { +// switch { +// case arrowSchema.Field(c).Type.(arrow.ExtensionType).ExtensionEquals(types.NewUUIDType()): +// bldr := array.NewExtensionBuilder(mem, types.NewUUIDType()) +// uuidBuilder := types.NewUUIDBuilder(bldr) +// j, err := arr.MarshalJSON() +// if err != nil { +// return nil, fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) +// } +// uuidBuilder.StorageBuilder().UnmarshalJSON(j) +// arr.Release() +// arr = uuidBuilder.NewArray() +// uuidBuilder.Release() +// case arrowSchema.Field(c).Type.(arrow.ExtensionType).ExtensionEquals(types.NewJSONType()): +// bldr := array.NewExtensionBuilder(mem, types.NewJSONType()) +// jsonBuilder := types.NewJSONBuilder(bldr) +// j, err := arr.MarshalJSON() +// if err != nil { +// return nil, fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) +// } +// jsonBuilder.StorageBuilder().UnmarshalJSON(j) +// arr.Release() +// arr = jsonBuilder.NewArray() +// case arrowSchema.Field(c).Type.(arrow.ExtensionType).ExtensionEquals(types.NewInetType()): +// bldr := array.NewExtensionBuilder(mem, types.NewInetType()) +// inetBuilder := types.NewInetBuilder(bldr) +// j, err := arr.MarshalJSON() +// if err != nil { +// return nil, fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) +// } +// inetBuilder.StorageBuilder().UnmarshalJSON(j) +// arr.Release() +// arr = inetBuilder.NewArray() +// } +// } else if arrow.TypeEqual(arrowSchema.Field(c).Type, arrow.ListOf(types.NewUUIDType())) { +// // TODO: handle +// continue +// } else { +// b, err := arr.MarshalJSON() +// if err != nil { +// return nil, fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) +// } +// err = rb.Field(c).UnmarshalJSON(b) +// if err != nil { +// return nil, fmt.Errorf("failed to unmarshal col %v: %w", rec.ColumnName(c), err) +// } +// } +// } +// return rb.NewRecord(), nil +//} diff --git a/parquet/write_read_test.go b/parquet/write_read_test.go index aec3cb21..59f3c572 100644 --- a/parquet/write_read_test.go +++ b/parquet/write_read_test.go @@ -3,14 +3,19 @@ package parquet import ( "bufio" "bytes" + "context" + "fmt" "io" "testing" "time" "github.com/apache/arrow/go/v12/arrow" + "github.com/apache/arrow/go/v12/arrow/array" + "github.com/apache/arrow/go/v12/arrow/compute" "github.com/apache/arrow/go/v12/arrow/memory" "github.com/cloudquery/plugin-sdk/plugins/destination" "github.com/cloudquery/plugin-sdk/testdata" + "github.com/cloudquery/plugin-sdk/types" ) func TestWriteRead(t *testing.T) { @@ -63,7 +68,11 @@ func TestWriteRead(t *testing.T) { }() totalCount := 0 for got := range ch { - if diff := destination.RecordDiff(records[totalCount], got); diff != "" { + baseRecord, err := castExtensionColsToStorageType(mem, records[totalCount]) + if err != nil { + t.Fatalf("failed to cast extensions to storage type for comparison: %v", err) + } + if diff := destination.RecordDiff(baseRecord, got); diff != "" { t.Fatalf("got diff: %s", diff) } totalCount++ @@ -75,3 +84,79 @@ func TestWriteRead(t *testing.T) { t.Fatalf("expected 1 row, got %d", totalCount) } } + +func castExtensionColsToStorageType(mem memory.Allocator, rec arrow.Record) (arrow.Record, error) { + oldFields := rec.Schema().Fields() + fields := make([]arrow.Field, len(oldFields)) + copy(fields, oldFields) + for i, f := range fields { + switch { + case f.Type.ID() == arrow.EXTENSION: + fields[i].Type = f.Type.(arrow.ExtensionType).StorageType() + case arrow.TypeEqual(f.Type, arrow.ListOf(types.NewUUIDType())), + arrow.TypeEqual(f.Type, arrow.ListOf(types.NewInetType())), + arrow.TypeEqual(f.Type, arrow.ListOf(types.NewJSONType())), + arrow.TypeEqual(f.Type, arrow.ListOf(types.NewMacType())): + fields[i].Type = arrow.ListOf(f.Type.(*arrow.ListType).Elem().(arrow.ExtensionType).StorageType()) + } + } + + md := rec.Schema().Metadata() + newSchema := arrow.NewSchema(fields, &md) + ctx := context.Background() + rb := array.NewRecordBuilder(mem, newSchema) + + defer rb.Release() + for c := 0; c < int(rec.NumCols()); c++ { + col := rec.Column(c) + if col.DataType().ID() == arrow.EXTENSION { + storageType := col.DataType().(arrow.ExtensionType).StorageType() + arr, err := compute.CastToType(ctx, rec.Column(c), storageType) + if err != nil { + return nil, fmt.Errorf("failed to cast col %v to %v: %w", rec.ColumnName(c), storageType, err) + } + b, err := arr.MarshalJSON() + if err != nil { + return nil, fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) + } + err = rb.Field(c).UnmarshalJSON(b) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal col %v: %w", rec.ColumnName(c), err) + } + } else if arrow.TypeEqual(col.DataType(), arrow.ListOf(types.NewUUIDType())) { + castListOf(ctx, rec, c, rb, types.NewUUIDType().StorageType()) + } else if arrow.TypeEqual(col.DataType(), arrow.ListOf(types.NewJSONType())) { + castListOf(ctx, rec, c, rb, types.NewJSONType().StorageType()) + } else if arrow.TypeEqual(col.DataType(), arrow.ListOf(types.NewInetType())) { + castListOf(ctx, rec, c, rb, types.NewInetType().StorageType()) + } else if arrow.TypeEqual(col.DataType(), arrow.ListOf(types.NewMacType())) { + castListOf(ctx, rec, c, rb, types.NewMacType().StorageType()) + } else { + b, err := rec.Column(c).MarshalJSON() + if err != nil { + return nil, fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) + } + err = rb.Field(c).UnmarshalJSON(b) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal col %v: %w", rec.ColumnName(c), err) + } + } + } + return rb.NewRecord(), nil +} + +func castListOf(ctx context.Context, rec arrow.Record, c int, rb *array.RecordBuilder, storageType arrow.DataType) error { + arr, err := compute.CastToType(ctx, rec.Column(c), arrow.ListOf(storageType)) + if err != nil { + return fmt.Errorf("failed to cast col %v to %v: %w", rec.ColumnName(c), storageType, err) + } + b, err := arr.MarshalJSON() + if err != nil { + return fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) + } + err = rb.Field(c).UnmarshalJSON(b) + if err != nil { + return fmt.Errorf("failed to unmarshal col %v: %w", rec.ColumnName(c), err) + } + return nil +} From 28669cf19f3553f449b0fcab17f7f82446e0ad99 Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Tue, 11 Apr 2023 15:47:11 +0100 Subject: [PATCH 06/25] Update all the things --- client.go | 2 +- csv/read.go | 10 +- csv/transformer.go | 4 +- csv/write.go | 4 +- csv/write_read_test.go | 12 +- go.mod | 21 +- go.sum | 1102 +----------------------------------- json/read.go | 13 +- json/transformer.go | 4 +- json/write.go | 3 +- json/write_read_test.go | 18 +- parquet/read.go | 9 +- parquet/transformer.go | 2 +- parquet/write.go | 5 +- parquet/write_read_test.go | 14 +- read.go | 22 +- transformer.go | 226 -------- write.go | 9 +- 18 files changed, 62 insertions(+), 1418 deletions(-) delete mode 100644 transformer.go diff --git a/client.go b/client.go index 2448e7bd..768a4806 100644 --- a/client.go +++ b/client.go @@ -4,7 +4,7 @@ import ( csvFile "github.com/cloudquery/filetypes/csv" jsonFile "github.com/cloudquery/filetypes/json" "github.com/cloudquery/filetypes/parquet" - "github.com/cloudquery/plugin-sdk/schema" + "github.com/cloudquery/plugin-sdk/v2/schema" ) type Client struct { diff --git a/csv/read.go b/csv/read.go index ad607960..fdf4c2ed 100644 --- a/csv/read.go +++ b/csv/read.go @@ -1,22 +1,14 @@ package csv import ( - "fmt" "io" "github.com/apache/arrow/go/v12/arrow" "github.com/apache/arrow/go/v12/arrow/csv" - "github.com/cloudquery/plugin-sdk/schema" ) -func (cl *Client) Read(r io.Reader, table *schema.Table, _ string, res chan<- arrow.Record) error { - arrowSchema := table.ToArrowSchema() - +func (cl *Client) Read(r io.Reader, arrowSchema *arrow.Schema, _ string, res chan<- arrow.Record) error { reader := csv.NewReader(r, arrowSchema, csv.WithComma(cl.Delimiter), csv.WithHeader(cl.IncludeHeaders), csv.WithNullReader(true, "")) - sourceNameIndex := table.Columns.Index(schema.CqSourceNameColumn.Name) - if sourceNameIndex == -1 { - return fmt.Errorf("could not find column %s in table %s", schema.CqSourceNameColumn.Name, table.Name) - } for reader.Next() { if reader.Err() != nil { return reader.Err() diff --git a/csv/transformer.go b/csv/transformer.go index 926604e0..90647ab6 100644 --- a/csv/transformer.go +++ b/csv/transformer.go @@ -1,8 +1,8 @@ package csv import ( - "github.com/cloudquery/plugin-sdk/plugins/destination" - "github.com/cloudquery/plugin-sdk/schema" + "github.com/cloudquery/plugin-sdk/v2/plugins/destination" + "github.com/cloudquery/plugin-sdk/v2/schema" ) type ReverseTransformer struct { diff --git a/csv/write.go b/csv/write.go index 4b30b0a1..dd0d25db 100644 --- a/csv/write.go +++ b/csv/write.go @@ -7,11 +7,9 @@ import ( "github.com/apache/arrow/go/v12/arrow" "github.com/apache/arrow/go/v12/arrow/csv" - "github.com/cloudquery/plugin-sdk/schema" ) -func (cl *Client) WriteTableBatch(w io.Writer, table *schema.Table, records []arrow.Record) error { - arrowSchema := table.ToArrowSchema() +func (cl *Client) WriteTableBatch(w io.Writer, arrowSchema *arrow.Schema, records []arrow.Record) error { writer := csv.NewWriter(w, arrowSchema, csv.WithComma(cl.Delimiter), csv.WithHeader(cl.IncludeHeaders), csv.WithNullWriter("")) for _, record := range records { if err := writer.Write(record); err != nil { diff --git a/csv/write_read_test.go b/csv/write_read_test.go index ea1fcf78..b7511f41 100644 --- a/csv/write_read_test.go +++ b/csv/write_read_test.go @@ -7,8 +7,8 @@ import ( "github.com/apache/arrow/go/v12/arrow" "github.com/apache/arrow/go/v12/arrow/memory" - "github.com/cloudquery/plugin-sdk/plugins/destination" - "github.com/cloudquery/plugin-sdk/testdata" + "github.com/cloudquery/plugin-sdk/v2/plugins/destination" + "github.com/cloudquery/plugin-sdk/v2/testdata" "github.com/google/uuid" ) @@ -28,7 +28,7 @@ func TestWriteRead(t *testing.T) { t.Run(tc.name, func(t *testing.T) { var b bytes.Buffer table := testdata.TestTable("test") - sch := table.ToArrowSchema() + arrowSchema := table.ToArrowSchema() sourceName := "test-source" syncTime := time.Now().UTC().Round(1 * time.Second) mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) @@ -39,7 +39,7 @@ func TestWriteRead(t *testing.T) { MaxRows: 1, StableUUID: uuid.Nil, } - records := testdata.GenTestData(mem, sch, opts) + records := testdata.GenTestData(mem, arrowSchema, opts) defer func() { for _, r := range records { r.Release() @@ -49,14 +49,14 @@ func TestWriteRead(t *testing.T) { if err != nil { t.Fatal(err) } - if err := cl.WriteTableBatch(&b, table, records); err != nil { + if err := cl.WriteTableBatch(&b, arrowSchema, records); err != nil { t.Fatal(err) } ch := make(chan arrow.Record) var readErr error go func() { - readErr = cl.Read(&b, table, "test-source", ch) + readErr = cl.Read(&b, arrowSchema, "test-source", ch) close(ch) }() totalCount := 0 diff --git a/go.mod b/go.mod index 53e0d144..0f25d182 100644 --- a/go.mod +++ b/go.mod @@ -3,15 +3,13 @@ module github.com/cloudquery/filetypes go 1.19 require ( - github.com/cloudquery/plugin-sdk v1.44.1 github.com/davecgh/go-spew v1.1.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/stretchr/testify v1.8.2 gopkg.in/yaml.v3 v3.0.1 // indirect + github.com/cloudquery/plugin-sdk/v2 v2.0.0 ) -replace github.com/cloudquery/plugin-sdk v1.44.1 => ../plugin-sdk - require ( github.com/ghodss/yaml v1.0.0 // indirect github.com/google/uuid v1.3.0 @@ -20,25 +18,23 @@ require ( github.com/mattn/go-isatty v0.0.18 // indirect; indirect // indirect github.com/rs/zerolog v1.29.0 // indirect; indirect // indirect github.com/thoas/go-funk v0.9.3 // indirect; indirect // indirect - golang.org/x/sys v0.6.0 // indirect; indirect // indirect // indirect + golang.org/x/sys v0.7.0 // indirect; indirect // indirect // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) -require ( - github.com/apache/arrow/go/v12 v12.0.0-20230401212133-adf33cc43010 -) +require github.com/apache/arrow/go/v12 v12.0.0-20230401212133-adf33cc43010 replace github.com/apache/arrow/go/v12 => ../arrow/go +replace github.com/cloudquery/plugin-sdk/v2 => ../plugin-sdk require golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect require ( github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect github.com/andybalholm/brotli v1.0.5 // indirect - github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516 // indirect github.com/apache/thrift v0.18.1 // indirect + // indirect github.com/goccy/go-json v0.10.2 // indirect - github.com/goccy/go-reflect v1.2.0 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/flatbuffers v2.0.8+incompatible // indirect @@ -48,18 +44,17 @@ require ( github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect golang.org/x/mod v0.9.0 // indirect - golang.org/x/net v0.8.0 // indirect + golang.org/x/net v0.9.0 // indirect golang.org/x/sync v0.1.0 // indirect golang.org/x/tools v0.7.0 // indirect golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect - google.golang.org/genproto v0.0.0-20230227214838-9b19f0bdc514 // indirect + google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633 // indirect google.golang.org/grpc v1.54.0 // indirect google.golang.org/protobuf v1.30.0 // indirect ) require ( - github.com/getsentry/sentry-go v0.20.0 // indirect; indirect // indirect github.com/klauspost/compress v1.16.3 // indirect github.com/pierrec/lz4/v4 v4.1.17 // indirect; indirect // indirect - golang.org/x/text v0.8.0 // indirect; indirect // indirect + golang.org/x/text v0.9.0 // indirect; indirect // indirect ) diff --git a/go.sum b/go.sum index 771dbd4b..4e41b628 100644 --- a/go.sum +++ b/go.sum @@ -1,1196 +1,114 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.82.0/go.mod h1:vlKccHJGuFBFufnAnuB08dfEH9Y3H7dzDzRECFdC2TA= -cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= -cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= -cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= -cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= -cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= -cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= -cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= -cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= -cloud.google.com/go v0.100.1/go.mod h1:fs4QogzfH5n2pBXBP9vRiU+eCny7lD2vmFZy79Iuw1U= -cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= -cloud.google.com/go/compute v1.2.0/go.mod h1:xlogom/6gr8RJGBe7nT2eGsQYAFUbbv8dbC29qE3Xmw= -cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= -cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= -cloud.google.com/go/iam v0.1.0/go.mod h1:vcUNEa0pEm0qRVpmWepWaFMIAI8/hjB9mO8rNCJtF6c= -cloud.google.com/go/iam v0.1.1/go.mod h1:CKqrcnI/suGpybEHxZ7BMehL0oA4LpdyJdUlTl9jVMw= -cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= -cloud.google.com/go/kms v1.1.0/go.mod h1:WdbppnCDMDpOvoYBMn1+gNmOeEoZYqAv+HeuKARGCXI= -cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxsnnOA= -cloud.google.com/go/monitoring v1.1.0/go.mod h1:L81pzz7HKn14QCMaCs6NTQkdBnE87TElyanS95vIcl4= -cloud.google.com/go/monitoring v1.4.0/go.mod h1:y6xnxfwI3hTFWOdkOaD7nfJVlwuC3/mS/5kvtT131p4= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/pubsub v1.19.0/go.mod h1:/O9kmSe9bb9KRnIAWkzmqhPjHo6LtzGOBYd/kr06XSs= -cloud.google.com/go/secretmanager v1.3.0/go.mod h1:+oLTkouyiYiabAQNugCeTS3PAArGiMJuBqvJnJsyH+U= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.21.0/go.mod h1:XmRlxkgPjlBONznT2dDUU/5XlpU2OjMnKuqnZI01LAA= -cloud.google.com/go/trace v1.0.0/go.mod h1:4iErSByzxkyHWzzlAj63/Gmjz0NH1ASqhJguHpGcr6A= -cloud.google.com/go/trace v1.2.0/go.mod h1:Wc8y/uYyOhPy12KEnXG9XGrvfMz5F5SrYecQlbW1rwM= -contrib.go.opencensus.io/exporter/aws v0.0.0-20200617204711-c478e41e60e9/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA= -contrib.go.opencensus.io/exporter/stackdriver v0.13.10/go.mod h1:I5htMbyta491eUxufwwZPQdcKvvgzMB4O9ni41YnIM8= -contrib.go.opencensus.io/integrations/ocsql v0.1.7/go.mod h1:8DsSdjz3F+APR+0z0WkU1aRorQCFfRxvqjUUPMbF3fE= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= -git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc= -github.com/Azure/azure-amqp-common-go/v3 v3.2.1/go.mod h1:O6X1iYHP7s2x7NjUKsXVhkwWrQhxrd+d8/3rRadj4CI= -github.com/Azure/azure-amqp-common-go/v3 v3.2.2/go.mod h1:O6X1iYHP7s2x7NjUKsXVhkwWrQhxrd+d8/3rRadj4CI= -github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= -github.com/Azure/azure-sdk-for-go v51.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v59.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.0.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.1/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.0.0/go.mod h1:+6sju8gk8FRmSajX3Oz4G5Gm7P+mbqE9FVaXXFYTkCM= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0= -github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1/go.mod h1:eZ4g6GUvXiGulfIbbhh1Xr4XwUYaYaWMqzGD/284wCA= -github.com/Azure/azure-service-bus-go v0.11.5/go.mod h1:MI6ge2CuQWBVq+ly456MY7XqNLJip5LO1iSFodbNLbU= -github.com/Azure/azure-storage-blob-go v0.14.0/go.mod h1:SMqIBi+SuiQH32bvyjngEewEeXoPfKMgWlBDaYf6fck= -github.com/Azure/go-amqp v0.16.0/go.mod h1:9YJ3RhxRT1gquYnzpZO1vcYMMpAdJT+QEg6fwmw9Zlg= -github.com/Azure/go-amqp v0.16.4/go.mod h1:9YJ3RhxRT1gquYnzpZO1vcYMMpAdJT+QEg6fwmw9Zlg= -github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= -github.com/Azure/go-autorest/autorest v0.11.19/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= -github.com/Azure/go-autorest/autorest v0.11.22/go.mod h1:BAWYUWGPEtKPzjVkp0Q6an0MJcJDsoh5Z1BFAEFs4Xs= -github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= -github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= -github.com/Azure/go-autorest/autorest/adal v0.9.14/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= -github.com/Azure/go-autorest/autorest/adal v0.9.17/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= -github.com/Azure/go-autorest/autorest/azure/auth v0.5.9/go.mod h1:hg3/1yw0Bq87O3KvvnJoAh34/0zbP7SFizX/qN5JvjU= -github.com/Azure/go-autorest/autorest/azure/cli v0.4.2/go.mod h1:7qkJkT+j6b+hIpzMOwPChJhTqS8VbsqqgULzMNRugoM= -github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= -github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= -github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/AzureAD/microsoft-authentication-library-for-go v0.4.0/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4= -github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/GoogleCloudPlatform/cloudsql-proxy v1.29.0/go.mod h1:spvB9eLJH9dutlbPSRmHvSXXHOwGRyeXh1jVdquA2G8= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= -github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY= -github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk= -github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= -github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM= -github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516 h1:byKBBF2CKWBjjA4J1ZL2JXttJULvWSl50LegTyRZ728= -github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516/go.mod h1:QNYViu/X0HXDHw7m3KXzWSVXIbfUvJqBFe6Gj8/pYA0= -github.com/apache/thrift v0.14.2/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= github.com/apache/thrift v0.18.1 h1:lNhK/1nqjbwbiOPDBPFJVKxgDEGSepKuTh6OLiXW8kg= github.com/apache/thrift v0.18.1/go.mod h1:rdQn/dCcDKEWjjylUeueum4vQEjG2v8v2PqriUnbr+I= -github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= -github.com/aws/aws-sdk-go v1.30.19/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= -github.com/aws/aws-sdk-go v1.43.31/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= -github.com/aws/aws-sdk-go-v2 v1.16.2/go.mod h1:ytwTPBG6fXTZLxxeeCCWj2/EMYp/xDUgX+OET6TLNNU= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.1/go.mod h1:n8Bs1ElDD2wJ9kCRTczA83gYbBmjSwZp3umc6zF4EeM= -github.com/aws/aws-sdk-go-v2/config v1.15.3/go.mod h1:9YL3v07Xc/ohTsxFXzan9ZpFpdTOFl4X65BAKYaz8jg= -github.com/aws/aws-sdk-go-v2/credentials v1.11.2/go.mod h1:j8YsY9TXTm31k4eFhspiQicfXPLZ0gYXA50i4gxPE8g= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.3/go.mod h1:uk1vhHHERfSVCUnqSqz8O48LBYDSC+k6brng09jcMOk= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.3/go.mod h1:0dHuD2HZZSiwfJSy1FO5bX1hQ1TxVV1QXXjpn3XUE44= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.9/go.mod h1:AnVH5pvai0pAF4lXRq0bmhbes1u9R8wTE+g+183bZNM= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.3/go.mod h1:ssOhaLpRlh88H3UmEcsBoVKq309quMvm3Ds8e9d4eJM= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.10/go.mod h1:8DcYQcz0+ZJaSxANlHIsbbi6S+zMwjwdDqwW3r9AzaE= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.1/go.mod h1:GeUru+8VzrTXV/83XyMJ80KpH8xO89VPoUileyNQ+tc= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.3/go.mod h1:Seb8KNmD6kVTjwRjVEgOT5hPin6sq+v4C2ycJQDwuH8= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.3/go.mod h1:wlY6SVjuwvh3TVRpTqdy4I1JpBFLX4UGeKZdWntaocw= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.3/go.mod h1:Bm/v2IaN6rZ+Op7zX+bOUMdL4fsrYZiD0dsjLhNKwZc= -github.com/aws/aws-sdk-go-v2/service/kms v1.16.3/go.mod h1:QuiHPBqlOFCi4LqdSskYYAWpQlx3PKmohy+rE2F+o5g= -github.com/aws/aws-sdk-go-v2/service/s3 v1.26.3/go.mod h1:g1qvDuRsJY+XghsV6zg00Z4KJ7DtFFCx8fJD2a491Ak= -github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.15.4/go.mod h1:PJc8s+lxyU8rrre0/4a0pn2wgwiDvOEzoOjcJUBr67o= -github.com/aws/aws-sdk-go-v2/service/sns v1.17.4/go.mod h1:kElt+uCcXxcqFyc+bQqZPFD9DME/eC6oHBXvFzQ9Bcw= -github.com/aws/aws-sdk-go-v2/service/sqs v1.18.3/go.mod h1:skmQo0UPvsjsuYYSYMVmrPc1HWCbHUJyrCEp+ZaLzqM= -github.com/aws/aws-sdk-go-v2/service/ssm v1.24.1/go.mod h1:NR/xoKjdbRJ+qx0pMR4mI+N/H1I1ynHwXnO6FowXJc0= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.3/go.mod h1:7UQ/e69kU7LDPtY40OyoHYgRmgfGM4mgsLYtcObdveU= -github.com/aws/aws-sdk-go-v2/service/sts v1.16.3/go.mod h1:bfBj0iVmsUyUg4weDB4NxktD9rDGeKSVWnjTnwbx9b8= -github.com/aws/smithy-go v1.11.2/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= -github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudquery/arrow/go/v12 v12.0.0-20230403113355-afc433992be2 h1:ifVrKVJXhP8hQvjiUjAXicRYv+zWzRg7B640gDvEWq4= -github.com/cloudquery/arrow/go/v12 v12.0.0-20230403113355-afc433992be2/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= -github.com/colinmarc/hdfs/v2 v2.1.1/go.mod h1:M3x+k8UKKmxtFu++uAZ0OtDU8jR3jnaZIAc6yK4Ue0c= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cloudquery/plugin-sdk/v2 v2.0.0 h1:+6V0nwmc331QFq/uUyYGoyD8IV2CIcz1ocfswLwxnAA= +github.com/cloudquery/plugin-sdk/v2 v2.0.0/go.mod h1:k5qZYqnlCjwW5Xdxl7P9xHTbYG1eOyvg/U/bkPUzWYA= github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/denisenkom/go-mssqldb v0.12.0/go.mod h1:iiK0YP1ZeepvmBQk/QpLEhhTNJgfzrpArPY/aFvc9yU= -github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY= -github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= -github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= -github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= -github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= -github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= -github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= -github.com/getsentry/sentry-go v0.20.0 h1:bwXW98iMRIWxn+4FgPW7vMrjmbym6HblXALmhjHmQaQ= -github.com/getsentry/sentry-go v0.20.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= -github.com/gin-gonic/gin v1.7.3/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY= -github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= -github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= -github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks= -github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= -github.com/go-fonts/liberation v0.2.0/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= -github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmnUIzUY= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U= -github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpxhq9o2S/CELCSUxEWWAuoCUcVCQWv7G2OCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= -github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= -github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= -github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= -github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= -github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= -github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= -github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= -github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/goccy/go-reflect v1.2.0 h1:O0T8rZCuNmGXewnATuKYnkL0xm6o8UNOJZd/gOkb9ms= -github.com/goccy/go-reflect v1.2.0/go.mod h1:n0oYZn8VcV2CkWTxi8B9QjkCoq6GTtCEdfmR66YhFtE= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= -github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= -github.com/golang-sql/sqlexp v0.0.0-20170517235910-f1bb20e5a188/go.mod h1:vXjM/+wXQnTPR4KqTKDgJukSZ6amVRtWMPEjE6sQoK8= -github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= -github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/flatbuffers v2.0.8+incompatible h1:ivUb1cGomAB101ZM1T0nOiWz9pSrTMoa9+EiY7igmkM= github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-replayers/grpcreplay v1.1.0/go.mod h1:qzAvJ8/wi57zq7gWqaE6AwLM6miiXUQwP1S+I9icmhk= -github.com/google/go-replayers/httpreplay v1.1.1/go.mod h1:gN9GeLIs7l6NUoVaSSnv2RiqK1NiwAmD0MrKeC9IIks= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= -github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210506205249-923b5ab0fc1a/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/wire v0.5.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= -github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= -github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= -github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hanwen/go-fuse v1.0.0/go.mod h1:unqXarDXqzAk0rt98O2tVndEPIpUgLD9+rwFisZH3Ok= -github.com/hanwen/go-fuse/v2 v2.1.0/go.mod h1:oRyA5eK+pvJyv5otpO/DgccS8y/RvYMaO00GgRLGryc= -github.com/hashicorp/go-uuid v0.0.0-20180228145832-27454136f036/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= -github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= -github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= -github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= -github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= -github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= -github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= -github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= -github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= -github.com/jackc/pgconn v1.11.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= -github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= -github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= -github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= -github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= -github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= -github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= -github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= -github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= -github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= -github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.2.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= -github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= -github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= -github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= -github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= -github.com/jackc/pgtype v1.10.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= -github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= -github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= -github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= -github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= -github.com/jackc/pgx/v4 v4.15.0/go.mod h1:D/zyOyXiaM1TmVWnOM18p0xdDtdakRBa0RsVGI3U3bw= -github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jcmturner/gofork v0.0.0-20180107083740-2aebee971930/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= -github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= -github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= -github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= -github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY= github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.1.0/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= -github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= -github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= -github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= -github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98= github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= -github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.34/go.mod h1:nCrRzjoSUQh8hgKKtu3Y708OLvRLtuASMg2/nvmbarw= -github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= -github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= -github.com/ncw/swift v1.0.52/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= -github.com/pborman/getopt v0.0.0-20180729010549-6fdd0a2c7117/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= -github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= -github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= -github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= -github.com/pierrec/lz4/v4 v4.1.8/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= -github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= -github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= -github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/rs/zerolog v1.29.0 h1:Zes4hju04hjbvkVkOhdl2HpZa+0PmVwigmo8XoORE5w= github.com/rs/zerolog v1.29.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= -github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= -github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= -github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/thoas/go-funk v0.9.3 h1:7+nAEx3kn5ZJcnDm2Bh23N2yOtweO14bi//dvRtgLpw= github.com/thoas/go-funk v0.9.3/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= -github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= -github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0/go.mod h1:HYhIKsdns7xz80OgkbgJYrtQY7FjHWHKH6cvN7+czGE= -github.com/xitongsys/parquet-go-source v0.0.0-20230312005205-fbbcdea5f512 h1:ghvb6MV6j1i4i/CVeQyaNTM6Egiw65O9gYBGQseeLs0= -github.com/xitongsys/parquet-go-source v0.0.0-20230312005205-fbbcdea5f512/go.mod h1:d87JD+hvj9dzdbZNd9zJb9ThSLQfoUNx50KQ6nuf9Wg= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= -github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= -github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= -go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= -go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= -gocloud.dev v0.26.0/go.mod h1:mkUgejbnbLotorqDyvedJO20XcZNTynmSeVSQS9btVg= -golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211115234514-b4de73f9ece8/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220511200225-c6db032c6c88/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug= golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= -golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20200430140353-33d19683fad8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20200618115811-c13761719519/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20201208152932-35266b937fa6/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20210216034530-4410531fe030/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= -golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= -golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= -golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220401154927-543a649e0bdd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210304124612-50617c2ba197/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210503080704-8803ae5d1324/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220204135822-1c1b9b1eba6a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220330033206-e17cdc41300f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220224211638-0e9765cccd65/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= -golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0= golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= -gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= -gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0= gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E= -gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA= -gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= -gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= -gonum.org/v1/plot v0.9.0/go.mod h1:3Pcqqmp6RHvJI72kgb8fThyUnav364FOsdDo2aGW5lY= -gonum.org/v1/plot v0.10.1/go.mod h1:VZW5OlhkL1mysU9vaqNHnsy86inf6Ot+jB3r+BczCEo= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.46.0/go.mod h1:ceL4oozhkAiTID8XMmJBsIxID/9wMXJVVFXPg4ylg3I= -google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= -google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= -google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= -google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= -google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= -google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= -google.golang.org/api v0.58.0/go.mod h1:cAbP2FsxoGVNwtgNAmmn3y5G1TWAiVYRmg4yku3lv+E= -google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= -google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= -google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= -google.golang.org/api v0.64.0/go.mod h1:931CdxA8Rm4t6zqTFGSsgwbAEZ2+GMYurbndwSimebM= -google.golang.org/api v0.66.0/go.mod h1:I1dmXYpX7HGwz/ejRxwQp2qj5bFAz93HiCU1C1oYd9M= -google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= -google.golang.org/api v0.68.0/go.mod h1:sOM8pTpwgflXRhz+oC8H2Dr+UcbMqkPPWNJo88Q7TH8= -google.golang.org/api v0.69.0/go.mod h1:boanBiw+h5c3s+tBPgEzLDRHfFLWV0qXxRHz3ws7C80= -google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= -google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= -google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210429181445-86c259c2b4ab/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210517163617-5e0236093d7a/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= -google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211018162055-cf77aa76bad2/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220111164026-67b88f271998/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220114231437-d2e6a121cae0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220201184016-50beb8ab5c44/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220204002441-d6cc3cc0770e/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220211171837-173942840c17/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220216160803-4663080d8bc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= -google.golang.org/genproto v0.0.0-20220401170504-314d38edb7de/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20230227214838-9b19f0bdc514 h1:rtNKfB++wz5mtDY2t5C8TXlU5y52ojSu7tZo0z7u8eQ= -google.golang.org/genproto v0.0.0-20230227214838-9b19f0bdc514/go.mod h1:TvhZT5f700eVlTNwND1xoEZQeWTB2RY/65kplwl/bFA= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633 h1:0BOZf6qNozI3pkN3fJLwNubheHJYHhMh91GRFOWWK08= +google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= -gopkg.in/ini.v1 v1.66.6/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= -gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= -gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= -gopkg.in/jcmturner/gokrb5.v7 v7.3.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= -gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= -lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= -lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= -modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= -modernc.org/cc/v3 v3.36.2/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= -modernc.org/cc/v3 v3.37.0/go.mod h1:vtL+3mdHx/wcj3iEGz84rQa8vEqR6XM84v5Lcvfph20= -modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0= -modernc.org/ccgo/v3 v3.0.0-20220428102840-41399a37e894/go.mod h1:eI31LL8EwEBKPpNpA4bU1/i+sKOwOrQy8D87zWUcRZc= -modernc.org/ccgo/v3 v3.0.0-20220430103911-bc99d88307be/go.mod h1:bwdAnOoaIt8Ax9YdWGjxWsdkPcZyRPHqrOvJxaKAKGw= -modernc.org/ccgo/v3 v3.0.0-20220904174949-82d86e1b6d56/go.mod h1:YSXjPL62P2AMSxBphRHPn7IkzhVHqkvOnRKAKh+W6ZI= -modernc.org/ccgo/v3 v3.16.4/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= -modernc.org/ccgo/v3 v3.16.6/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= -modernc.org/ccgo/v3 v3.16.8/go.mod h1:zNjwkizS+fIFDrDjIAgBSCLkWbJuHF+ar3QRn+Z9aws= -modernc.org/ccgo/v3 v3.16.9/go.mod h1:zNMzC9A9xeNUepy6KuZBbugn3c0Mc9TeiJO4lgvkJDo= -modernc.org/ccgo/v3 v3.16.13-0.20221017192402-261537637ce8/go.mod h1:fUB3Vn0nVPReA+7IG7yZDfjv1TMWjhQP8gCxrFAtL5g= -modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY= -modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= -modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= -modernc.org/libc v0.0.0-20220428101251-2d5f3daf273b/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= -modernc.org/libc v1.16.0/go.mod h1:N4LD6DBE9cf+Dzf9buBlzVJndKr/iJHG97vGLHYnb5A= -modernc.org/libc v1.16.1/go.mod h1:JjJE0eu4yeK7tab2n4S1w8tlWd9MxXLRzheaRnAKymU= -modernc.org/libc v1.16.17/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU= -modernc.org/libc v1.16.19/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= -modernc.org/libc v1.17.0/go.mod h1:XsgLldpP4aWlPlsjqKRdHPqCxCjISdHfM/yeWC5GyW0= -modernc.org/libc v1.17.4/go.mod h1:WNg2ZH56rDEwdropAJeZPQkXmDwh+JCA1s/htl6r2fA= -modernc.org/libc v1.18.0/go.mod h1:vj6zehR5bfc98ipowQOM2nIDUZnVew/wNC/2tOGS+q0= -modernc.org/libc v1.20.3/go.mod h1:ZRfIaEkgrYgZDl6pa4W39HgN5G/yDW+NRmNKZBDFrk0= -modernc.org/libc v1.21.4/go.mod h1:przBsL5RDOZajTVslkugzLBj1evTue36jEomFQOoYuI= -modernc.org/libc v1.22.2/go.mod h1:uvQavJ1pZ0hIoC/jfqNoMLURIMhKzINIWypNM17puug= -modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= -modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= -modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= -modernc.org/memory v1.1.1/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= -modernc.org/memory v1.2.0/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= -modernc.org/memory v1.3.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= -modernc.org/memory v1.4.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= -modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= -modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/sqlite v1.18.2/go.mod h1:kvrTLEWgxUcHa2GfHBQtanR1H9ht3hTJNtKpzH9k1u0= -modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= -modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= -modernc.org/tcl v1.13.2/go.mod h1:7CLiGIPo1M8Rv1Mitpv5akc2+8fxUd2y2UzC/MfMzy0= -modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -modernc.org/z v1.5.1/go.mod h1:eWFB510QWW5Th9YGZT81s+LwvaAs3Q2yr4sP0rmLkv8= -nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/json/read.go b/json/read.go index 18652757..6a02a79d 100644 --- a/json/read.go +++ b/json/read.go @@ -2,25 +2,19 @@ package json import ( "bufio" - "fmt" "io" "github.com/apache/arrow/go/v12/arrow" "github.com/apache/arrow/go/v12/arrow/array" "github.com/apache/arrow/go/v12/arrow/memory" - "github.com/cloudquery/plugin-sdk/schema" ) const maxJSONSize = 1024 * 1024 * 20 -func (*Client) Read(r io.Reader, table *schema.Table, sourceName string, res chan<- arrow.Record) error { - sourceNameIndex := table.Columns.Index(schema.CqSourceNameColumn.Name) - if sourceNameIndex == -1 { - return fmt.Errorf("could not find column %s in table %s", schema.CqSourceNameColumn.Name, table.Name) - } +func (*Client) Read(r io.Reader, arrowSchema *arrow.Schema, sourceName string, res chan<- arrow.Record) error { scanner := bufio.NewScanner(r) scanner.Buffer(make([]byte, maxJSONSize), maxJSONSize) - arrowSchema := table.ToArrowSchema() + //arrowSchema := table.ToArrowSchema() rb := array.NewRecordBuilder(memory.DefaultAllocator, arrowSchema) defer rb.Release() for scanner.Scan() { @@ -30,9 +24,6 @@ func (*Client) Read(r io.Reader, table *schema.Table, sourceName string, res cha return err } r := rb.NewRecord() - if r.Column(sourceNameIndex).(*array.String).Value(0) != sourceName { - continue - } res <- r } diff --git a/json/transformer.go b/json/transformer.go index cd649822..78415616 100644 --- a/json/transformer.go +++ b/json/transformer.go @@ -1,8 +1,8 @@ package json import ( - "github.com/cloudquery/plugin-sdk/plugins/destination" - "github.com/cloudquery/plugin-sdk/schema" + "github.com/cloudquery/plugin-sdk/v2/plugins/destination" + "github.com/cloudquery/plugin-sdk/v2/schema" ) type ReverseTransformer struct { diff --git a/json/write.go b/json/write.go index c1a294f0..310fed8c 100644 --- a/json/write.go +++ b/json/write.go @@ -6,10 +6,9 @@ import ( "github.com/apache/arrow/go/v12/arrow" "github.com/apache/arrow/go/v12/arrow/array" - "github.com/cloudquery/plugin-sdk/schema" ) -func (c *Client) WriteTableBatch(w io.Writer, _ *schema.Table, records []arrow.Record) error { +func (c *Client) WriteTableBatch(w io.Writer, _ *arrow.Schema, records []arrow.Record) error { for _, r := range records { err := c.writeTableBatch(w, r) if err != nil { diff --git a/json/write_read_test.go b/json/write_read_test.go index 538d0c59..e2ba313d 100644 --- a/json/write_read_test.go +++ b/json/write_read_test.go @@ -7,14 +7,14 @@ import ( "github.com/apache/arrow/go/v12/arrow" "github.com/apache/arrow/go/v12/arrow/memory" - "github.com/cloudquery/plugin-sdk/plugins/destination" - "github.com/cloudquery/plugin-sdk/testdata" + "github.com/cloudquery/plugin-sdk/v2/plugins/destination" + "github.com/cloudquery/plugin-sdk/v2/testdata" ) func TestWrite(t *testing.T) { var b bytes.Buffer table := testdata.TestTable("test") - sch := table.ToArrowSchema() + arrowSchema := table.ToArrowSchema() sourceName := "test-source" syncTime := time.Now().UTC().Round(1 * time.Second) mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) @@ -24,7 +24,7 @@ func TestWrite(t *testing.T) { SyncTime: syncTime, MaxRows: 1, } - records := testdata.GenTestData(mem, sch, opts) + records := testdata.GenTestData(mem, arrowSchema, opts) defer func() { for _, r := range records { r.Release() @@ -34,7 +34,7 @@ func TestWrite(t *testing.T) { if err != nil { t.Fatal(err) } - if err := cl.WriteTableBatch(&b, table, records); err != nil { + if err := cl.WriteTableBatch(&b, arrowSchema, records); err != nil { t.Fatal(err) } t.Log(b.String()) @@ -43,7 +43,7 @@ func TestWrite(t *testing.T) { func TestWriteRead(t *testing.T) { var b bytes.Buffer table := testdata.TestTable("test") - sch := table.ToArrowSchema() + arrowSchema := table.ToArrowSchema() sourceName := "test-source" syncTime := time.Now().UTC().Round(1 * time.Second) mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) @@ -53,7 +53,7 @@ func TestWriteRead(t *testing.T) { SyncTime: syncTime, MaxRows: 1, } - records := testdata.GenTestData(mem, sch, opts) + records := testdata.GenTestData(mem, arrowSchema, opts) defer func() { for _, r := range records { r.Release() @@ -63,14 +63,14 @@ func TestWriteRead(t *testing.T) { if err != nil { t.Fatal(err) } - if err := cl.WriteTableBatch(&b, table, records); err != nil { + if err := cl.WriteTableBatch(&b, arrowSchema, records); err != nil { t.Fatal(err) } ch := make(chan arrow.Record) var readErr error go func() { - readErr = cl.Read(&b, table, "test-source", ch) + readErr = cl.Read(&b, arrowSchema, "test-source", ch) close(ch) }() totalCount := 0 diff --git a/parquet/read.go b/parquet/read.go index 4c7b1e92..4266252f 100644 --- a/parquet/read.go +++ b/parquet/read.go @@ -2,14 +2,12 @@ package parquet import ( "context" - "fmt" "io" "github.com/apache/arrow/go/v12/arrow" "github.com/apache/arrow/go/v12/arrow/memory" "github.com/apache/arrow/go/v12/parquet/file" "github.com/apache/arrow/go/v12/parquet/pqarrow" - "github.com/cloudquery/plugin-sdk/schema" ) type ReaderAtSeeker interface { @@ -18,12 +16,7 @@ type ReaderAtSeeker interface { io.Seeker } -func (*Client) Read(f ReaderAtSeeker, table *schema.Table, sourceName string, res chan<- arrow.Record) error { - // arrowSchema := table.ToArrowSchema() - sourceNameIndex := int64(table.Columns.Index(schema.CqSourceNameColumn.Name)) - if sourceNameIndex == -1 { - return fmt.Errorf("could not find column %s in table %s", schema.CqSourceNameColumn.Name, table.Name) - } +func (*Client) Read(f ReaderAtSeeker, arrowSchema *arrow.Schema, sourceName string, res chan<- arrow.Record) error { mem := memory.DefaultAllocator ctx := context.Background() rdr, err := file.NewParquetReader(f) diff --git a/parquet/transformer.go b/parquet/transformer.go index f8357179..5cdf248c 100644 --- a/parquet/transformer.go +++ b/parquet/transformer.go @@ -4,7 +4,7 @@ import ( "fmt" "time" - "github.com/cloudquery/plugin-sdk/schema" + "github.com/cloudquery/plugin-sdk/v2/schema" ) type ReverseTransformer struct{} diff --git a/parquet/write.go b/parquet/write.go index 803cb52f..b58656b5 100644 --- a/parquet/write.go +++ b/parquet/write.go @@ -6,13 +6,12 @@ import ( "github.com/apache/arrow/go/v12/arrow" "github.com/apache/arrow/go/v12/parquet" "github.com/apache/arrow/go/v12/parquet/pqarrow" - "github.com/cloudquery/plugin-sdk/schema" ) -func (*Client) WriteTableBatch(w io.Writer, table *schema.Table, records []arrow.Record) error { +func (*Client) WriteTableBatch(w io.Writer, arrowSchema *arrow.Schema, records []arrow.Record) error { props := parquet.NewWriterProperties() arrprops := pqarrow.DefaultWriterProps() - fw, err := pqarrow.NewFileWriter(table.ToArrowSchema(), w, props, arrprops) + fw, err := pqarrow.NewFileWriter(arrowSchema, w, props, arrprops) if err != nil { return err } diff --git a/parquet/write_read_test.go b/parquet/write_read_test.go index 59f3c572..786cd802 100644 --- a/parquet/write_read_test.go +++ b/parquet/write_read_test.go @@ -13,15 +13,15 @@ import ( "github.com/apache/arrow/go/v12/arrow/array" "github.com/apache/arrow/go/v12/arrow/compute" "github.com/apache/arrow/go/v12/arrow/memory" - "github.com/cloudquery/plugin-sdk/plugins/destination" - "github.com/cloudquery/plugin-sdk/testdata" - "github.com/cloudquery/plugin-sdk/types" + "github.com/cloudquery/plugin-sdk/v2/plugins/destination" + "github.com/cloudquery/plugin-sdk/v2/testdata" + "github.com/cloudquery/plugin-sdk/v2/types" ) func TestWriteRead(t *testing.T) { var b bytes.Buffer table := testdata.TestTable("test") - sch := table.ToArrowSchema() + arrowSchema := table.ToArrowSchema() sourceName := "test-source" syncTime := time.Now().UTC().Round(1 * time.Second) // TODO: use checked allocator here; can't right now because there @@ -34,7 +34,7 @@ func TestWriteRead(t *testing.T) { SyncTime: syncTime, MaxRows: 1, } - records := testdata.GenTestData(mem, sch, opts) + records := testdata.GenTestData(mem, arrowSchema, opts) defer func() { for _, r := range records { r.Release() @@ -47,7 +47,7 @@ func TestWriteRead(t *testing.T) { if err != nil { t.Fatal(err) } - if err := cl.WriteTableBatch(writer, table, records); err != nil { + if err := cl.WriteTableBatch(writer, arrowSchema, records); err != nil { t.Fatal(err) } err = writer.Flush() @@ -63,7 +63,7 @@ func TestWriteRead(t *testing.T) { ch := make(chan arrow.Record) var readErr error go func() { - readErr = cl.Read(byteReader, table, "test-source", ch) + readErr = cl.Read(byteReader, arrowSchema, "test-source", ch) close(ch) }() totalCount := 0 diff --git a/read.go b/read.go index dbce1c80..6c2796cf 100644 --- a/read.go +++ b/read.go @@ -4,21 +4,20 @@ import ( "os" "github.com/apache/arrow/go/v12/arrow" - "github.com/cloudquery/plugin-sdk/schema" ) -func (cl *Client) Read(f *os.File, table *schema.Table, sourceName string, res chan<- arrow.Record) error { +func (cl *Client) Read(f *os.File, sc *arrow.Schema, sourceName string, res chan<- arrow.Record) error { switch cl.spec.Format { case FormatTypeCSV: - if err := cl.csv.Read(f, table, sourceName, res); err != nil { + if err := cl.csv.Read(f, sc, sourceName, res); err != nil { return err } case FormatTypeJSON: - if err := cl.json.Read(f, table, sourceName, res); err != nil { + if err := cl.json.Read(f, sc, sourceName, res); err != nil { return err } case FormatTypeParquet: - if err := cl.parquet.Read(f, table, sourceName, res); err != nil { + if err := cl.parquet.Read(f, sc, sourceName, res); err != nil { return err } default: @@ -26,16 +25,3 @@ func (cl *Client) Read(f *os.File, table *schema.Table, sourceName string, res c } return nil } - -func (cl *Client) ReverseTransformValues(table *schema.Table, values []any) (schema.CQTypes, error) { - switch cl.spec.Format { - case FormatTypeCSV: - return cl.csvReverseTransformer.ReverseTransformValues(table, values) - case FormatTypeJSON: - return cl.jsonReverseTransformer.ReverseTransformValues(table, values) - case FormatTypeParquet: - return cl.parquetReverseTransformer.ReverseTransformValues(table, values) - default: - panic("unknown format " + cl.spec.Format) - } -} diff --git a/transformer.go b/transformer.go deleted file mode 100644 index d995bd93..00000000 --- a/transformer.go +++ /dev/null @@ -1,226 +0,0 @@ -package filetypes - -import ( - "github.com/cloudquery/plugin-sdk/schema" -) - -func (c *Client) TransformBool(v *schema.Bool) any { - switch c.spec.Format { - case FormatTypeCSV: - return c.csvTransformer.TransformBool(v) - case FormatTypeJSON: - return c.jsonTransformer.TransformBool(v) - case FormatTypeParquet: - return c.parquetTransformer.TransformBool(v) - default: - panic("unknown format " + c.spec.Format) - } -} - -func (c *Client) TransformBytea(v *schema.Bytea) any { - switch c.spec.Format { - case FormatTypeCSV: - return c.csvTransformer.TransformBytea(v) - case FormatTypeJSON: - return c.jsonTransformer.TransformBytea(v) - case FormatTypeParquet: - return c.parquetTransformer.TransformBytea(v) - default: - panic("unknown format " + c.spec.Format) - } -} - -func (c *Client) TransformFloat8(v *schema.Float8) any { - switch c.spec.Format { - case FormatTypeCSV: - return c.csvTransformer.TransformFloat8(v) - case FormatTypeJSON: - return c.jsonTransformer.TransformFloat8(v) - case FormatTypeParquet: - return c.parquetTransformer.TransformFloat8(v) - default: - panic("unknown format " + c.spec.Format) - } -} - -func (c *Client) TransformInt8(v *schema.Int8) any { - switch c.spec.Format { - case FormatTypeCSV: - return c.csvTransformer.TransformInt8(v) - case FormatTypeJSON: - return c.jsonTransformer.TransformInt8(v) - case FormatTypeParquet: - return c.parquetTransformer.TransformInt8(v) - default: - panic("unknown format " + c.spec.Format) - } -} - -func (c *Client) TransformInt8Array(v *schema.Int8Array) any { - switch c.spec.Format { - case FormatTypeCSV: - return c.csvTransformer.TransformInt8Array(v) - case FormatTypeJSON: - return c.jsonTransformer.TransformInt8Array(v) - case FormatTypeParquet: - return c.parquetTransformer.TransformInt8Array(v) - default: - panic("unknown format " + c.spec.Format) - } -} - -func (c *Client) TransformJSON(v *schema.JSON) any { - switch c.spec.Format { - case FormatTypeCSV: - return c.csvTransformer.TransformJSON(v) - case FormatTypeJSON: - return c.jsonTransformer.TransformJSON(v) - case FormatTypeParquet: - return c.parquetTransformer.TransformJSON(v) - default: - panic("unknown format " + c.spec.Format) - } -} - -func (c *Client) TransformText(v *schema.Text) any { - switch c.spec.Format { - case FormatTypeCSV: - return c.csvTransformer.TransformText(v) - case FormatTypeJSON: - return c.jsonTransformer.TransformText(v) - case FormatTypeParquet: - return c.parquetTransformer.TransformText(v) - default: - panic("unknown format " + c.spec.Format) - } -} - -func (c *Client) TransformTextArray(v *schema.TextArray) any { - switch c.spec.Format { - case FormatTypeCSV: - return c.csvTransformer.TransformTextArray(v) - case FormatTypeJSON: - return c.jsonTransformer.TransformTextArray(v) - case FormatTypeParquet: - return c.parquetTransformer.TransformTextArray(v) - default: - panic("unknown format " + c.spec.Format) - } -} - -func (c *Client) TransformTimestamptz(v *schema.Timestamptz) any { - switch c.spec.Format { - case FormatTypeCSV: - return c.csvTransformer.TransformTimestamptz(v) - case FormatTypeJSON: - return c.jsonTransformer.TransformTimestamptz(v) - case FormatTypeParquet: - return c.parquetTransformer.TransformTimestamptz(v) - default: - panic("unknown format " + c.spec.Format) - } -} - -func (c *Client) TransformUUID(v *schema.UUID) any { - switch c.spec.Format { - case FormatTypeCSV: - return c.csvTransformer.TransformUUID(v) - case FormatTypeJSON: - return c.jsonTransformer.TransformUUID(v) - case FormatTypeParquet: - return c.parquetTransformer.TransformUUID(v) - default: - panic("unknown format " + c.spec.Format) - } -} - -func (c *Client) TransformUUIDArray(v *schema.UUIDArray) any { - switch c.spec.Format { - case FormatTypeCSV: - return c.csvTransformer.TransformUUIDArray(v) - case FormatTypeJSON: - return c.jsonTransformer.TransformUUIDArray(v) - case FormatTypeParquet: - return c.parquetTransformer.TransformUUIDArray(v) - default: - panic("unknown format " + c.spec.Format) - } -} - -func (c *Client) TransformCIDR(v *schema.CIDR) any { - switch c.spec.Format { - case FormatTypeCSV: - return c.csvTransformer.TransformCIDR(v) - case FormatTypeJSON: - return c.jsonTransformer.TransformCIDR(v) - case FormatTypeParquet: - return c.parquetTransformer.TransformCIDR(v) - default: - panic("unknown format " + c.spec.Format) - } -} - -func (c *Client) TransformCIDRArray(v *schema.CIDRArray) any { - switch c.spec.Format { - case FormatTypeCSV: - return c.csvTransformer.TransformCIDRArray(v) - case FormatTypeJSON: - return c.jsonTransformer.TransformCIDRArray(v) - case FormatTypeParquet: - return c.parquetTransformer.TransformCIDRArray(v) - default: - panic("unknown format " + c.spec.Format) - } -} - -func (c *Client) TransformInet(v *schema.Inet) any { - switch c.spec.Format { - case FormatTypeCSV: - return c.csvTransformer.TransformInet(v) - case FormatTypeJSON: - return c.jsonTransformer.TransformInet(v) - case FormatTypeParquet: - return c.parquetTransformer.TransformInet(v) - default: - panic("unknown format " + c.spec.Format) - } -} - -func (c *Client) TransformInetArray(v *schema.InetArray) any { - switch c.spec.Format { - case FormatTypeCSV: - return c.csvTransformer.TransformInetArray(v) - case FormatTypeJSON: - return c.jsonTransformer.TransformInetArray(v) - case FormatTypeParquet: - return c.parquetTransformer.TransformInetArray(v) - default: - panic("unknown format " + c.spec.Format) - } -} - -func (c *Client) TransformMacaddr(v *schema.Macaddr) any { - switch c.spec.Format { - case FormatTypeCSV: - return c.csvTransformer.TransformMacaddr(v) - case FormatTypeJSON: - return c.jsonTransformer.TransformMacaddr(v) - case FormatTypeParquet: - return c.parquetTransformer.TransformMacaddr(v) - default: - panic("unknown format " + c.spec.Format) - } -} - -func (c *Client) TransformMacaddrArray(v *schema.MacaddrArray) any { - switch c.spec.Format { - case FormatTypeCSV: - return c.csvTransformer.TransformMacaddrArray(v) - case FormatTypeJSON: - return c.jsonTransformer.TransformMacaddrArray(v) - case FormatTypeParquet: - return c.parquetTransformer.TransformMacaddrArray(v) - default: - panic("unknown format " + c.spec.Format) - } -} diff --git a/write.go b/write.go index 9abb3dfd..d072bd91 100644 --- a/write.go +++ b/write.go @@ -4,21 +4,20 @@ import ( "io" "github.com/apache/arrow/go/v12/arrow" - "github.com/cloudquery/plugin-sdk/schema" ) -func (cl *Client) WriteTableBatchFile(w io.Writer, table *schema.Table, resources []arrow.Record) error { +func (cl *Client) WriteTableBatchFile(w io.Writer, arrowSchema *arrow.Schema, resources []arrow.Record) error { switch cl.spec.Format { case FormatTypeCSV: - if err := cl.csv.WriteTableBatch(w, table, resources); err != nil { + if err := cl.csv.WriteTableBatch(w, arrowSchema, resources); err != nil { return err } case FormatTypeJSON: - if err := cl.json.WriteTableBatch(w, table, resources); err != nil { + if err := cl.json.WriteTableBatch(w, arrowSchema, resources); err != nil { return err } case FormatTypeParquet: - if err := cl.parquet.WriteTableBatch(w, table, resources); err != nil { + if err := cl.parquet.WriteTableBatch(w, arrowSchema, resources); err != nil { return err } default: From 2dc9c7f445c846d7e3fd297ab66dcb39af3a4b10 Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Tue, 11 Apr 2023 15:50:40 +0100 Subject: [PATCH 07/25] Clean-up --- json/read.go | 1 - parquet/read.go | 70 ------------------------------------------------ parquet/write.go | 37 ------------------------- 3 files changed, 108 deletions(-) diff --git a/json/read.go b/json/read.go index 6a02a79d..8a637a47 100644 --- a/json/read.go +++ b/json/read.go @@ -14,7 +14,6 @@ const maxJSONSize = 1024 * 1024 * 20 func (*Client) Read(r io.Reader, arrowSchema *arrow.Schema, sourceName string, res chan<- arrow.Record) error { scanner := bufio.NewScanner(r) scanner.Buffer(make([]byte, maxJSONSize), maxJSONSize) - //arrowSchema := table.ToArrowSchema() rb := array.NewRecordBuilder(memory.DefaultAllocator, arrowSchema) defer rb.Release() for scanner.Scan() { diff --git a/parquet/read.go b/parquet/read.go index 4266252f..34e57f0c 100644 --- a/parquet/read.go +++ b/parquet/read.go @@ -34,10 +34,6 @@ func (*Client) Read(f ReaderAtSeeker, arrowSchema *arrow.Schema, sourceName stri } for rr.Next() { rec := rr.Record() - //castRec, err := castRecord(mem, rec, arrowSchema) - //if err != nil { - // return err - //} rec.Retain() res <- rec _, err = rr.Read() @@ -51,69 +47,3 @@ func (*Client) Read(f ReaderAtSeeker, arrowSchema *arrow.Schema, sourceName stri return nil } - -// -//func castRecord(mem memory.Allocator, rec arrow.Record, arrowSchema *arrow.Schema) (arrow.Record, error) { -// ctx := context.Background() -// rb := array.NewRecordBuilder(mem, arrowSchema) -// defer rb.Release() -// for c := 0; c < int(rec.NumCols()); c++ { -// tp := arrowSchema.Field(c).Type -// if arrowSchema.Field(c).Type.ID() == arrow.EXTENSION { -// tp = arrowSchema.Field(c).Type.(arrow.ExtensionType).StorageType() -// } -// arr, err := compute.CastToType(ctx, rec.Column(c), tp) -// if err != nil { -// return nil, fmt.Errorf("failed to cast col %v to %v: %w", rec.ColumnName(c), arrowSchema.Field(c).Type, err) -// } -// // Ideally these cases should be handled by Arrow itself. -// if arrowSchema.Field(c).Type.ID() == arrow.EXTENSION { -// switch { -// case arrowSchema.Field(c).Type.(arrow.ExtensionType).ExtensionEquals(types.NewUUIDType()): -// bldr := array.NewExtensionBuilder(mem, types.NewUUIDType()) -// uuidBuilder := types.NewUUIDBuilder(bldr) -// j, err := arr.MarshalJSON() -// if err != nil { -// return nil, fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) -// } -// uuidBuilder.StorageBuilder().UnmarshalJSON(j) -// arr.Release() -// arr = uuidBuilder.NewArray() -// uuidBuilder.Release() -// case arrowSchema.Field(c).Type.(arrow.ExtensionType).ExtensionEquals(types.NewJSONType()): -// bldr := array.NewExtensionBuilder(mem, types.NewJSONType()) -// jsonBuilder := types.NewJSONBuilder(bldr) -// j, err := arr.MarshalJSON() -// if err != nil { -// return nil, fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) -// } -// jsonBuilder.StorageBuilder().UnmarshalJSON(j) -// arr.Release() -// arr = jsonBuilder.NewArray() -// case arrowSchema.Field(c).Type.(arrow.ExtensionType).ExtensionEquals(types.NewInetType()): -// bldr := array.NewExtensionBuilder(mem, types.NewInetType()) -// inetBuilder := types.NewInetBuilder(bldr) -// j, err := arr.MarshalJSON() -// if err != nil { -// return nil, fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) -// } -// inetBuilder.StorageBuilder().UnmarshalJSON(j) -// arr.Release() -// arr = inetBuilder.NewArray() -// } -// } else if arrow.TypeEqual(arrowSchema.Field(c).Type, arrow.ListOf(types.NewUUIDType())) { -// // TODO: handle -// continue -// } else { -// b, err := arr.MarshalJSON() -// if err != nil { -// return nil, fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) -// } -// err = rb.Field(c).UnmarshalJSON(b) -// if err != nil { -// return nil, fmt.Errorf("failed to unmarshal col %v: %w", rec.ColumnName(c), err) -// } -// } -// } -// return rb.NewRecord(), nil -//} diff --git a/parquet/write.go b/parquet/write.go index b58656b5..df971022 100644 --- a/parquet/write.go +++ b/parquet/write.go @@ -21,41 +21,4 @@ func (*Client) WriteTableBatch(w io.Writer, arrowSchema *arrow.Schema, records [ } } return fw.Close() - //parquetSchema, err := makeSchema(table.Name, table.Columns) - //if err != nil { - // return fmt.Errorf("failed to create parquet schema: %w", err) - //} - //wf := writerfile.NewWriterFile(w) - //arrowSchema := table.ToArrowSchema() - //pw, err := writer.NewArrowWriter(arrowSchema, wf, 2) - //if err != nil { - // return fmt.Errorf("can't create parquet writer: %w", err) - //} - // - //pw, err := writer.NewJSONWriterFromWriter(parquetSchema, w, 2) - //if err != nil { - // return fmt.Errorf("can't create parquet writer: %w", err) - //} - // - //pw.RowGroupSize = 128 * 1024 * 1024 // 128M - //pw.CompressionType = parquet.CompressionCodec_SNAPPY - // - //for _, rec := range records { - // for r := int64(0); r < rec.NumRows(); r++ { - // m := rec.NewSlice(r, r+1) - // b, err := json.Marshal(m) - // if err != nil { - // return fmt.Errorf("can't marshal record: %w", err) - // } - // // hacky, but we don't have a better way to get json for a single row - // b = bytes.TrimPrefix(b, []byte("[")) - // b = bytes.TrimSuffix(b, []byte("]")) - // if err := pw.Write(b); err != nil { - // return err - // } - // } - // pw.Flush(true) - //} - // - //return pw.WriteStop() } From c8573cd2f8d9884fde338ece258c6f270c7121b2 Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Tue, 11 Apr 2023 15:59:53 +0100 Subject: [PATCH 08/25] Linting --- client.go | 15 +++------------ json/read.go | 2 +- json/write.go | 4 ++-- parquet/read.go | 12 ++++++++---- parquet/write_read_test.go | 33 +++++++++++++++++++++++---------- 5 files changed, 37 insertions(+), 29 deletions(-) diff --git a/client.go b/client.go index 768a4806..360e33b7 100644 --- a/client.go +++ b/client.go @@ -4,23 +4,14 @@ import ( csvFile "github.com/cloudquery/filetypes/csv" jsonFile "github.com/cloudquery/filetypes/json" "github.com/cloudquery/filetypes/parquet" - "github.com/cloudquery/plugin-sdk/v2/schema" ) type Client struct { spec *FileSpec - csv *csvFile.Client - csvTransformer schema.DefaultTransformer - csvReverseTransformer csvFile.ReverseTransformer - - json *jsonFile.Client - jsonTransformer schema.DefaultTransformer - jsonReverseTransformer jsonFile.ReverseTransformer - - parquet *parquet.Client - parquetTransformer parquet.Transformer - parquetReverseTransformer parquet.ReverseTransformer + csv *csvFile.Client + json *jsonFile.Client + parquet *parquet.Client } // NewClient creates a new client for the given spec diff --git a/json/read.go b/json/read.go index 8a637a47..eab51049 100644 --- a/json/read.go +++ b/json/read.go @@ -11,7 +11,7 @@ import ( const maxJSONSize = 1024 * 1024 * 20 -func (*Client) Read(r io.Reader, arrowSchema *arrow.Schema, sourceName string, res chan<- arrow.Record) error { +func (*Client) Read(r io.Reader, arrowSchema *arrow.Schema, _ string, res chan<- arrow.Record) error { scanner := bufio.NewScanner(r) scanner.Buffer(make([]byte, maxJSONSize), maxJSONSize) rb := array.NewRecordBuilder(memory.DefaultAllocator, arrowSchema) diff --git a/json/write.go b/json/write.go index 310fed8c..29e671ae 100644 --- a/json/write.go +++ b/json/write.go @@ -10,7 +10,7 @@ import ( func (c *Client) WriteTableBatch(w io.Writer, _ *arrow.Schema, records []arrow.Record) error { for _, r := range records { - err := c.writeTableBatch(w, r) + err := c.writeRecord(w, r) if err != nil { return err } @@ -18,7 +18,7 @@ func (c *Client) WriteTableBatch(w io.Writer, _ *arrow.Schema, records []arrow.R return nil } -func (c *Client) writeTableBatch(w io.Writer, record arrow.Record) error { +func (*Client) writeRecord(w io.Writer, record arrow.Record) error { arr := array.RecordToStructArray(record) defer arr.Release() enc := json.NewEncoder(w) diff --git a/parquet/read.go b/parquet/read.go index 34e57f0c..6d96dba3 100644 --- a/parquet/read.go +++ b/parquet/read.go @@ -2,6 +2,7 @@ package parquet import ( "context" + "fmt" "io" "github.com/apache/arrow/go/v12/arrow" @@ -16,21 +17,24 @@ type ReaderAtSeeker interface { io.Seeker } -func (*Client) Read(f ReaderAtSeeker, arrowSchema *arrow.Schema, sourceName string, res chan<- arrow.Record) error { +func (*Client) Read(f ReaderAtSeeker, _ *arrow.Schema, _ string, res chan<- arrow.Record) error { mem := memory.DefaultAllocator ctx := context.Background() rdr, err := file.NewParquetReader(f) if err != nil { - return err + return fmt.Errorf("failed to create new parquet reader: %w", err) } arrProps := pqarrow.ArrowReadProperties{ Parallel: false, BatchSize: 1024, } fr, err := pqarrow.NewFileReader(rdr, arrProps, mem) + if err != nil { + return fmt.Errorf("failed to create new parquet file reader: %w", err) + } rr, err := fr.GetRecordReader(ctx, nil, nil) if err != nil { - return err + return fmt.Errorf("failed to get parquet record reader: %w", err) } for rr.Next() { rec := rr.Record() @@ -40,7 +44,7 @@ func (*Client) Read(f ReaderAtSeeker, arrowSchema *arrow.Schema, sourceName stri if err == io.EOF { break } else if err != nil { - return err + return fmt.Errorf("error while reading record: %w", err) } } rr.Release() diff --git a/parquet/write_read_test.go b/parquet/write_read_test.go index 786cd802..3f7027d5 100644 --- a/parquet/write_read_test.go +++ b/parquet/write_read_test.go @@ -109,7 +109,8 @@ func castExtensionColsToStorageType(mem memory.Allocator, rec arrow.Record) (arr defer rb.Release() for c := 0; c < int(rec.NumCols()); c++ { col := rec.Column(c) - if col.DataType().ID() == arrow.EXTENSION { + switch { + case col.DataType().ID() == arrow.EXTENSION: storageType := col.DataType().(arrow.ExtensionType).StorageType() arr, err := compute.CastToType(ctx, rec.Column(c), storageType) if err != nil { @@ -123,15 +124,27 @@ func castExtensionColsToStorageType(mem memory.Allocator, rec arrow.Record) (arr if err != nil { return nil, fmt.Errorf("failed to unmarshal col %v: %w", rec.ColumnName(c), err) } - } else if arrow.TypeEqual(col.DataType(), arrow.ListOf(types.NewUUIDType())) { - castListOf(ctx, rec, c, rb, types.NewUUIDType().StorageType()) - } else if arrow.TypeEqual(col.DataType(), arrow.ListOf(types.NewJSONType())) { - castListOf(ctx, rec, c, rb, types.NewJSONType().StorageType()) - } else if arrow.TypeEqual(col.DataType(), arrow.ListOf(types.NewInetType())) { - castListOf(ctx, rec, c, rb, types.NewInetType().StorageType()) - } else if arrow.TypeEqual(col.DataType(), arrow.ListOf(types.NewMacType())) { - castListOf(ctx, rec, c, rb, types.NewMacType().StorageType()) - } else { + case arrow.TypeEqual(col.DataType(), arrow.ListOf(types.NewUUIDType())): + err := castListOf(ctx, rec, c, rb, types.NewUUIDType().StorageType()) + if err != nil { + return nil, fmt.Errorf("failed to cast col %v: %w", rec.ColumnName(c), err) + } + case arrow.TypeEqual(col.DataType(), arrow.ListOf(types.NewJSONType())): + err := castListOf(ctx, rec, c, rb, types.NewJSONType().StorageType()) + if err != nil { + return nil, fmt.Errorf("failed to cast col %v: %w", rec.ColumnName(c), err) + } + case arrow.TypeEqual(col.DataType(), arrow.ListOf(types.NewInetType())): + err := castListOf(ctx, rec, c, rb, types.NewInetType().StorageType()) + if err != nil { + return nil, fmt.Errorf("failed to cast col %v: %w", rec.ColumnName(c), err) + } + case arrow.TypeEqual(col.DataType(), arrow.ListOf(types.NewMacType())): + err := castListOf(ctx, rec, c, rb, types.NewMacType().StorageType()) + if err != nil { + return nil, fmt.Errorf("failed to cast col %v: %w", rec.ColumnName(c), err) + } + default: b, err := rec.Column(c).MarshalJSON() if err != nil { return nil, fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) From 35ed197498333d3dffc823a85a823c4abfe9ba55 Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Tue, 11 Apr 2023 16:53:51 +0100 Subject: [PATCH 09/25] Add snapshot tests for csv --- csv/.snapshots/TestWriteRead-default.csv | Bin 0 -> 596 bytes .../TestWriteRead-with_delimiter.csv | Bin 0 -> 582 bytes .../TestWriteRead-with_delimiter_headers.csv | Bin 0 -> 801 bytes csv/.snapshots/TestWriteRead-with_headers.csv | Bin 0 -> 815 bytes csv/write_read_test.go | 39 +++++++++++++----- go.mod | 5 ++- go.sum | 4 ++ 7 files changed, 36 insertions(+), 12 deletions(-) create mode 100644 csv/.snapshots/TestWriteRead-default.csv create mode 100644 csv/.snapshots/TestWriteRead-with_delimiter.csv create mode 100644 csv/.snapshots/TestWriteRead-with_delimiter_headers.csv create mode 100644 csv/.snapshots/TestWriteRead-with_headers.csv diff --git a/csv/.snapshots/TestWriteRead-default.csv b/csv/.snapshots/TestWriteRead-default.csv new file mode 100644 index 0000000000000000000000000000000000000000..3e413f4d16716ea8e188e79c0cb5d6003adde89e GIT binary patch literal 596 zcmbtRy$*sf5O(G%%*_Y(Dwuei#>K_NI5-0qB_WzX8I0lG?GgL|7mb(e{omJotu>YP zt4m#`2DQ@UE77r1+$yS0eDMR*wdlJcTh;~%Dfn`Jv9o1OFI zu2%>*7g|EGx`BQ^k_}mC8-z0OWT;;Z0pJE85CHrL8dr629m65<_8opi2P$x9 yqC*GWOW%{1gwvD{2R4SY%t3$Fh*)c$PB{!Yig-3_z!S!Q|2%$*=iLFsYKpJ` literal 0 HcmV?d00001 diff --git a/csv/.snapshots/TestWriteRead-with_delimiter.csv b/csv/.snapshots/TestWriteRead-with_delimiter.csv new file mode 100644 index 0000000000000000000000000000000000000000..4529958654b33f4a632e181a5dc51e9ce275a078 GIT binary patch literal 582 zcmbtRJr9B~5OqC%h3QWbQ+P_@#Aikk=MWb+9mb26@^ zCp@zjGBB6vb-dsFK#U@aV@eWAQ&Nhu45|^t5lKq&eF_2!Ls~9t$145%uVHMC^8z0+ Bg3tf} literal 0 HcmV?d00001 diff --git a/csv/.snapshots/TestWriteRead-with_delimiter_headers.csv b/csv/.snapshots/TestWriteRead-with_delimiter_headers.csv new file mode 100644 index 0000000000000000000000000000000000000000..547076474fbc56acb5f7e414207d70e3717c478b GIT binary patch literal 801 zcmb_ZO;5ux4BgfH6)5L!G}(tlZbMwSAR!JMqcqJRs7puc3Z`oR9jD8nY?mPsrFP!) zs~tPaysM^egR#o%jRh6)&Krd;Ay@X~RtM`*5i07s0_PFdRjm=)mN>MZ*cA|xnnZtMQixwB!q8gaz^51LS5Fw6e}UwmX=I}Z8rdyPtbx_z z{T+1i_B@ML?0=zAlt(quJfq-uZ{nvn##qN#{F^a(JnV^uF12>raX)}#+t~#os?2Z1 zG@sIUE#`EY0m_Ep4??YFQAlHCSxRf=dJWwfm2obGkh7Vb&*fqvm&^VCyfvJ!@%RgQ C0M%Ro literal 0 HcmV?d00001 diff --git a/csv/.snapshots/TestWriteRead-with_headers.csv b/csv/.snapshots/TestWriteRead-with_headers.csv new file mode 100644 index 0000000000000000000000000000000000000000..fa52bc15f99fb71d82a2b0a919734d9fb864fe8f GIT binary patch literal 815 zcmb_ZO;5ux4BdIZ0_E(LCi{@cZHNmOB*cMZl%^R3wdqJ*!8Gl^<8&Ex+hvo8QakV2 zd9EF0K2_Uv!C2*WZ9zr6@kXIb$dx_0*1>vIgo>uAzB~i)m*)fvZ zWZxTzT<}RA_f4M5enXztB>EGRLj1BM44nlIJi9P|`GUduH%R`KMmBn)k=@h8T39|m zJ^Qd%olE0R9vIF~}m=~T{Uaz2-f#g6^Y%ftB^kADE{ CKHxq8 literal 0 HcmV?d00001 diff --git a/csv/write_read_test.go b/csv/write_read_test.go index b7511f41..6c26fc15 100644 --- a/csv/write_read_test.go +++ b/csv/write_read_test.go @@ -1,12 +1,15 @@ package csv import ( + "bufio" "bytes" + "io" "testing" "time" "github.com/apache/arrow/go/v12/arrow" "github.com/apache/arrow/go/v12/arrow/memory" + "github.com/bradleyjkemp/cupaloy/v2" "github.com/cloudquery/plugin-sdk/v2/plugins/destination" "github.com/cloudquery/plugin-sdk/v2/testdata" "github.com/google/uuid" @@ -18,10 +21,10 @@ func TestWriteRead(t *testing.T) { options []Options outputCount int }{ - {name: "default", outputCount: 1}, - {name: "with_headers", options: []Options{WithHeader()}, outputCount: 1}, - {name: "with_delimiter", options: []Options{WithDelimiter('\t')}, outputCount: 1}, - {name: "with_delimiter_headers", options: []Options{WithDelimiter('\t'), WithHeader()}, outputCount: 1}, + {name: "default", outputCount: 2}, + {name: "with_headers", options: []Options{WithHeader()}, outputCount: 2}, + {name: "with_delimiter", options: []Options{WithDelimiter('\t')}, outputCount: 2}, + {name: "with_delimiter_headers", options: []Options{WithDelimiter('\t'), WithHeader()}, outputCount: 2}, } for _, tc := range cases { @@ -30,14 +33,15 @@ func TestWriteRead(t *testing.T) { table := testdata.TestTable("test") arrowSchema := table.ToArrowSchema() sourceName := "test-source" - syncTime := time.Now().UTC().Round(1 * time.Second) + syncTime := time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC) mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) defer mem.AssertSize(t, 0) opts := testdata.GenTestDataOptions{ SourceName: sourceName, SyncTime: syncTime, - MaxRows: 1, - StableUUID: uuid.Nil, + MaxRows: 2, + StableUUID: uuid.MustParse("00000000-0000-0000-0000-000000000001"), + StableTime: time.Date(2021, 1, 2, 0, 0, 0, 0, time.UTC), } records := testdata.GenTestData(mem, arrowSchema, opts) defer func() { @@ -49,21 +53,34 @@ func TestWriteRead(t *testing.T) { if err != nil { t.Fatal(err) } - if err := cl.WriteTableBatch(&b, arrowSchema, records); err != nil { + + writer := bufio.NewWriter(&b) + reader := bufio.NewReader(&b) + + if err := cl.WriteTableBatch(writer, arrowSchema, records); err != nil { t.Fatal(err) } + rawBytes, err := io.ReadAll(reader) + if err != nil { + t.Fatal(err) + } + snap := cupaloy.New(cupaloy.SnapshotFileExtension(".csv")) + snap.SnapshotT(t, string(rawBytes)) + + byteReader := bytes.NewReader(rawBytes) + ch := make(chan arrow.Record) var readErr error go func() { - readErr = cl.Read(&b, arrowSchema, "test-source", ch) + readErr = cl.Read(byteReader, arrowSchema, "test-source", ch) close(ch) }() totalCount := 0 for got := range ch { if diff := destination.RecordDiff(records[totalCount], got); diff != "" { got.Release() - t.Fatalf("got diff: %s", diff) + t.Errorf("got diff: %s", diff) } got.Release() totalCount++ @@ -72,7 +89,7 @@ func TestWriteRead(t *testing.T) { t.Fatal(readErr) } if totalCount != tc.outputCount { - t.Fatalf("got %d row(s), want %d", totalCount, tc.outputCount) + t.Errorf("got %d row(s), want %d", totalCount, tc.outputCount) } }) } diff --git a/go.mod b/go.mod index 546541e7..e1872d3c 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,10 @@ replace github.com/apache/arrow/go/v12 => ../arrow/go replace github.com/cloudquery/plugin-sdk/v2 => ../plugin-sdk -require golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect +require ( + github.com/bradleyjkemp/cupaloy/v2 v2.8.0 // indirect + golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect +) require ( github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect diff --git a/go.sum b/go.sum index 525c259b..3344eb70 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/ github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/apache/thrift v0.18.1 h1:lNhK/1nqjbwbiOPDBPFJVKxgDEGSepKuTh6OLiXW8kg= github.com/apache/thrift v0.18.1/go.mod h1:rdQn/dCcDKEWjjylUeueum4vQEjG2v8v2PqriUnbr+I= +github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= +github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -58,10 +60,12 @@ github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.29.0 h1:Zes4hju04hjbvkVkOhdl2HpZa+0PmVwigmo8XoORE5w= github.com/rs/zerolog v1.29.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= From e29eaee9cdbbfcdbff9b389bfcd84d8bfbea049e Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Tue, 11 Apr 2023 17:01:47 +0100 Subject: [PATCH 10/25] Update expectations to match 2.0.1 --- csv/.snapshots/TestWriteRead-default.csv | Bin 596 -> 592 bytes csv/.snapshots/TestWriteRead-with_delimiter.csv | Bin 582 -> 578 bytes .../TestWriteRead-with_delimiter_headers.csv | Bin 801 -> 797 bytes csv/.snapshots/TestWriteRead-with_headers.csv | Bin 815 -> 811 bytes 4 files changed, 0 insertions(+), 0 deletions(-) diff --git a/csv/.snapshots/TestWriteRead-default.csv b/csv/.snapshots/TestWriteRead-default.csv index 3e413f4d16716ea8e188e79c0cb5d6003adde89e..f6455a52735cf43819adbe681e780abe6f5f347f 100644 GIT binary patch delta 38 ncmcb@a)D(7GoyI5l2S=(afya)o6BGo!SUQniv&NosM4l9CmSQLCh+q@z?lQNd$!65~4n23!n| diff --git a/csv/.snapshots/TestWriteRead-with_delimiter.csv b/csv/.snapshots/TestWriteRead-with_delimiter.csv index 4529958654b33f4a632e181a5dc51e9ce275a078..e1bff50c1d6804c6d4838895c8f4cc3afa01bd4f 100644 GIT binary patch delta 37 mcmX@ca)@QZU$JT>rIOU*5+x-o7^7B+vwEU}%4TauZbkt1?F(!G delta 35 kcmX@aa*SodUvVX+Y9*zT)Z!8)B`X-CR!M2I0HYBj0M*$Fb^rhX diff --git a/csv/.snapshots/TestWriteRead-with_delimiter_headers.csv b/csv/.snapshots/TestWriteRead-with_delimiter_headers.csv index 547076474fbc56acb5f7e414207d70e3717c478b..ce164f3543789289e131bdc423cdb8b8e797c18a 100644 GIT binary patch delta 32 hcmZ3;HkWO~4MyQ=C8d(o;u0k#D;T48^Ls`kMgXzw3V#3q delta 43 scmbQswvcVZ4Mr&?rD`RmlGNf7B_%5uqgF{tiL-jL0;9@iAtr7{03Cu0i2wiq diff --git a/csv/.snapshots/TestWriteRead-with_headers.csv b/csv/.snapshots/TestWriteRead-with_headers.csv index fa52bc15f99fb71d82a2b0a919734d9fb864fe8f..f5e447ecb8f4a6d9a93b71fd9ac70396b459db56 100644 GIT binary patch delta 27 hcmZ3_wwi6jJ;uoo7)6+Ml&U8yFscHv@?=M*cL0Nu2)+OS delta 36 lcmZ3@ww`UnJw|aQrD`RmlGNf7B_%5uqgF|2^G8N2MgZ3)3pW4& From d87cdca3cf9ad08c3ba604faca42b0bae91a3be2 Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Wed, 12 Apr 2023 14:53:26 +0100 Subject: [PATCH 11/25] Introduce snapshot test for json --- csv/write_read_test.go | 5 +++-- json/.snapshots/TestWriteRead.jsonl | 3 +++ json/write.go | 1 + json/write_read_test.go | 34 +++++++++++++++++++++++------ 4 files changed, 34 insertions(+), 9 deletions(-) create mode 100644 json/.snapshots/TestWriteRead.jsonl diff --git a/csv/write_read_test.go b/csv/write_read_test.go index 6c26fc15..029dada2 100644 --- a/csv/write_read_test.go +++ b/csv/write_read_test.go @@ -29,7 +29,6 @@ func TestWriteRead(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - var b bytes.Buffer table := testdata.TestTable("test") arrowSchema := table.ToArrowSchema() sourceName := "test-source" @@ -54,13 +53,15 @@ func TestWriteRead(t *testing.T) { t.Fatal(err) } + var b bytes.Buffer writer := bufio.NewWriter(&b) reader := bufio.NewReader(&b) if err := cl.WriteTableBatch(writer, arrowSchema, records); err != nil { t.Fatal(err) } - + writer.Flush() + rawBytes, err := io.ReadAll(reader) if err != nil { t.Fatal(err) diff --git a/json/.snapshots/TestWriteRead.jsonl b/json/.snapshots/TestWriteRead.jsonl new file mode 100644 index 00000000..0191cb56 --- /dev/null +++ b/json/.snapshots/TestWriteRead.jsonl @@ -0,0 +1,3 @@ +{"_cq_id":"00000000-0000-0000-0000-000000000001","_cq_parent_id":"00000000-0000-0000-0000-000000000001","_cq_source_name":"test-source","_cq_sync_time":"2021-01-01 00:00:00","bool":true,"bytea":"AQID","cidr":"192.0.2.0/24","cidr_array":["192.0.2.0/24","192.0.2.0/24"],"float":1.1,"inet":"192.0.2.0/24","inet_array":["192.0.2.0/24","192.0.2.0/24"],"int":1,"int_array":[1,2],"json":{"test":"test"},"macaddr":"aa:bb:cc:dd:ee:ff","macaddr_array":["aa:bb:cc:dd:ee:ff","11:22:33:44:55:66"],"text":"AString","text_array":["test1","test2"],"text_array_with_null":["test1","test2\u0000WithNull"],"text_with_null":"AStringWith\u0000NullBytes","timestamp":"2021-01-02 00:00:00","uuid":"00000000-0000-0000-0000-000000000001","uuid_array":["00000000-0000-0000-0000-000000000001","00000000-0000-0000-0000-000000000002"]} +{"_cq_id":"00000000-0000-0000-0000-000000000001","_cq_parent_id":null,"_cq_source_name":"test-source","_cq_sync_time":"2021-01-01 00:00:00","bool":null,"bytea":null,"cidr":null,"cidr_array":null,"float":null,"inet":null,"inet_array":null,"int":null,"int_array":null,"json":null,"macaddr":null,"macaddr_array":null,"text":null,"text_array":null,"text_array_with_null":null,"text_with_null":null,"timestamp":null,"uuid":null,"uuid_array":null} + diff --git a/json/write.go b/json/write.go index 29e671ae..1524eadc 100644 --- a/json/write.go +++ b/json/write.go @@ -22,6 +22,7 @@ func (*Client) writeRecord(w io.Writer, record arrow.Record) error { arr := array.RecordToStructArray(record) defer arr.Release() enc := json.NewEncoder(w) + enc.SetEscapeHTML(false) for i := 0; i < arr.Len(); i++ { if err := enc.Encode(arr.GetOneForMarshal(i)); err != nil { return err diff --git a/json/write_read_test.go b/json/write_read_test.go index e2ba313d..08b492ac 100644 --- a/json/write_read_test.go +++ b/json/write_read_test.go @@ -1,14 +1,18 @@ package json import ( + "bufio" "bytes" + "io" "testing" "time" "github.com/apache/arrow/go/v12/arrow" "github.com/apache/arrow/go/v12/arrow/memory" + "github.com/bradleyjkemp/cupaloy/v2" "github.com/cloudquery/plugin-sdk/v2/plugins/destination" "github.com/cloudquery/plugin-sdk/v2/testdata" + "github.com/google/uuid" ) func TestWrite(t *testing.T) { @@ -41,17 +45,18 @@ func TestWrite(t *testing.T) { } func TestWriteRead(t *testing.T) { - var b bytes.Buffer table := testdata.TestTable("test") arrowSchema := table.ToArrowSchema() sourceName := "test-source" - syncTime := time.Now().UTC().Round(1 * time.Second) + syncTime := time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC) mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) defer mem.AssertSize(t, 0) opts := testdata.GenTestDataOptions{ SourceName: sourceName, SyncTime: syncTime, - MaxRows: 1, + MaxRows: 2, + StableUUID: uuid.MustParse("00000000-0000-0000-0000-000000000001"), + StableTime: time.Date(2021, 1, 2, 0, 0, 0, 0, time.UTC), } records := testdata.GenTestData(mem, arrowSchema, opts) defer func() { @@ -63,14 +68,29 @@ func TestWriteRead(t *testing.T) { if err != nil { t.Fatal(err) } - if err := cl.WriteTableBatch(&b, arrowSchema, records); err != nil { + + var b bytes.Buffer + writer := bufio.NewWriter(&b) + reader := bufio.NewReader(&b) + + if err := cl.WriteTableBatch(writer, arrowSchema, records); err != nil { + t.Fatal(err) + } + writer.Flush() + + rawBytes, err := io.ReadAll(reader) + if err != nil { t.Fatal(err) } + snap := cupaloy.New(cupaloy.SnapshotFileExtension(".jsonl")) + snap.SnapshotT(t, string(rawBytes)) + + byteReader := bytes.NewReader(rawBytes) ch := make(chan arrow.Record) var readErr error go func() { - readErr = cl.Read(&b, arrowSchema, "test-source", ch) + readErr = cl.Read(byteReader, arrowSchema, "test-source", ch) close(ch) }() totalCount := 0 @@ -83,7 +103,7 @@ func TestWriteRead(t *testing.T) { if readErr != nil { t.Fatal(readErr) } - if totalCount != 1 { - t.Fatalf("expected 1 row, got %d", totalCount) + if totalCount != 2 { + t.Fatalf("expected 2 rows, got %d", totalCount) } } From 1a2d15f7c57e2fb4e964f7381629a4805232360f Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Thu, 13 Apr 2023 17:04:19 +0100 Subject: [PATCH 12/25] Fix csv and parquet --- csv/write_read_test.go | 13 +-- parquet/write.go | 140 ++++++++++++++++++++++++++++++- parquet/write_read_test.go | 168 ++++++++++++++++--------------------- 3 files changed, 219 insertions(+), 102 deletions(-) diff --git a/csv/write_read_test.go b/csv/write_read_test.go index 029dada2..7d5e3b72 100644 --- a/csv/write_read_test.go +++ b/csv/write_read_test.go @@ -10,7 +10,6 @@ import ( "github.com/apache/arrow/go/v12/arrow" "github.com/apache/arrow/go/v12/arrow/memory" "github.com/bradleyjkemp/cupaloy/v2" - "github.com/cloudquery/plugin-sdk/v2/plugins/destination" "github.com/cloudquery/plugin-sdk/v2/testdata" "github.com/google/uuid" ) @@ -61,7 +60,7 @@ func TestWriteRead(t *testing.T) { t.Fatal(err) } writer.Flush() - + rawBytes, err := io.ReadAll(reader) if err != nil { t.Fatal(err) @@ -79,10 +78,12 @@ func TestWriteRead(t *testing.T) { }() totalCount := 0 for got := range ch { - if diff := destination.RecordDiff(records[totalCount], got); diff != "" { - got.Release() - t.Errorf("got diff: %s", diff) - } + // TODO: compare records. This is currently failing because the CSV reader fails + // to read the JSON column properly. We rely on snapshot tests for now. + //if diff := destination.RecordDiff(records[totalCount], got); diff != "" { + // got.Release() + // t.Errorf("got diff: %s", diff) + //} got.Release() totalCount++ } diff --git a/parquet/write.go b/parquet/write.go index df971022..a71a5a4c 100644 --- a/parquet/write.go +++ b/parquet/write.go @@ -1,24 +1,160 @@ package parquet import ( + "context" + "encoding/json" + "fmt" "io" "github.com/apache/arrow/go/v12/arrow" + "github.com/apache/arrow/go/v12/arrow/array" + "github.com/apache/arrow/go/v12/arrow/compute" + "github.com/apache/arrow/go/v12/arrow/memory" "github.com/apache/arrow/go/v12/parquet" "github.com/apache/arrow/go/v12/parquet/pqarrow" + "github.com/cloudquery/plugin-sdk/v2/types" ) func (*Client) WriteTableBatch(w io.Writer, arrowSchema *arrow.Schema, records []arrow.Record) error { props := parquet.NewWriterProperties() arrprops := pqarrow.DefaultWriterProps() - fw, err := pqarrow.NewFileWriter(arrowSchema, w, props, arrprops) + newSchema := convertSchema(arrowSchema) + fw, err := pqarrow.NewFileWriter(newSchema, w, props, arrprops) if err != nil { return err } + mem := memory.DefaultAllocator for _, rec := range records { - if err := fw.Write(rec); err != nil { + castRec, err := castExtensionColsToString(mem, rec) + if err != nil { + return fmt.Errorf("failed to cast to string: %w", err) + } + if err := fw.Write(castRec); err != nil { return err } } return fw.Close() } + +func convertSchema(sch *arrow.Schema) *arrow.Schema { + oldFields := sch.Fields() + fields := make([]arrow.Field, len(oldFields)) + copy(fields, oldFields) + for i, f := range fields { + switch { + case f.Type.ID() == arrow.EXTENSION: + fields[i].Type = arrow.BinaryTypes.String + case arrow.TypeEqual(f.Type, arrow.ListOf(types.NewUUIDType())), + arrow.TypeEqual(f.Type, arrow.ListOf(types.NewInetType())), + arrow.TypeEqual(f.Type, arrow.ListOf(types.NewJSONType())), + arrow.TypeEqual(f.Type, arrow.ListOf(types.NewMacType())): + fields[i].Type = arrow.ListOf(arrow.BinaryTypes.String) + } + } + + md := sch.Metadata() + newSchema := arrow.NewSchema(fields, &md) + return newSchema +} + +func castExtensionColsToString(mem memory.Allocator, rec arrow.Record) (arrow.Record, error) { + newSchema := convertSchema(rec.Schema()) + rb := array.NewRecordBuilder(mem, newSchema) + + defer rb.Release() + for c := 0; c < int(rec.NumCols()); c++ { + col := rec.Column(c) + switch { + case arrow.TypeEqual(col.DataType(), types.NewUUIDType()): + arr := col.(*types.UUIDArray) + b, err := arr.MarshalJSON() + if err != nil { + return nil, fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) + } + err = rb.Field(c).UnmarshalJSON(b) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal col %v: %w", rec.ColumnName(c), err) + } + case arrow.TypeEqual(col.DataType(), types.NewInetType()): + arr := col.(*types.InetArray) + b, err := arr.MarshalJSON() + if err != nil { + return nil, fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) + } + err = rb.Field(c).UnmarshalJSON(b) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal col %v: %w", rec.ColumnName(c), err) + } + case arrow.TypeEqual(col.DataType(), types.NewJSONType()): + arr := col.(*types.JSONArray) + b, err := arr.MarshalJSON() + if err != nil { + return nil, fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) + } + a := make([]any, arr.Len()) + err = json.Unmarshal(b, &a) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal col %v: %w", rec.ColumnName(c), err) + } + for _, v := range a { + if v == nil { + rb.Field(c).(*array.StringBuilder).AppendNull() + } + b, err := json.Marshal(v) + if err != nil { + return nil, fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) + } + rb.Field(c).(*array.StringBuilder).Append(string(b)) + } + case arrow.TypeEqual(col.DataType(), types.NewMacType()): + arr := col.(*types.MacArray) + b, err := arr.MarshalJSON() + if err != nil { + return nil, fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) + } + err = rb.Field(c).UnmarshalJSON(b) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal col %v: %w", rec.ColumnName(c), err) + } + case arrow.TypeEqual(col.DataType(), arrow.ListOf(types.NewUUIDType())), + arrow.TypeEqual(col.DataType(), arrow.ListOf(types.NewJSONType())), + arrow.TypeEqual(col.DataType(), arrow.ListOf(types.NewInetType())), + arrow.TypeEqual(col.DataType(), arrow.ListOf(types.NewMacType())): + arr := col.(*array.List) + b, err := arr.MarshalJSON() + if err != nil { + return nil, fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) + } + err = rb.Field(c).UnmarshalJSON(b) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal col %v: %w", rec.ColumnName(c), err) + } + default: + b, err := rec.Column(c).MarshalJSON() + if err != nil { + return nil, fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) + } + err = rb.Field(c).UnmarshalJSON(b) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal col %v: %w", rec.ColumnName(c), err) + } + } + } + return rb.NewRecord(), nil +} + +func castListOf(ctx context.Context, rec arrow.Record, c int, rb *array.RecordBuilder, storageType arrow.DataType) error { + arr, err := compute.CastToType(ctx, rec.Column(c), arrow.ListOf(storageType)) + if err != nil { + return fmt.Errorf("failed to cast col %v to %v: %w", rec.ColumnName(c), storageType, err) + } + b, err := arr.MarshalJSON() + if err != nil { + return fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) + } + err = rb.Field(c).UnmarshalJSON(b) + if err != nil { + return fmt.Errorf("failed to unmarshal col %v: %w", rec.ColumnName(c), err) + } + return nil +} diff --git a/parquet/write_read_test.go b/parquet/write_read_test.go index 3f7027d5..e1d581f1 100644 --- a/parquet/write_read_test.go +++ b/parquet/write_read_test.go @@ -3,19 +3,14 @@ package parquet import ( "bufio" "bytes" - "context" - "fmt" "io" "testing" "time" "github.com/apache/arrow/go/v12/arrow" - "github.com/apache/arrow/go/v12/arrow/array" - "github.com/apache/arrow/go/v12/arrow/compute" "github.com/apache/arrow/go/v12/arrow/memory" "github.com/cloudquery/plugin-sdk/v2/plugins/destination" "github.com/cloudquery/plugin-sdk/v2/testdata" - "github.com/cloudquery/plugin-sdk/v2/types" ) func TestWriteRead(t *testing.T) { @@ -68,7 +63,7 @@ func TestWriteRead(t *testing.T) { }() totalCount := 0 for got := range ch { - baseRecord, err := castExtensionColsToStorageType(mem, records[totalCount]) + baseRecord, err := castExtensionColsToString(mem, records[totalCount]) if err != nil { t.Fatalf("failed to cast extensions to storage type for comparison: %v", err) } @@ -85,91 +80,76 @@ func TestWriteRead(t *testing.T) { } } -func castExtensionColsToStorageType(mem memory.Allocator, rec arrow.Record) (arrow.Record, error) { - oldFields := rec.Schema().Fields() - fields := make([]arrow.Field, len(oldFields)) - copy(fields, oldFields) - for i, f := range fields { - switch { - case f.Type.ID() == arrow.EXTENSION: - fields[i].Type = f.Type.(arrow.ExtensionType).StorageType() - case arrow.TypeEqual(f.Type, arrow.ListOf(types.NewUUIDType())), - arrow.TypeEqual(f.Type, arrow.ListOf(types.NewInetType())), - arrow.TypeEqual(f.Type, arrow.ListOf(types.NewJSONType())), - arrow.TypeEqual(f.Type, arrow.ListOf(types.NewMacType())): - fields[i].Type = arrow.ListOf(f.Type.(*arrow.ListType).Elem().(arrow.ExtensionType).StorageType()) - } - } - - md := rec.Schema().Metadata() - newSchema := arrow.NewSchema(fields, &md) - ctx := context.Background() - rb := array.NewRecordBuilder(mem, newSchema) - - defer rb.Release() - for c := 0; c < int(rec.NumCols()); c++ { - col := rec.Column(c) - switch { - case col.DataType().ID() == arrow.EXTENSION: - storageType := col.DataType().(arrow.ExtensionType).StorageType() - arr, err := compute.CastToType(ctx, rec.Column(c), storageType) - if err != nil { - return nil, fmt.Errorf("failed to cast col %v to %v: %w", rec.ColumnName(c), storageType, err) - } - b, err := arr.MarshalJSON() - if err != nil { - return nil, fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) - } - err = rb.Field(c).UnmarshalJSON(b) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal col %v: %w", rec.ColumnName(c), err) - } - case arrow.TypeEqual(col.DataType(), arrow.ListOf(types.NewUUIDType())): - err := castListOf(ctx, rec, c, rb, types.NewUUIDType().StorageType()) - if err != nil { - return nil, fmt.Errorf("failed to cast col %v: %w", rec.ColumnName(c), err) - } - case arrow.TypeEqual(col.DataType(), arrow.ListOf(types.NewJSONType())): - err := castListOf(ctx, rec, c, rb, types.NewJSONType().StorageType()) - if err != nil { - return nil, fmt.Errorf("failed to cast col %v: %w", rec.ColumnName(c), err) - } - case arrow.TypeEqual(col.DataType(), arrow.ListOf(types.NewInetType())): - err := castListOf(ctx, rec, c, rb, types.NewInetType().StorageType()) - if err != nil { - return nil, fmt.Errorf("failed to cast col %v: %w", rec.ColumnName(c), err) - } - case arrow.TypeEqual(col.DataType(), arrow.ListOf(types.NewMacType())): - err := castListOf(ctx, rec, c, rb, types.NewMacType().StorageType()) - if err != nil { - return nil, fmt.Errorf("failed to cast col %v: %w", rec.ColumnName(c), err) - } - default: - b, err := rec.Column(c).MarshalJSON() - if err != nil { - return nil, fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) - } - err = rb.Field(c).UnmarshalJSON(b) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal col %v: %w", rec.ColumnName(c), err) - } - } - } - return rb.NewRecord(), nil -} - -func castListOf(ctx context.Context, rec arrow.Record, c int, rb *array.RecordBuilder, storageType arrow.DataType) error { - arr, err := compute.CastToType(ctx, rec.Column(c), arrow.ListOf(storageType)) - if err != nil { - return fmt.Errorf("failed to cast col %v to %v: %w", rec.ColumnName(c), storageType, err) - } - b, err := arr.MarshalJSON() - if err != nil { - return fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) - } - err = rb.Field(c).UnmarshalJSON(b) - if err != nil { - return fmt.Errorf("failed to unmarshal col %v: %w", rec.ColumnName(c), err) - } - return nil -} +// +//func castExtensionColsToStringType(mem memory.Allocator, rec arrow.Record) (arrow.Record, error) { +// oldFields := rec.Schema().Fields() +// fields := make([]arrow.Field, len(oldFields)) +// copy(fields, oldFields) +// for i, f := range fields { +// switch { +// case f.Type.ID() == arrow.EXTENSION: +// fields[i].Type = f.Type.(arrow.ExtensionType).StorageType() +// case arrow.TypeEqual(f.Type, arrow.ListOf(types.NewUUIDType())), +// arrow.TypeEqual(f.Type, arrow.ListOf(types.NewInetType())), +// arrow.TypeEqual(f.Type, arrow.ListOf(types.NewJSONType())), +// arrow.TypeEqual(f.Type, arrow.ListOf(types.NewMacType())): +// fields[i].Type = arrow.ListOf(arrow.BinaryTypes.String) +// } +// } +// +// md := rec.Schema().Metadata() +// newSchema := arrow.NewSchema(fields, &md) +// ctx := context.Background() +// rb := array.NewRecordBuilder(mem, newSchema) +// +// defer rb.Release() +// for c := 0; c < int(rec.NumCols()); c++ { +// col := rec.Column(c) +// switch { +// case col.DataType().ID() == arrow.EXTENSION: +// storageType := col.DataType().(arrow.ExtensionType).StorageType() +// arr, err := compute.CastToType(ctx, rec.Column(c), storageType) +// if err != nil { +// return nil, fmt.Errorf("failed to cast col %v to %v: %w", rec.ColumnName(c), storageType, err) +// } +// b, err := arr.MarshalJSON() +// if err != nil { +// return nil, fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) +// } +// err = rb.Field(c).UnmarshalJSON(b) +// if err != nil { +// return nil, fmt.Errorf("failed to unmarshal col %v: %w", rec.ColumnName(c), err) +// } +// case arrow.TypeEqual(col.DataType(), arrow.ListOf(types.NewUUIDType())): +// err := castListOf(ctx, rec, c, rb, types.NewUUIDType().StorageType()) +// if err != nil { +// return nil, fmt.Errorf("failed to cast col %v: %w", rec.ColumnName(c), err) +// } +// case arrow.TypeEqual(col.DataType(), arrow.ListOf(types.NewJSONType())): +// err := castListOf(ctx, rec, c, rb, types.NewJSONType().StorageType()) +// if err != nil { +// return nil, fmt.Errorf("failed to cast col %v: %w", rec.ColumnName(c), err) +// } +// case arrow.TypeEqual(col.DataType(), arrow.ListOf(types.NewInetType())): +// err := castListOf(ctx, rec, c, rb, types.NewInetType().StorageType()) +// if err != nil { +// return nil, fmt.Errorf("failed to cast col %v: %w", rec.ColumnName(c), err) +// } +// case arrow.TypeEqual(col.DataType(), arrow.ListOf(types.NewMacType())): +// err := castListOf(ctx, rec, c, rb, types.NewMacType().StorageType()) +// if err != nil { +// return nil, fmt.Errorf("failed to cast col %v: %w", rec.ColumnName(c), err) +// } +// default: +// b, err := rec.Column(c).MarshalJSON() +// if err != nil { +// return nil, fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) +// } +// err = rb.Field(c).UnmarshalJSON(b) +// if err != nil { +// return nil, fmt.Errorf("failed to unmarshal col %v: %w", rec.ColumnName(c), err) +// } +// } +// } +// return rb.NewRecord(), nil +//} From 320f0814a8c3574e7c9d4429380d656eea4e8c35 Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Thu, 13 Apr 2023 20:44:11 +0100 Subject: [PATCH 13/25] Remove transformers --- csv/transformer.go | 14 -------------- json/transformer.go | 14 -------------- 2 files changed, 28 deletions(-) delete mode 100644 csv/transformer.go delete mode 100644 json/transformer.go diff --git a/csv/transformer.go b/csv/transformer.go deleted file mode 100644 index 90647ab6..00000000 --- a/csv/transformer.go +++ /dev/null @@ -1,14 +0,0 @@ -package csv - -import ( - "github.com/cloudquery/plugin-sdk/v2/plugins/destination" - "github.com/cloudquery/plugin-sdk/v2/schema" -) - -type ReverseTransformer struct { - *destination.DefaultReverseTransformer -} - -var _ interface { - ReverseTransformValues(table *schema.Table, values []any) (schema.CQTypes, error) -} = ReverseTransformer{} diff --git a/json/transformer.go b/json/transformer.go deleted file mode 100644 index 78415616..00000000 --- a/json/transformer.go +++ /dev/null @@ -1,14 +0,0 @@ -package json - -import ( - "github.com/cloudquery/plugin-sdk/v2/plugins/destination" - "github.com/cloudquery/plugin-sdk/v2/schema" -) - -type ReverseTransformer struct { - *destination.DefaultReverseTransformer -} - -var _ interface { - ReverseTransformValues(table *schema.Table, values []any) (schema.CQTypes, error) -} = ReverseTransformer{} From eb8d941f2cf0859fee35e12296482902e2b9d6bf Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Mon, 17 Apr 2023 10:11:13 +0100 Subject: [PATCH 14/25] Update tests --- csv/.snapshots/TestWriteRead-default.csv | Bin 592 -> 718 bytes csv/.snapshots/TestWriteRead-with_delimiter.csv | Bin 578 -> 704 bytes .../TestWriteRead-with_delimiter_headers.csv | Bin 797 -> 941 bytes csv/.snapshots/TestWriteRead-with_headers.csv | Bin 811 -> 955 bytes go.mod | 6 ++---- go.sum | 4 ++++ json/.snapshots/TestWriteRead.jsonl | 4 ++-- 7 files changed, 8 insertions(+), 6 deletions(-) diff --git a/csv/.snapshots/TestWriteRead-default.csv b/csv/.snapshots/TestWriteRead-default.csv index f6455a52735cf43819adbe681e780abe6f5f347f..480815d32d9b2e209138e10bed6cfd73784d507c 100644 GIT binary patch delta 63 tcmcb>a*lOE)?|A|WllpK$KaBp%)IoCnU@$rB9m&Fj#1_3*!;607y^K9_BR@kUo6M$VF=(o{}EPD8!P28`;HYZy5;?_um_6y?MOTwDNyPzquI diff --git a/csv/.snapshots/TestWriteRead-with_headers.csv b/csv/.snapshots/TestWriteRead-with_headers.csv index f5e447ecb8f4a6d9a93b71fd9ac70396b459db56..28b6265404dd7a2652f68e2681aa215efd7a0af0 100644 GIT binary patch delta 97 zcmZ3@wwrx|v0!OwW=ecPwoY+LQD$B`khRfBm~pZKqpFZ&Fjz>Zq^LAi$56*m&v5bw WCbi9_j0?f?AT=^DH98o8iwgi2aU3ZC delta 55 vcmdnZzM5@<@kUo6M$VF=(o`Ko9Yejz28`;H=QDF`-orSPQB(&LaB%?uxUvn9 diff --git a/go.mod b/go.mod index ff80bb9a..7edd7047 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/cloudquery/filetypes go 1.19 require ( - github.com/cloudquery/plugin-sdk/v2 v2.0.0 + github.com/cloudquery/plugin-sdk/v2 v2.2.0 github.com/davecgh/go-spew v1.1.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/stretchr/testify v1.8.2 @@ -28,9 +28,7 @@ require ( github.com/bradleyjkemp/cupaloy/v2 v2.8.0 ) -replace github.com/apache/arrow/go/v12 => ../arrow/go - -replace github.com/cloudquery/plugin-sdk/v2 => ../plugin-sdk +replace github.com/apache/arrow/go/v12 => github.com/cloudquery/arrow/go/v12 v12.0.0-20230417085404-0d8ebd3b01ce require golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect diff --git a/go.sum b/go.sum index 3344eb70..65a62786 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,10 @@ github.com/apache/thrift v0.18.1 h1:lNhK/1nqjbwbiOPDBPFJVKxgDEGSepKuTh6OLiXW8kg= github.com/apache/thrift v0.18.1/go.mod h1:rdQn/dCcDKEWjjylUeueum4vQEjG2v8v2PqriUnbr+I= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= +github.com/cloudquery/arrow/go/v12 v12.0.0-20230417085404-0d8ebd3b01ce h1:+wD+TQofOeFl9u90sY/0N9gkTUiJsPdp6SuhNv2ENnw= +github.com/cloudquery/arrow/go/v12 v12.0.0-20230417085404-0d8ebd3b01ce/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg= +github.com/cloudquery/plugin-sdk/v2 v2.2.0 h1:N5t/1M+CVGGBepJeNGbXH0jABN1S8qbvOo6fjz8P91M= +github.com/cloudquery/plugin-sdk/v2 v2.2.0/go.mod h1:pN05LsLG3WHRwri/IJBsYA+qMajUM7tRubjMWB2chvY= github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/json/.snapshots/TestWriteRead.jsonl b/json/.snapshots/TestWriteRead.jsonl index 0191cb56..e4fbdfea 100644 --- a/json/.snapshots/TestWriteRead.jsonl +++ b/json/.snapshots/TestWriteRead.jsonl @@ -1,3 +1,3 @@ -{"_cq_id":"00000000-0000-0000-0000-000000000001","_cq_parent_id":"00000000-0000-0000-0000-000000000001","_cq_source_name":"test-source","_cq_sync_time":"2021-01-01 00:00:00","bool":true,"bytea":"AQID","cidr":"192.0.2.0/24","cidr_array":["192.0.2.0/24","192.0.2.0/24"],"float":1.1,"inet":"192.0.2.0/24","inet_array":["192.0.2.0/24","192.0.2.0/24"],"int":1,"int_array":[1,2],"json":{"test":"test"},"macaddr":"aa:bb:cc:dd:ee:ff","macaddr_array":["aa:bb:cc:dd:ee:ff","11:22:33:44:55:66"],"text":"AString","text_array":["test1","test2"],"text_array_with_null":["test1","test2\u0000WithNull"],"text_with_null":"AStringWith\u0000NullBytes","timestamp":"2021-01-02 00:00:00","uuid":"00000000-0000-0000-0000-000000000001","uuid_array":["00000000-0000-0000-0000-000000000001","00000000-0000-0000-0000-000000000002"]} -{"_cq_id":"00000000-0000-0000-0000-000000000001","_cq_parent_id":null,"_cq_source_name":"test-source","_cq_sync_time":"2021-01-01 00:00:00","bool":null,"bytea":null,"cidr":null,"cidr_array":null,"float":null,"inet":null,"inet_array":null,"int":null,"int_array":null,"json":null,"macaddr":null,"macaddr_array":null,"text":null,"text_array":null,"text_array_with_null":null,"text_with_null":null,"timestamp":null,"uuid":null,"uuid_array":null} +{"_cq_id":"00000000-0000-0000-0000-000000000001","_cq_parent_id":"00000000-0000-0000-0000-000000000001","_cq_source_name":"test-source","_cq_sync_time":"2021-01-01 00:00:00","bool":true,"bytea":"AQID","cidr":"192.0.2.0/24","cidr_array":["192.0.2.0/24","192.0.2.0/24"],"float":1.1,"inet":"192.0.2.0/24","inet_array":["192.0.2.0/24","192.0.2.0/24"],"int":1,"int_array":[1,2],"json":{"test":"test"},"macaddr":"aa:bb:cc:dd:ee:ff","macaddr_array":["aa:bb:cc:dd:ee:ff","11:22:33:44:55:66"],"string_pk":"AString","text":"AString","text_array":["test1","test2"],"text_array_with_null":["test1","test2\u0000WithNull"],"text_with_null":"AStringWith\u0000NullBytes","timestamp":"2021-01-02 00:00:00","uuid":"00000000-0000-0000-0000-000000000001","uuid_array":["00000000-0000-0000-0000-000000000001","00000000-0000-0000-0000-000000000002"],"uuid_pk":"00000000-0000-0000-0000-000000000001"} +{"_cq_id":"00000000-0000-0000-0000-000000000001","_cq_parent_id":"00000000-0000-0000-0000-000000000001","_cq_source_name":"test-source","_cq_sync_time":"2021-01-01 00:00:00","bool":null,"bytea":null,"cidr":null,"cidr_array":null,"float":null,"inet":null,"inet_array":null,"int":null,"int_array":null,"json":null,"macaddr":null,"macaddr_array":null,"string_pk":"AString","text":null,"text_array":null,"text_array_with_null":null,"text_with_null":null,"timestamp":null,"uuid":null,"uuid_array":null,"uuid_pk":"00000000-0000-0000-0000-000000000001"} From a479137a7f7c4fbd2800c281e3abc6af440bff85 Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Mon, 17 Apr 2023 10:11:38 +0100 Subject: [PATCH 15/25] Remove unused function --- parquet/write.go | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/parquet/write.go b/parquet/write.go index a71a5a4c..0a711e72 100644 --- a/parquet/write.go +++ b/parquet/write.go @@ -1,14 +1,12 @@ package parquet import ( - "context" "encoding/json" "fmt" "io" "github.com/apache/arrow/go/v12/arrow" "github.com/apache/arrow/go/v12/arrow/array" - "github.com/apache/arrow/go/v12/arrow/compute" "github.com/apache/arrow/go/v12/arrow/memory" "github.com/apache/arrow/go/v12/parquet" "github.com/apache/arrow/go/v12/parquet/pqarrow" @@ -142,19 +140,3 @@ func castExtensionColsToString(mem memory.Allocator, rec arrow.Record) (arrow.Re } return rb.NewRecord(), nil } - -func castListOf(ctx context.Context, rec arrow.Record, c int, rb *array.RecordBuilder, storageType arrow.DataType) error { - arr, err := compute.CastToType(ctx, rec.Column(c), arrow.ListOf(storageType)) - if err != nil { - return fmt.Errorf("failed to cast col %v to %v: %w", rec.ColumnName(c), storageType, err) - } - b, err := arr.MarshalJSON() - if err != nil { - return fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) - } - err = rb.Field(c).UnmarshalJSON(b) - if err != nil { - return fmt.Errorf("failed to unmarshal col %v: %w", rec.ColumnName(c), err) - } - return nil -} From d4eaee234b77d79c5715609b3f1a02b96f1197ee Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Mon, 17 Apr 2023 10:23:24 +0100 Subject: [PATCH 16/25] Linting fixes --- csv/write_read_test.go | 4 +- parquet/write_read_test.go | 78 +------------------------------------- 2 files changed, 4 insertions(+), 78 deletions(-) diff --git a/csv/write_read_test.go b/csv/write_read_test.go index 7d5e3b72..828d95b6 100644 --- a/csv/write_read_test.go +++ b/csv/write_read_test.go @@ -80,10 +80,10 @@ func TestWriteRead(t *testing.T) { for got := range ch { // TODO: compare records. This is currently failing because the CSV reader fails // to read the JSON column properly. We rely on snapshot tests for now. - //if diff := destination.RecordDiff(records[totalCount], got); diff != "" { + // if diff := destination.RecordDiff(records[totalCount], got); diff != "" { // got.Release() // t.Errorf("got diff: %s", diff) - //} + // } got.Release() totalCount++ } diff --git a/parquet/write_read_test.go b/parquet/write_read_test.go index e1d581f1..b800771e 100644 --- a/parquet/write_read_test.go +++ b/parquet/write_read_test.go @@ -19,8 +19,8 @@ func TestWriteRead(t *testing.T) { arrowSchema := table.ToArrowSchema() sourceName := "test-source" syncTime := time.Now().UTC().Round(1 * time.Second) - // TODO: use checked allocator here; can't right now because there - // are memory leaks in the arrow parquet reader implementation :( + // TODO: use checked allocator here; there are some memory leaks in the parquet writer + // at the time of writing, so leaving this as a to-do // mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) // defer mem.AssertSize(t, 0) mem := memory.NewGoAllocator() @@ -79,77 +79,3 @@ func TestWriteRead(t *testing.T) { t.Fatalf("expected 1 row, got %d", totalCount) } } - -// -//func castExtensionColsToStringType(mem memory.Allocator, rec arrow.Record) (arrow.Record, error) { -// oldFields := rec.Schema().Fields() -// fields := make([]arrow.Field, len(oldFields)) -// copy(fields, oldFields) -// for i, f := range fields { -// switch { -// case f.Type.ID() == arrow.EXTENSION: -// fields[i].Type = f.Type.(arrow.ExtensionType).StorageType() -// case arrow.TypeEqual(f.Type, arrow.ListOf(types.NewUUIDType())), -// arrow.TypeEqual(f.Type, arrow.ListOf(types.NewInetType())), -// arrow.TypeEqual(f.Type, arrow.ListOf(types.NewJSONType())), -// arrow.TypeEqual(f.Type, arrow.ListOf(types.NewMacType())): -// fields[i].Type = arrow.ListOf(arrow.BinaryTypes.String) -// } -// } -// -// md := rec.Schema().Metadata() -// newSchema := arrow.NewSchema(fields, &md) -// ctx := context.Background() -// rb := array.NewRecordBuilder(mem, newSchema) -// -// defer rb.Release() -// for c := 0; c < int(rec.NumCols()); c++ { -// col := rec.Column(c) -// switch { -// case col.DataType().ID() == arrow.EXTENSION: -// storageType := col.DataType().(arrow.ExtensionType).StorageType() -// arr, err := compute.CastToType(ctx, rec.Column(c), storageType) -// if err != nil { -// return nil, fmt.Errorf("failed to cast col %v to %v: %w", rec.ColumnName(c), storageType, err) -// } -// b, err := arr.MarshalJSON() -// if err != nil { -// return nil, fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) -// } -// err = rb.Field(c).UnmarshalJSON(b) -// if err != nil { -// return nil, fmt.Errorf("failed to unmarshal col %v: %w", rec.ColumnName(c), err) -// } -// case arrow.TypeEqual(col.DataType(), arrow.ListOf(types.NewUUIDType())): -// err := castListOf(ctx, rec, c, rb, types.NewUUIDType().StorageType()) -// if err != nil { -// return nil, fmt.Errorf("failed to cast col %v: %w", rec.ColumnName(c), err) -// } -// case arrow.TypeEqual(col.DataType(), arrow.ListOf(types.NewJSONType())): -// err := castListOf(ctx, rec, c, rb, types.NewJSONType().StorageType()) -// if err != nil { -// return nil, fmt.Errorf("failed to cast col %v: %w", rec.ColumnName(c), err) -// } -// case arrow.TypeEqual(col.DataType(), arrow.ListOf(types.NewInetType())): -// err := castListOf(ctx, rec, c, rb, types.NewInetType().StorageType()) -// if err != nil { -// return nil, fmt.Errorf("failed to cast col %v: %w", rec.ColumnName(c), err) -// } -// case arrow.TypeEqual(col.DataType(), arrow.ListOf(types.NewMacType())): -// err := castListOf(ctx, rec, c, rb, types.NewMacType().StorageType()) -// if err != nil { -// return nil, fmt.Errorf("failed to cast col %v: %w", rec.ColumnName(c), err) -// } -// default: -// b, err := rec.Column(c).MarshalJSON() -// if err != nil { -// return nil, fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) -// } -// err = rb.Field(c).UnmarshalJSON(b) -// if err != nil { -// return nil, fmt.Errorf("failed to unmarshal col %v: %w", rec.ColumnName(c), err) -// } -// } -// } -// return rb.NewRecord(), nil -//} From a73a7987dd552c2cd7469c47a5f4c8382e76ed8e Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Mon, 17 Apr 2023 17:13:57 +0100 Subject: [PATCH 17/25] Fixes for CSV and parquet --- csv/read.go | 6 ++- csv/write.go | 6 ++- csv/write_read_test.go | 9 ++-- go.mod | 7 ++- go.sum | 6 +-- parquet/read.go | 99 ++++++++++++++++++++++++++++++++++++-- parquet/write.go | 2 +- parquet/write_read_test.go | 6 +-- 8 files changed, 120 insertions(+), 21 deletions(-) diff --git a/csv/read.go b/csv/read.go index fdf4c2ed..ba8d92ee 100644 --- a/csv/read.go +++ b/csv/read.go @@ -8,7 +8,11 @@ import ( ) func (cl *Client) Read(r io.Reader, arrowSchema *arrow.Schema, _ string, res chan<- arrow.Record) error { - reader := csv.NewReader(r, arrowSchema, csv.WithComma(cl.Delimiter), csv.WithHeader(cl.IncludeHeaders), csv.WithNullReader(true, "")) + reader := csv.NewReader(r, arrowSchema, + csv.WithComma(cl.Delimiter), + csv.WithHeader(cl.IncludeHeaders), + csv.WithNullReader(true, ""), + ) for reader.Next() { if reader.Err() != nil { return reader.Err() diff --git a/csv/write.go b/csv/write.go index dd0d25db..39708fa2 100644 --- a/csv/write.go +++ b/csv/write.go @@ -10,7 +10,11 @@ import ( ) func (cl *Client) WriteTableBatch(w io.Writer, arrowSchema *arrow.Schema, records []arrow.Record) error { - writer := csv.NewWriter(w, arrowSchema, csv.WithComma(cl.Delimiter), csv.WithHeader(cl.IncludeHeaders), csv.WithNullWriter("")) + writer := csv.NewWriter(w, arrowSchema, + csv.WithComma(cl.Delimiter), + csv.WithHeader(cl.IncludeHeaders), + csv.WithNullWriter(""), + ) for _, record := range records { if err := writer.Write(record); err != nil { return fmt.Errorf("failed to write record to csv: %w", err) diff --git a/csv/write_read_test.go b/csv/write_read_test.go index 828d95b6..619364eb 100644 --- a/csv/write_read_test.go +++ b/csv/write_read_test.go @@ -10,6 +10,7 @@ import ( "github.com/apache/arrow/go/v12/arrow" "github.com/apache/arrow/go/v12/arrow/memory" "github.com/bradleyjkemp/cupaloy/v2" + "github.com/cloudquery/plugin-sdk/v2/plugins/destination" "github.com/cloudquery/plugin-sdk/v2/testdata" "github.com/google/uuid" ) @@ -80,10 +81,10 @@ func TestWriteRead(t *testing.T) { for got := range ch { // TODO: compare records. This is currently failing because the CSV reader fails // to read the JSON column properly. We rely on snapshot tests for now. - // if diff := destination.RecordDiff(records[totalCount], got); diff != "" { - // got.Release() - // t.Errorf("got diff: %s", diff) - // } + if diff := destination.RecordDiff(records[totalCount], got); diff != "" { + got.Release() + t.Errorf("got diff: %s", diff) + } got.Release() totalCount++ } diff --git a/go.mod b/go.mod index 7edd7047..0ea6fbdd 100644 --- a/go.mod +++ b/go.mod @@ -24,11 +24,14 @@ require ( ) require ( - github.com/apache/arrow/go/v12 v12.0.0-20230401212133-adf33cc43010 + github.com/apache/arrow/go/v12 v12.0.0-20230417014917-9888ac36c142 github.com/bradleyjkemp/cupaloy/v2 v2.8.0 ) -replace github.com/apache/arrow/go/v12 => github.com/cloudquery/arrow/go/v12 v12.0.0-20230417085404-0d8ebd3b01ce +replace github.com/apache/arrow/go/v12 => github.com/cloudquery/arrow/go/v12 v12.0.0-20230417154311-f9add0212acd + +//replace github.com/apache/arrow/go/v12 => ../arrow/go +replace github.com/cloudquery/plugin-sdk/v2 v2.2.0 => ../plugin-sdk require golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect diff --git a/go.sum b/go.sum index 65a62786..6a1c7fdf 100644 --- a/go.sum +++ b/go.sum @@ -6,10 +6,8 @@ github.com/apache/thrift v0.18.1 h1:lNhK/1nqjbwbiOPDBPFJVKxgDEGSepKuTh6OLiXW8kg= github.com/apache/thrift v0.18.1/go.mod h1:rdQn/dCcDKEWjjylUeueum4vQEjG2v8v2PqriUnbr+I= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= -github.com/cloudquery/arrow/go/v12 v12.0.0-20230417085404-0d8ebd3b01ce h1:+wD+TQofOeFl9u90sY/0N9gkTUiJsPdp6SuhNv2ENnw= -github.com/cloudquery/arrow/go/v12 v12.0.0-20230417085404-0d8ebd3b01ce/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg= -github.com/cloudquery/plugin-sdk/v2 v2.2.0 h1:N5t/1M+CVGGBepJeNGbXH0jABN1S8qbvOo6fjz8P91M= -github.com/cloudquery/plugin-sdk/v2 v2.2.0/go.mod h1:pN05LsLG3WHRwri/IJBsYA+qMajUM7tRubjMWB2chvY= +github.com/cloudquery/arrow/go/v12 v12.0.0-20230417154311-f9add0212acd h1:G093N165IqQvq84MK3Ozi7QwfAWyfYywXijkOyxGJdI= +github.com/cloudquery/arrow/go/v12 v12.0.0-20230417154311-f9add0212acd/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg= github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/parquet/read.go b/parquet/read.go index 6d96dba3..513c714e 100644 --- a/parquet/read.go +++ b/parquet/read.go @@ -2,13 +2,16 @@ package parquet import ( "context" + "encoding/json" "fmt" "io" "github.com/apache/arrow/go/v12/arrow" + "github.com/apache/arrow/go/v12/arrow/array" "github.com/apache/arrow/go/v12/arrow/memory" "github.com/apache/arrow/go/v12/parquet/file" "github.com/apache/arrow/go/v12/parquet/pqarrow" + "github.com/cloudquery/plugin-sdk/v2/types" ) type ReaderAtSeeker interface { @@ -17,7 +20,7 @@ type ReaderAtSeeker interface { io.Seeker } -func (*Client) Read(f ReaderAtSeeker, _ *arrow.Schema, _ string, res chan<- arrow.Record) error { +func (*Client) Read(f ReaderAtSeeker, arrowSchema *arrow.Schema, _ string, res chan<- arrow.Record) error { mem := memory.DefaultAllocator ctx := context.Background() rdr, err := file.NewParquetReader(f) @@ -38,8 +41,12 @@ func (*Client) Read(f ReaderAtSeeker, _ *arrow.Schema, _ string, res chan<- arro } for rr.Next() { rec := rr.Record() - rec.Retain() - res <- rec + castRec, err := castStringsToExtensions(mem, rec, arrowSchema) + if err != nil { + return fmt.Errorf("failed to cast extension types: %w", err) + } + castRec.Retain() + res <- castRec _, err = rr.Read() if err == io.EOF { break @@ -51,3 +58,89 @@ func (*Client) Read(f ReaderAtSeeker, _ *arrow.Schema, _ string, res chan<- arro return nil } + +func castStringsToExtensions(mem memory.Allocator, rec arrow.Record, arrowSchema *arrow.Schema) (arrow.Record, error) { + rb := array.NewRecordBuilder(mem, arrowSchema) + + defer rb.Release() + for c := 0; c < int(rec.NumCols()); c++ { + col := rec.Column(c) + switch { + case arrow.TypeEqual(arrowSchema.Field(c).Type, types.NewUUIDType()): + arr := col.(*array.String) + b, err := arr.MarshalJSON() + if err != nil { + return nil, fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) + } + err = rb.Field(c).UnmarshalJSON(b) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal col %v: %w", rec.ColumnName(c), err) + } + case arrow.TypeEqual(arrowSchema.Field(c).Type, types.NewInetType()): + arr := col.(*array.String) + b, err := arr.MarshalJSON() + if err != nil { + return nil, fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) + } + err = rb.Field(c).UnmarshalJSON(b) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal col %v: %w", rec.ColumnName(c), err) + } + case arrow.TypeEqual(arrowSchema.Field(c).Type, types.NewJSONType()): + arr := col.(*array.String) + b, err := arr.MarshalJSON() + if err != nil { + return nil, fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) + } + a := make([]any, arr.Len()) + err = json.Unmarshal(b, &a) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal col %v: %w", rec.ColumnName(c), err) + } + for _, v := range a { + if v == nil { + rb.Field(c).(*types.JSONBuilder).AppendNull() + continue + } + var v2 any + err = json.Unmarshal([]byte(v.(string)), &v2) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal col %v: %w", rec.ColumnName(c), err) + } + rb.Field(c).(*types.JSONBuilder).Append(v2) + } + case arrow.TypeEqual(arrowSchema.Field(c).Type, types.NewMacType()): + arr := col.(*array.String) + b, err := arr.MarshalJSON() + if err != nil { + return nil, fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) + } + err = rb.Field(c).UnmarshalJSON(b) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal col %v: %w", rec.ColumnName(c), err) + } + case arrow.TypeEqual(arrowSchema.Field(c).Type, arrow.ListOf(types.NewUUIDType())), + arrow.TypeEqual(arrowSchema.Field(c).Type, arrow.ListOf(types.NewInetType())), + arrow.TypeEqual(arrowSchema.Field(c).Type, arrow.ListOf(types.NewMacType())): + arr := col.(*array.List) + b, err := arr.MarshalJSON() + if err != nil { + return nil, fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) + } + err = rb.Field(c).UnmarshalJSON(b) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal col %v: %w", rec.ColumnName(c), err) + } + default: + b, err := rec.Column(c).MarshalJSON() + if err != nil { + return nil, fmt.Errorf("failed to marshal col %v: %w", rec.ColumnName(c), err) + } + err = rb.Field(c).UnmarshalJSON(b) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal col %v: %w", rec.ColumnName(c), err) + } + } + } + return rb.NewRecord(), nil +} diff --git a/parquet/write.go b/parquet/write.go index 0a711e72..caa780a9 100644 --- a/parquet/write.go +++ b/parquet/write.go @@ -97,6 +97,7 @@ func castExtensionColsToString(mem memory.Allocator, rec arrow.Record) (arrow.Re for _, v := range a { if v == nil { rb.Field(c).(*array.StringBuilder).AppendNull() + continue } b, err := json.Marshal(v) if err != nil { @@ -115,7 +116,6 @@ func castExtensionColsToString(mem memory.Allocator, rec arrow.Record) (arrow.Re return nil, fmt.Errorf("failed to unmarshal col %v: %w", rec.ColumnName(c), err) } case arrow.TypeEqual(col.DataType(), arrow.ListOf(types.NewUUIDType())), - arrow.TypeEqual(col.DataType(), arrow.ListOf(types.NewJSONType())), arrow.TypeEqual(col.DataType(), arrow.ListOf(types.NewInetType())), arrow.TypeEqual(col.DataType(), arrow.ListOf(types.NewMacType())): arr := col.(*array.List) diff --git a/parquet/write_read_test.go b/parquet/write_read_test.go index b800771e..faa281c4 100644 --- a/parquet/write_read_test.go +++ b/parquet/write_read_test.go @@ -63,11 +63,7 @@ func TestWriteRead(t *testing.T) { }() totalCount := 0 for got := range ch { - baseRecord, err := castExtensionColsToString(mem, records[totalCount]) - if err != nil { - t.Fatalf("failed to cast extensions to storage type for comparison: %v", err) - } - if diff := destination.RecordDiff(baseRecord, got); diff != "" { + if diff := destination.RecordDiff(records[totalCount], got); diff != "" { t.Fatalf("got diff: %s", diff) } totalCount++ From 7fb5a0c454a979a1d2aeec8710c88642a822a359 Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Tue, 18 Apr 2023 10:06:16 +0100 Subject: [PATCH 18/25] Address review comments --- .gitattributes | 1 + client.go | 6 +++--- .../TestWriteRead-default.csv | Bin .../TestWriteRead-with_delimiter.csv | Bin .../TestWriteRead-with_delimiter_headers.csv | Bin .../TestWriteRead-with_headers.csv | Bin csv/write_read_test.go | 7 ++++--- go.mod | 3 +-- json/{.snapshots => testdata}/TestWriteRead.jsonl | 0 json/write_read_test.go | 5 ++++- spec.go | 6 +++--- spec_test.go | 4 ++-- 12 files changed, 18 insertions(+), 14 deletions(-) create mode 100644 .gitattributes rename csv/{.snapshots => testdata}/TestWriteRead-default.csv (100%) rename csv/{.snapshots => testdata}/TestWriteRead-with_delimiter.csv (100%) rename csv/{.snapshots => testdata}/TestWriteRead-with_delimiter_headers.csv (100%) rename csv/{.snapshots => testdata}/TestWriteRead-with_headers.csv (100%) rename json/{.snapshots => testdata}/TestWriteRead.jsonl (100%) diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..449d7b19 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.csv diff \ No newline at end of file diff --git a/client.go b/client.go index 360e33b7..866202bb 100644 --- a/client.go +++ b/client.go @@ -1,9 +1,9 @@ package filetypes import ( - csvFile "github.com/cloudquery/filetypes/csv" - jsonFile "github.com/cloudquery/filetypes/json" - "github.com/cloudquery/filetypes/parquet" + csvFile "github.com/cloudquery/filetypes/v2/csv" + jsonFile "github.com/cloudquery/filetypes/v2/json" + "github.com/cloudquery/filetypes/v2/parquet" ) type Client struct { diff --git a/csv/.snapshots/TestWriteRead-default.csv b/csv/testdata/TestWriteRead-default.csv similarity index 100% rename from csv/.snapshots/TestWriteRead-default.csv rename to csv/testdata/TestWriteRead-default.csv diff --git a/csv/.snapshots/TestWriteRead-with_delimiter.csv b/csv/testdata/TestWriteRead-with_delimiter.csv similarity index 100% rename from csv/.snapshots/TestWriteRead-with_delimiter.csv rename to csv/testdata/TestWriteRead-with_delimiter.csv diff --git a/csv/.snapshots/TestWriteRead-with_delimiter_headers.csv b/csv/testdata/TestWriteRead-with_delimiter_headers.csv similarity index 100% rename from csv/.snapshots/TestWriteRead-with_delimiter_headers.csv rename to csv/testdata/TestWriteRead-with_delimiter_headers.csv diff --git a/csv/.snapshots/TestWriteRead-with_headers.csv b/csv/testdata/TestWriteRead-with_headers.csv similarity index 100% rename from csv/.snapshots/TestWriteRead-with_headers.csv rename to csv/testdata/TestWriteRead-with_headers.csv diff --git a/csv/write_read_test.go b/csv/write_read_test.go index 619364eb..da64c50a 100644 --- a/csv/write_read_test.go +++ b/csv/write_read_test.go @@ -66,7 +66,10 @@ func TestWriteRead(t *testing.T) { if err != nil { t.Fatal(err) } - snap := cupaloy.New(cupaloy.SnapshotFileExtension(".csv")) + snap := cupaloy.New( + cupaloy.SnapshotFileExtension(".csv"), + cupaloy.SnapshotSubdirectory("testdata"), + ) snap.SnapshotT(t, string(rawBytes)) byteReader := bytes.NewReader(rawBytes) @@ -79,8 +82,6 @@ func TestWriteRead(t *testing.T) { }() totalCount := 0 for got := range ch { - // TODO: compare records. This is currently failing because the CSV reader fails - // to read the JSON column properly. We rely on snapshot tests for now. if diff := destination.RecordDiff(records[totalCount], got); diff != "" { got.Release() t.Errorf("got diff: %s", diff) diff --git a/go.mod b/go.mod index 0ea6fbdd..28185ed4 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/cloudquery/filetypes +module github.com/cloudquery/filetypes/v2 go 1.19 @@ -30,7 +30,6 @@ require ( replace github.com/apache/arrow/go/v12 => github.com/cloudquery/arrow/go/v12 v12.0.0-20230417154311-f9add0212acd -//replace github.com/apache/arrow/go/v12 => ../arrow/go replace github.com/cloudquery/plugin-sdk/v2 v2.2.0 => ../plugin-sdk require golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect diff --git a/json/.snapshots/TestWriteRead.jsonl b/json/testdata/TestWriteRead.jsonl similarity index 100% rename from json/.snapshots/TestWriteRead.jsonl rename to json/testdata/TestWriteRead.jsonl diff --git a/json/write_read_test.go b/json/write_read_test.go index 08b492ac..461a45a3 100644 --- a/json/write_read_test.go +++ b/json/write_read_test.go @@ -82,7 +82,10 @@ func TestWriteRead(t *testing.T) { if err != nil { t.Fatal(err) } - snap := cupaloy.New(cupaloy.SnapshotFileExtension(".jsonl")) + snap := cupaloy.New( + cupaloy.SnapshotFileExtension(".jsonl"), + cupaloy.SnapshotSubdirectory("testdata"), + ) snap.SnapshotT(t, string(rawBytes)) byteReader := bytes.NewReader(rawBytes) diff --git a/spec.go b/spec.go index 97f6fd75..6343219f 100644 --- a/spec.go +++ b/spec.go @@ -5,9 +5,9 @@ import ( "encoding/json" "fmt" - "github.com/cloudquery/filetypes/csv" - jsonFile "github.com/cloudquery/filetypes/json" - "github.com/cloudquery/filetypes/parquet" + "github.com/cloudquery/filetypes/v2/csv" + jsonFile "github.com/cloudquery/filetypes/v2/json" + "github.com/cloudquery/filetypes/v2/parquet" ) type FormatType string diff --git a/spec_test.go b/spec_test.go index aaec2e5e..54eebcdc 100644 --- a/spec_test.go +++ b/spec_test.go @@ -3,8 +3,8 @@ package filetypes import ( "testing" - "github.com/cloudquery/filetypes/csv" - "github.com/cloudquery/filetypes/json" + "github.com/cloudquery/filetypes/v2/csv" + "github.com/cloudquery/filetypes/v2/json" "github.com/stretchr/testify/assert" ) From 95f527f3834f973eb741052a95f3bc8654e8ff8f Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Tue, 18 Apr 2023 10:35:58 +0100 Subject: [PATCH 19/25] Update dependency --- go.mod | 4 +--- go.sum | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 28185ed4..0a71be0d 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/cloudquery/filetypes/v2 go 1.19 require ( - github.com/cloudquery/plugin-sdk/v2 v2.2.0 + github.com/cloudquery/plugin-sdk/v2 v2.3.0 github.com/davecgh/go-spew v1.1.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/stretchr/testify v1.8.2 @@ -30,8 +30,6 @@ require ( replace github.com/apache/arrow/go/v12 => github.com/cloudquery/arrow/go/v12 v12.0.0-20230417154311-f9add0212acd -replace github.com/cloudquery/plugin-sdk/v2 v2.2.0 => ../plugin-sdk - require golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect require ( diff --git a/go.sum b/go.sum index 6a1c7fdf..82c9fe35 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,8 @@ github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oM github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= github.com/cloudquery/arrow/go/v12 v12.0.0-20230417154311-f9add0212acd h1:G093N165IqQvq84MK3Ozi7QwfAWyfYywXijkOyxGJdI= github.com/cloudquery/arrow/go/v12 v12.0.0-20230417154311-f9add0212acd/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg= +github.com/cloudquery/plugin-sdk/v2 v2.3.0 h1:+xZdmo8ESO2G61ji1jQif4q9EAIN8sMwOmtFJMiyytw= +github.com/cloudquery/plugin-sdk/v2 v2.3.0/go.mod h1:/wAbhyQbdIUAMEL+Yo9zkgoBls83xt3ev6jLpJblIoU= github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= From 20d781d459377ed5f7bd77916668e85e3c146702 Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Tue, 18 Apr 2023 10:38:13 +0100 Subject: [PATCH 20/25] Update gitattributes --- .gitattributes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitattributes b/.gitattributes index 449d7b19..6b414c3d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1 @@ -*.csv diff \ No newline at end of file +**/*.csv diff \ No newline at end of file From ae72cca21751c40cb42ccb5d211637d78fc8c084 Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Tue, 18 Apr 2023 10:43:06 +0100 Subject: [PATCH 21/25] try linguist-language --- .gitattributes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitattributes b/.gitattributes index 6b414c3d..cb5912b4 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1 @@ -**/*.csv diff \ No newline at end of file +**/*.csv linguist-language=csv \ No newline at end of file From 60e369dbf1a56eeec97838878e639df85b62903f Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Tue, 18 Apr 2023 10:44:06 +0100 Subject: [PATCH 22/25] One last try --- .gitattributes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitattributes b/.gitattributes index cb5912b4..4a6252f0 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1 @@ -**/*.csv linguist-language=csv \ No newline at end of file +*.csv linguist-language=csv \ No newline at end of file From 23a6a0d533353146baa22f26655441e8243d6958 Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Tue, 18 Apr 2023 10:45:45 +0100 Subject: [PATCH 23/25] Remove gitattributes --- .gitattributes | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index 4a6252f0..00000000 --- a/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -*.csv linguist-language=csv \ No newline at end of file From 31a6979d6f5dbe8782fa0a3c16681f3c9ba006d9 Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Tue, 18 Apr 2023 11:04:04 +0100 Subject: [PATCH 24/25] Release records, use checked allocator --- parquet/write_read_test.go | 7 ++----- write.go | 14 ++++++++++---- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/parquet/write_read_test.go b/parquet/write_read_test.go index faa281c4..3a7d76cf 100644 --- a/parquet/write_read_test.go +++ b/parquet/write_read_test.go @@ -19,11 +19,8 @@ func TestWriteRead(t *testing.T) { arrowSchema := table.ToArrowSchema() sourceName := "test-source" syncTime := time.Now().UTC().Round(1 * time.Second) - // TODO: use checked allocator here; there are some memory leaks in the parquet writer - // at the time of writing, so leaving this as a to-do - // mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) - // defer mem.AssertSize(t, 0) - mem := memory.NewGoAllocator() + mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + defer mem.AssertSize(t, 0) opts := testdata.GenTestDataOptions{ SourceName: sourceName, SyncTime: syncTime, diff --git a/write.go b/write.go index d072bd91..03af2bac 100644 --- a/write.go +++ b/write.go @@ -6,18 +6,24 @@ import ( "github.com/apache/arrow/go/v12/arrow" ) -func (cl *Client) WriteTableBatchFile(w io.Writer, arrowSchema *arrow.Schema, resources []arrow.Record) error { +func (cl *Client) WriteTableBatchFile(w io.Writer, arrowSchema *arrow.Schema, records []arrow.Record) error { + defer func() { + for _, r := range records { + r.Release() + } + }() + switch cl.spec.Format { case FormatTypeCSV: - if err := cl.csv.WriteTableBatch(w, arrowSchema, resources); err != nil { + if err := cl.csv.WriteTableBatch(w, arrowSchema, records); err != nil { return err } case FormatTypeJSON: - if err := cl.json.WriteTableBatch(w, arrowSchema, resources); err != nil { + if err := cl.json.WriteTableBatch(w, arrowSchema, records); err != nil { return err } case FormatTypeParquet: - if err := cl.parquet.WriteTableBatch(w, arrowSchema, resources); err != nil { + if err := cl.parquet.WriteTableBatch(w, arrowSchema, records); err != nil { return err } default: From 1f115032288438a20ce96020f0ec8672aecd42a6 Mon Sep 17 00:00:00 2001 From: Herman Schaaf Date: Tue, 18 Apr 2023 12:07:08 +0100 Subject: [PATCH 25/25] Use interface --- read.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/read.go b/read.go index 6c2796cf..1a3bc9e9 100644 --- a/read.go +++ b/read.go @@ -1,12 +1,18 @@ package filetypes import ( - "os" + "io" "github.com/apache/arrow/go/v12/arrow" ) -func (cl *Client) Read(f *os.File, sc *arrow.Schema, sourceName string, res chan<- arrow.Record) error { +type ReaderAtSeeker interface { + io.Reader + io.ReaderAt + io.Seeker +} + +func (cl *Client) Read(f ReaderAtSeeker, sc *arrow.Schema, sourceName string, res chan<- arrow.Record) error { switch cl.spec.Format { case FormatTypeCSV: if err := cl.csv.Read(f, sc, sourceName, res); err != nil {