From b479909cb0efb079f503823478f9154d267aff37 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Mon, 19 Jun 2023 13:02:08 +0100 Subject: [PATCH 01/16] feat: Upgrade to plugin-sdk v4 --- client.go | 8 ++++---- csv/read.go | 2 +- csv/testdata/TestWriteRead-default.csv | 4 ++-- csv/testdata/TestWriteRead-with_delimiter.csv | 4 ++-- csv/testdata/TestWriteRead-with_delimiter_headers.csv | 6 +++--- csv/testdata/TestWriteRead-with_headers.csv | 6 +++--- csv/write.go | 4 ++-- csv/write_read_test.go | 8 ++++---- go.mod | 11 +++++------ go.sum | 10 ++-------- json/read.go | 2 +- json/testdata/TestWriteRead.jsonl | 4 ++-- json/write.go | 4 ++-- json/write_read_test.go | 8 ++++---- parquet/read.go | 2 +- parquet/write.go | 6 +++--- parquet/write_read_test.go | 8 ++++---- read.go | 2 +- spec.go | 6 +++--- spec_test.go | 4 ++-- types/types.go | 2 +- write.go | 4 ++-- 22 files changed, 54 insertions(+), 61 deletions(-) diff --git a/client.go b/client.go index ad828750..2b075b20 100644 --- a/client.go +++ b/client.go @@ -1,10 +1,10 @@ package filetypes import ( - csvFile "github.com/cloudquery/filetypes/v3/csv" - jsonFile "github.com/cloudquery/filetypes/v3/json" - "github.com/cloudquery/filetypes/v3/parquet" - "github.com/cloudquery/filetypes/v3/types" + csvFile "github.com/cloudquery/filetypes/v4/csv" + jsonFile "github.com/cloudquery/filetypes/v4/json" + "github.com/cloudquery/filetypes/v4/parquet" + "github.com/cloudquery/filetypes/v4/types" ) type Client struct { diff --git a/csv/read.go b/csv/read.go index 0aaec2d3..a9534204 100644 --- a/csv/read.go +++ b/csv/read.go @@ -8,7 +8,7 @@ import ( "github.com/apache/arrow/go/v13/arrow/array" "github.com/apache/arrow/go/v13/arrow/csv" "github.com/apache/arrow/go/v13/arrow/memory" - "github.com/cloudquery/plugin-sdk/v3/schema" + "github.com/cloudquery/plugin-sdk/v4/schema" ) func (cl *Client) Read(r io.Reader, table *schema.Table, _ string, res chan<- arrow.Record) error { diff --git a/csv/testdata/TestWriteRead-default.csv b/csv/testdata/TestWriteRead-default.csv index abfb1d8c..c77a27b4 100644 --- a/csv/testdata/TestWriteRead-default.csv +++ b/csv/testdata/TestWriteRead-default.csv @@ -1,3 +1,3 @@ -test-source,2021-01-01 00:00:00,00000000-0000-0000-0000-000000000001,00000000-0000-0000-0000-000000000001,7049Ug==,true,2023-07-16,2023-07-16,"{""days"":6783,""milliseconds"":6783}",6476783ms,6476783ns,6476783s,6476783us,783.936,783.936,192.0.2.0/24,-30727,-1781555540,-5474557666971701248,-123,7049Ug==,AString76783,aa:bb:cc:dd:ee:ff,"{""months"":6783,""days"":6783,""nanoseconds"":6783}",1,AString76783,00:00:00.000,00:00:00,00:00:00.000000000,00:00:00.000000,2021-01-02 00:00:00,2021-01-02 00:00:00,2021-01-02 00:00:00,2021-01-02 00:00:00,63063,506910753,14697929703826477056,78,00000000-0000-0000-0000-000000000001,826476783.5364813824,"{""test"":[""a"",""b"",3]}","[{""test"":""test""},123,{""test_number"":456}]","[true,null,true]","[""2023-07-16"",null,""2023-07-16""]","[""2023-07-16"",null,""2023-07-16""]","[{""days"":6783,""milliseconds"":6783},null,{""days"":6783,""milliseconds"":6783}]","[""6476783ms"",null,""6476783ms""]","[""6476783ns"",null,""6476783ns""]","[""6476783s"",null,""6476783s""]","[""6476783us"",null,""6476783us""]","[783.936,null,783.936]","[783.936,null,783.936]","[""192.0.2.0/24"",null,""192.0.2.0/24""]","[-30727,null,-30727]","[-1781555540,null,-1781555540]","[-5474557666971701248,null,-5474557666971701248]","[-123,null,-123]","[""AString76783"",null,""AString76783""]","[""aa:bb:cc:dd:ee:ff"",null,""aa:bb:cc:dd:ee:ff""]","[{""months"":6783,""days"":6783,""nanoseconds"":6783},null,{""months"":6783,""days"":6783,""nanoseconds"":6783}]","[{""months"":1},null,{""months"":1}]","[""AString76783"",null,""AString76783""]","[""00:00:00"",null,""00:00:00""]","[""00:00:00"",null,""00:00:00""]","[""00:00:00"",null,""00:00:00""]","[""00:00:00"",null,""00:00:00""]","[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]","[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]","[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]","[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]","[63063,null,63063]","[506910753,null,506910753]","[14697929703826477056,null,14697929703826477056]","[78,null,78]","[""00000000-0000-0000-0000-000000000001"",null,""00000000-0000-0000-0000-000000000001""]","[""826476783.5364813824"",null,""826476783.5364813824""]","[{""key"":""AString76783"",""value"":true},{""key"":""AString89351"",""value"":true}]","[{""key"":-5474557666971701248,""value"":true},{""key"":-3812638179211989504,""value"":true}]","[{""key"":""AString76783"",""value"":""2023-07-16""},{""key"":""AString89351"",""value"":""2023-06-14""}]","[{""key"":-5474557666971701248,""value"":""2023-07-16""},{""key"":-3812638179211989504,""value"":""2023-06-14""}]","[{""key"":""AString76783"",""value"":""2023-07-16""},{""key"":""AString89351"",""value"":""2023-06-14""}]","[{""key"":-5474557666971701248,""value"":""2023-07-16""},{""key"":-3812638179211989504,""value"":""2023-06-14""}]","[{""key"":""AString76783"",""value"":{""days"":6783,""milliseconds"":6783}},{""key"":""AString89351"",""value"":{""days"":9351,""milliseconds"":9351}}]","[{""key"":-5474557666971701248,""value"":{""days"":6783,""milliseconds"":6783}},{""key"":-3812638179211989504,""value"":{""days"":9351,""milliseconds"":9351}}]","[{""key"":""AString76783"",""value"":""6476783ms""},{""key"":""AString89351"",""value"":""1989351ms""}]","[{""key"":-5474557666971701248,""value"":""6476783ms""},{""key"":-3812638179211989504,""value"":""1989351ms""}]","[{""key"":""AString76783"",""value"":""6476783ns""},{""key"":""AString89351"",""value"":""1989351ns""}]","[{""key"":-5474557666971701248,""value"":""6476783ns""},{""key"":-3812638179211989504,""value"":""1989351ns""}]","[{""key"":""AString76783"",""value"":""6476783s""},{""key"":""AString89351"",""value"":""1989351s""}]","[{""key"":-5474557666971701248,""value"":""6476783s""},{""key"":-3812638179211989504,""value"":""1989351s""}]","[{""key"":""AString76783"",""value"":""6476783us""},{""key"":""AString89351"",""value"":""1989351us""}]","[{""key"":-5474557666971701248,""value"":""6476783us""},{""key"":-3812638179211989504,""value"":""1989351us""}]","[{""key"":""AString76783"",""value"":783.936},{""key"":""AString89351"",""value"":351.121}]","[{""key"":-5474557666971701248,""value"":783.936},{""key"":-3812638179211989504,""value"":351.121}]","[{""key"":""AString76783"",""value"":783.936},{""key"":""AString89351"",""value"":351.121}]","[{""key"":-5474557666971701248,""value"":783.936},{""key"":-3812638179211989504,""value"":351.121}]","[{""key"":""AString76783"",""value"":""192.0.2.0/24""},{""key"":""AString89351"",""value"":""192.0.2.0/24""}]","[{""key"":-5474557666971701248,""value"":""192.0.2.0/24""},{""key"":-3812638179211989504,""value"":""192.0.2.0/24""}]","[{""key"":""AString76783"",""value"":-30727},{""key"":""AString89351"",""value"":-25508}]","[{""key"":-5474557666971701248,""value"":-30727},{""key"":-3812638179211989504,""value"":-25508}]","[{""key"":""AString76783"",""value"":-1781555540},{""key"":""AString89351"",""value"":-853032410}]","[{""key"":-5474557666971701248,""value"":-1781555540},{""key"":-3812638179211989504,""value"":-853032410}]","[{""key"":""AString76783"",""value"":-5474557666971701248},{""key"":""AString89351"",""value"":-3812638179211989504}]","[{""key"":-5474557666971701248,""value"":-5474557666971701248},{""key"":-3812638179211989504,""value"":-3812638179211989504}]","[{""key"":""AString76783"",""value"":-123},{""key"":""AString89351"",""value"":-60}]","[{""key"":-5474557666971701248,""value"":-123},{""key"":-3812638179211989504,""value"":-60}]","[{""key"":""AString76783"",""value"":""AString76783""},{""key"":""AString89351"",""value"":""AString89351""}]","[{""key"":-5474557666971701248,""value"":""AString76783""},{""key"":-3812638179211989504,""value"":""AString89351""}]","[{""key"":""AString76783"",""value"":""aa:bb:cc:dd:ee:ff""},{""key"":""AString89351"",""value"":""aa:bb:cc:dd:ee:ff""}]","[{""key"":-5474557666971701248,""value"":""aa:bb:cc:dd:ee:ff""},{""key"":-3812638179211989504,""value"":""aa:bb:cc:dd:ee:ff""}]","[{""key"":""AString76783"",""value"":{""months"":6783,""days"":6783,""nanoseconds"":6783}},{""key"":""AString89351"",""value"":{""months"":9351,""days"":9351,""nanoseconds"":9351}}]","[{""key"":-5474557666971701248,""value"":{""months"":6783,""days"":6783,""nanoseconds"":6783}},{""key"":-3812638179211989504,""value"":{""months"":9351,""days"":9351,""nanoseconds"":9351}}]","[{""key"":""AString76783"",""value"":{""months"":1}},{""key"":""AString89351"",""value"":{""months"":1}}]","[{""key"":-5474557666971701248,""value"":{""months"":1}},{""key"":-3812638179211989504,""value"":{""months"":1}}]","[{""key"":""AString76783"",""value"":""AString76783""},{""key"":""AString89351"",""value"":""AString89351""}]","[{""key"":-5474557666971701248,""value"":""AString76783""},{""key"":-3812638179211989504,""value"":""AString89351""}]","[{""key"":""AString76783"",""value"":""00:00:00""},{""key"":""AString89351"",""value"":""00:00:00""}]","[{""key"":-5474557666971701248,""value"":""00:00:00""},{""key"":-3812638179211989504,""value"":""00:00:00""}]","[{""key"":""AString76783"",""value"":""00:00:00""},{""key"":""AString89351"",""value"":""00:00:00""}]","[{""key"":-5474557666971701248,""value"":""00:00:00""},{""key"":-3812638179211989504,""value"":""00:00:00""}]","[{""key"":""AString76783"",""value"":""00:00:00""},{""key"":""AString89351"",""value"":""00:00:00""}]","[{""key"":-5474557666971701248,""value"":""00:00:00""},{""key"":-3812638179211989504,""value"":""00:00:00""}]","[{""key"":""AString76783"",""value"":""00:00:00""},{""key"":""AString89351"",""value"":""00:00:00""}]","[{""key"":-5474557666971701248,""value"":""00:00:00""},{""key"":-3812638179211989504,""value"":""00:00:00""}]","[{""key"":""AString76783"",""value"":""2021-01-02 00:00:00""},{""key"":""AString89351"",""value"":""2021-01-02 00:00:00""}]","[{""key"":-5474557666971701248,""value"":""2021-01-02 00:00:00""},{""key"":-3812638179211989504,""value"":""2021-01-02 00:00:00""}]","[{""key"":""AString76783"",""value"":""2021-01-02 00:00:00""},{""key"":""AString89351"",""value"":""2021-01-02 00:00:00""}]","[{""key"":-5474557666971701248,""value"":""2021-01-02 00:00:00""},{""key"":-3812638179211989504,""value"":""2021-01-02 00:00:00""}]","[{""key"":""AString76783"",""value"":""2021-01-02 00:00:00""},{""key"":""AString89351"",""value"":""2021-01-02 00:00:00""}]","[{""key"":-5474557666971701248,""value"":""2021-01-02 00:00:00""},{""key"":-3812638179211989504,""value"":""2021-01-02 00:00:00""}]","[{""key"":""AString76783"",""value"":""2021-01-02 00:00:00""},{""key"":""AString89351"",""value"":""2021-01-02 00:00:00""}]","[{""key"":-5474557666971701248,""value"":""2021-01-02 00:00:00""},{""key"":-3812638179211989504,""value"":""2021-01-02 00:00:00""}]","[{""key"":""AString76783"",""value"":63063},{""key"":""AString89351"",""value"":34191}]","[{""key"":-5474557666971701248,""value"":63063},{""key"":-3812638179211989504,""value"":34191}]","[{""key"":""AString76783"",""value"":506910753},{""key"":""AString89351"",""value"":2112817056}]","[{""key"":-5474557666971701248,""value"":506910753},{""key"":-3812638179211989504,""value"":2112817056}]","[{""key"":""AString76783"",""value"":14697929703826477056},{""key"":""AString89351"",""value"":3812638179211989504}]","[{""key"":-5474557666971701248,""value"":14697929703826477056},{""key"":-3812638179211989504,""value"":3812638179211989504}]","[{""key"":""AString76783"",""value"":78},{""key"":""AString89351"",""value"":21}]","[{""key"":-5474557666971701248,""value"":78},{""key"":-3812638179211989504,""value"":21}]","[{""key"":""AString76783"",""value"":""00000000-0000-0000-0000-000000000001""},{""key"":""AString89351"",""value"":""00000000-0000-0000-0000-000000000001""}]","[{""key"":-5474557666971701248,""value"":""00000000-0000-0000-0000-000000000001""},{""key"":-3812638179211989504,""value"":""00000000-0000-0000-0000-000000000001""}]","[{""key"":""AString76783"",""value"":""826476783.5364813824""},{""key"":""AString89351"",""value"":""211989351.8533091328""}]","[{""key"":-5474557666971701248,""value"":""826476783.5364813824""},{""key"":-3812638179211989504,""value"":""211989351.8533091328""}]","{""binary"":""7049Ug=="",""boolean"":true,""date32"":""2023-07-16"",""date64"":""2023-07-16"",""daytimeinterval"":{""days"":6783,""milliseconds"":6783},""decimal"":""826476783.5364813824"",""duration_ms"":""6476783ms"",""duration_ns"":""6476783ns"",""duration_s"":""6476783s"",""duration_us"":""6476783us"",""float32"":783.936,""float64"":783.936,""inet"":""192.0.2.0/24"",""int16"":-30727,""int32"":-1781555540,""int64"":-5474557666971701248,""int8"":-123,""json"":{""test"":[""a"",""b"",3]},""json_array"":[{""test"":""test""},123,{""test_number"":456}],""largebinary"":""7049Ug=="",""largestring"":""AString76783"",""mac"":""aa:bb:cc:dd:ee:ff"",""monthdaynanointerval"":{""months"":6783,""days"":6783,""nanoseconds"":6783},""monthinterval"":{""months"":1},""string"":""AString76783"",""time32ms"":""00:00:00"",""time32s"":""00:00:00"",""time64ns"":""00:00:00"",""time64us"":""00:00:00"",""timestamp_ms"":""2021-01-02 00:00:00"",""timestamp_ns"":""2021-01-02 00:00:00"",""timestamp_s"":""2021-01-02 00:00:00"",""timestamp_us"":""2021-01-02 00:00:00"",""uint16"":63063,""uint32"":506910753,""uint64"":14697929703826477056,""uint8"":78,""uuid"":""00000000-0000-0000-0000-000000000001""}","{""inner"":{""binary"":""7049Ug=="",""boolean"":true,""date32"":""2023-07-16"",""date64"":""2023-07-16"",""daytimeinterval"":{""days"":6783,""milliseconds"":6783},""decimal"":""826476783.5364813824"",""duration_ms"":""6476783ms"",""duration_ns"":""6476783ns"",""duration_s"":""6476783s"",""duration_us"":""6476783us"",""float32"":783.936,""float64"":783.936,""inet"":""192.0.2.0/24"",""int16"":-30727,""int32"":-1781555540,""int64"":-5474557666971701248,""int8"":-123,""json"":{""test"":[""a"",""b"",3]},""json_array"":[{""test"":""test""},123,{""test_number"":456}],""largebinary"":""7049Ug=="",""largestring"":""AString76783"",""mac"":""aa:bb:cc:dd:ee:ff"",""monthdaynanointerval"":{""months"":6783,""days"":6783,""nanoseconds"":6783},""monthinterval"":{""months"":1},""string"":""AString76783"",""time32ms"":""00:00:00"",""time32s"":""00:00:00"",""time64ns"":""00:00:00"",""time64us"":""00:00:00"",""timestamp_ms"":""2021-01-02 00:00:00"",""timestamp_ns"":""2021-01-02 00:00:00"",""timestamp_s"":""2021-01-02 00:00:00"",""timestamp_us"":""2021-01-02 00:00:00"",""uint16"":63063,""uint32"":506910753,""uint64"":14697929703826477056,""uint8"":78,""uuid"":""00000000-0000-0000-0000-000000000001""}}" -test-source,2021-01-01 00:00:00,00000000-0000-0000-0000-000000000001,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +test-source,2021-01-01 00:00:00,00000000-0000-0000-0000-000000000001,00000000-0000-0000-0000-000000000001,AQIDBA==,true,2023-04-24,2023-04-24,"{""days"":1,""milliseconds"":1}",123456789ms,123456789ns,123456789s,123456789us,1.1,1.1,192.0.2.0/24,-1,-1,-1,-1,AQIDBA==,AString,aa:bb:cc:dd:ee:ff,"{""months"":1,""days"":1,""nanoseconds"":1}",1,AString,00:00:00.000,00:00:00,00:00:00.000000000,00:00:00.000000,2021-01-02 00:00:00,2021-01-02 00:00:00,2021-01-02 00:00:00,2021-01-02 00:00:00,1,1,1,1,00000000-0000-0000-0000-000000000001,"{""test"":[""a"",""b"",3]}","[{""test"":""test""},123,{""test_number"":456}]","[true,null,true]","[""2023-04-24"",null,""2023-04-24""]","[""2023-04-24"",null,""2023-04-24""]","[{""days"":1,""milliseconds"":1},null,{""days"":1,""milliseconds"":1}]","[""123456789ms"",null,""123456789ms""]","[""123456789ns"",null,""123456789ns""]","[""123456789s"",null,""123456789s""]","[""123456789us"",null,""123456789us""]","[1.1,null,1.1]","[1.1,null,1.1]","[""192.0.2.0/24"",null,""192.0.2.0/24""]","[-1,null,-1]","[-1,null,-1]","[-1,null,-1]","[-1,null,-1]","[""AString"",null,""AString""]","[""aa:bb:cc:dd:ee:ff"",null,""aa:bb:cc:dd:ee:ff""]","[{""months"":1,""days"":1,""nanoseconds"":1},null,{""months"":1,""days"":1,""nanoseconds"":1}]","[{""months"":1},null,{""months"":1}]","[""AString"",null,""AString""]","[""00:00:00"",null,""00:00:00""]","[""00:00:00"",null,""00:00:00""]","[""00:00:00"",null,""00:00:00""]","[""00:00:00"",null,""00:00:00""]","[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]","[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]","[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]","[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]","[1,null,1]","[1,null,1]","[1,null,1]","[1,null,1]","[""00000000-0000-0000-0000-000000000001"",null,""00000000-0000-0000-0000-000000000001""]","{""binary"":""AQIDBA=="",""boolean"":true,""date32"":""2023-04-24"",""date64"":""2023-04-24"",""daytimeinterval"":{""days"":1,""milliseconds"":1},""duration_ms"":""123456789ms"",""duration_ns"":""123456789ns"",""duration_s"":""123456789s"",""duration_us"":""123456789us"",""float32"":1.1,""float64"":1.1,""inet"":""192.0.2.0/24"",""int16"":-1,""int32"":-1,""int64"":-1,""int8"":-1,""json"":{""test"":[""a"",""b"",3]},""json_array"":[{""test"":""test""},123,{""test_number"":456}],""largebinary"":""AQIDBA=="",""largestring"":""AString"",""mac"":""aa:bb:cc:dd:ee:ff"",""monthdaynanointerval"":{""months"":1,""days"":1,""nanoseconds"":1},""monthinterval"":{""months"":1},""string"":""AString"",""time32ms"":""00:00:00"",""time32s"":""00:00:00"",""time64ns"":""00:00:00"",""time64us"":""00:00:00"",""timestamp_ms"":""2021-01-02 00:00:00"",""timestamp_ns"":""2021-01-02 00:00:00"",""timestamp_s"":""2021-01-02 00:00:00"",""timestamp_us"":""2021-01-02 00:00:00"",""uint16"":1,""uint32"":1,""uint64"":1,""uint8"":1,""uuid"":""00000000-0000-0000-0000-000000000001""}","{""inner"":{""binary"":""AQIDBA=="",""boolean"":true,""date32"":""2023-04-24"",""date64"":""2023-04-24"",""daytimeinterval"":{""days"":1,""milliseconds"":1},""duration_ms"":""123456789ms"",""duration_ns"":""123456789ns"",""duration_s"":""123456789s"",""duration_us"":""123456789us"",""float32"":1.1,""float64"":1.1,""inet"":""192.0.2.0/24"",""int16"":-1,""int32"":-1,""int64"":-1,""int8"":-1,""json"":{""test"":[""a"",""b"",3]},""json_array"":[{""test"":""test""},123,{""test_number"":456}],""largebinary"":""AQIDBA=="",""largestring"":""AString"",""mac"":""aa:bb:cc:dd:ee:ff"",""monthdaynanointerval"":{""months"":1,""days"":1,""nanoseconds"":1},""monthinterval"":{""months"":1},""string"":""AString"",""time32ms"":""00:00:00"",""time32s"":""00:00:00"",""time64ns"":""00:00:00"",""time64us"":""00:00:00"",""timestamp_ms"":""2021-01-02 00:00:00"",""timestamp_ns"":""2021-01-02 00:00:00"",""timestamp_s"":""2021-01-02 00:00:00"",""timestamp_us"":""2021-01-02 00:00:00"",""uint16"":1,""uint32"":1,""uint64"":1,""uint8"":1,""uuid"":""00000000-0000-0000-0000-000000000001""}}" +test-source,2021-01-01 00:00:00,00000000-0000-0000-0000-000000000001,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, diff --git a/csv/testdata/TestWriteRead-with_delimiter.csv b/csv/testdata/TestWriteRead-with_delimiter.csv index b9655aac..7f667976 100644 --- a/csv/testdata/TestWriteRead-with_delimiter.csv +++ b/csv/testdata/TestWriteRead-with_delimiter.csv @@ -1,3 +1,3 @@ -test-source 2021-01-01 00:00:00 00000000-0000-0000-0000-000000000001 00000000-0000-0000-0000-000000000001 7049Ug== true 2023-07-16 2023-07-16 "{""days"":6783,""milliseconds"":6783}" 6476783ms 6476783ns 6476783s 6476783us 783.936 783.936 192.0.2.0/24 -30727 -1781555540 -5474557666971701248 -123 7049Ug== AString76783 aa:bb:cc:dd:ee:ff "{""months"":6783,""days"":6783,""nanoseconds"":6783}" 1 AString76783 00:00:00.000 00:00:00 00:00:00.000000000 00:00:00.000000 2021-01-02 00:00:00 2021-01-02 00:00:00 2021-01-02 00:00:00 2021-01-02 00:00:00 63063 506910753 14697929703826477056 78 00000000-0000-0000-0000-000000000001 826476783.5364813824 "{""test"":[""a"",""b"",3]}" "[{""test"":""test""},123,{""test_number"":456}]" [true,null,true] "[""2023-07-16"",null,""2023-07-16""]" "[""2023-07-16"",null,""2023-07-16""]" "[{""days"":6783,""milliseconds"":6783},null,{""days"":6783,""milliseconds"":6783}]" "[""6476783ms"",null,""6476783ms""]" "[""6476783ns"",null,""6476783ns""]" "[""6476783s"",null,""6476783s""]" "[""6476783us"",null,""6476783us""]" [783.936,null,783.936] [783.936,null,783.936] "[""192.0.2.0/24"",null,""192.0.2.0/24""]" [-30727,null,-30727] [-1781555540,null,-1781555540] [-5474557666971701248,null,-5474557666971701248] [-123,null,-123] "[""AString76783"",null,""AString76783""]" "[""aa:bb:cc:dd:ee:ff"",null,""aa:bb:cc:dd:ee:ff""]" "[{""months"":6783,""days"":6783,""nanoseconds"":6783},null,{""months"":6783,""days"":6783,""nanoseconds"":6783}]" "[{""months"":1},null,{""months"":1}]" "[""AString76783"",null,""AString76783""]" "[""00:00:00"",null,""00:00:00""]" "[""00:00:00"",null,""00:00:00""]" "[""00:00:00"",null,""00:00:00""]" "[""00:00:00"",null,""00:00:00""]" "[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]" "[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]" "[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]" "[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]" [63063,null,63063] [506910753,null,506910753] [14697929703826477056,null,14697929703826477056] [78,null,78] "[""00000000-0000-0000-0000-000000000001"",null,""00000000-0000-0000-0000-000000000001""]" "[""826476783.5364813824"",null,""826476783.5364813824""]" "[{""key"":""AString76783"",""value"":true},{""key"":""AString89351"",""value"":true}]" "[{""key"":-5474557666971701248,""value"":true},{""key"":-3812638179211989504,""value"":true}]" "[{""key"":""AString76783"",""value"":""2023-07-16""},{""key"":""AString89351"",""value"":""2023-06-14""}]" "[{""key"":-5474557666971701248,""value"":""2023-07-16""},{""key"":-3812638179211989504,""value"":""2023-06-14""}]" "[{""key"":""AString76783"",""value"":""2023-07-16""},{""key"":""AString89351"",""value"":""2023-06-14""}]" "[{""key"":-5474557666971701248,""value"":""2023-07-16""},{""key"":-3812638179211989504,""value"":""2023-06-14""}]" "[{""key"":""AString76783"",""value"":{""days"":6783,""milliseconds"":6783}},{""key"":""AString89351"",""value"":{""days"":9351,""milliseconds"":9351}}]" "[{""key"":-5474557666971701248,""value"":{""days"":6783,""milliseconds"":6783}},{""key"":-3812638179211989504,""value"":{""days"":9351,""milliseconds"":9351}}]" "[{""key"":""AString76783"",""value"":""6476783ms""},{""key"":""AString89351"",""value"":""1989351ms""}]" "[{""key"":-5474557666971701248,""value"":""6476783ms""},{""key"":-3812638179211989504,""value"":""1989351ms""}]" "[{""key"":""AString76783"",""value"":""6476783ns""},{""key"":""AString89351"",""value"":""1989351ns""}]" "[{""key"":-5474557666971701248,""value"":""6476783ns""},{""key"":-3812638179211989504,""value"":""1989351ns""}]" "[{""key"":""AString76783"",""value"":""6476783s""},{""key"":""AString89351"",""value"":""1989351s""}]" "[{""key"":-5474557666971701248,""value"":""6476783s""},{""key"":-3812638179211989504,""value"":""1989351s""}]" "[{""key"":""AString76783"",""value"":""6476783us""},{""key"":""AString89351"",""value"":""1989351us""}]" "[{""key"":-5474557666971701248,""value"":""6476783us""},{""key"":-3812638179211989504,""value"":""1989351us""}]" "[{""key"":""AString76783"",""value"":783.936},{""key"":""AString89351"",""value"":351.121}]" "[{""key"":-5474557666971701248,""value"":783.936},{""key"":-3812638179211989504,""value"":351.121}]" "[{""key"":""AString76783"",""value"":783.936},{""key"":""AString89351"",""value"":351.121}]" "[{""key"":-5474557666971701248,""value"":783.936},{""key"":-3812638179211989504,""value"":351.121}]" "[{""key"":""AString76783"",""value"":""192.0.2.0/24""},{""key"":""AString89351"",""value"":""192.0.2.0/24""}]" "[{""key"":-5474557666971701248,""value"":""192.0.2.0/24""},{""key"":-3812638179211989504,""value"":""192.0.2.0/24""}]" "[{""key"":""AString76783"",""value"":-30727},{""key"":""AString89351"",""value"":-25508}]" "[{""key"":-5474557666971701248,""value"":-30727},{""key"":-3812638179211989504,""value"":-25508}]" "[{""key"":""AString76783"",""value"":-1781555540},{""key"":""AString89351"",""value"":-853032410}]" "[{""key"":-5474557666971701248,""value"":-1781555540},{""key"":-3812638179211989504,""value"":-853032410}]" "[{""key"":""AString76783"",""value"":-5474557666971701248},{""key"":""AString89351"",""value"":-3812638179211989504}]" "[{""key"":-5474557666971701248,""value"":-5474557666971701248},{""key"":-3812638179211989504,""value"":-3812638179211989504}]" "[{""key"":""AString76783"",""value"":-123},{""key"":""AString89351"",""value"":-60}]" "[{""key"":-5474557666971701248,""value"":-123},{""key"":-3812638179211989504,""value"":-60}]" "[{""key"":""AString76783"",""value"":""AString76783""},{""key"":""AString89351"",""value"":""AString89351""}]" "[{""key"":-5474557666971701248,""value"":""AString76783""},{""key"":-3812638179211989504,""value"":""AString89351""}]" "[{""key"":""AString76783"",""value"":""aa:bb:cc:dd:ee:ff""},{""key"":""AString89351"",""value"":""aa:bb:cc:dd:ee:ff""}]" "[{""key"":-5474557666971701248,""value"":""aa:bb:cc:dd:ee:ff""},{""key"":-3812638179211989504,""value"":""aa:bb:cc:dd:ee:ff""}]" "[{""key"":""AString76783"",""value"":{""months"":6783,""days"":6783,""nanoseconds"":6783}},{""key"":""AString89351"",""value"":{""months"":9351,""days"":9351,""nanoseconds"":9351}}]" "[{""key"":-5474557666971701248,""value"":{""months"":6783,""days"":6783,""nanoseconds"":6783}},{""key"":-3812638179211989504,""value"":{""months"":9351,""days"":9351,""nanoseconds"":9351}}]" "[{""key"":""AString76783"",""value"":{""months"":1}},{""key"":""AString89351"",""value"":{""months"":1}}]" "[{""key"":-5474557666971701248,""value"":{""months"":1}},{""key"":-3812638179211989504,""value"":{""months"":1}}]" "[{""key"":""AString76783"",""value"":""AString76783""},{""key"":""AString89351"",""value"":""AString89351""}]" "[{""key"":-5474557666971701248,""value"":""AString76783""},{""key"":-3812638179211989504,""value"":""AString89351""}]" "[{""key"":""AString76783"",""value"":""00:00:00""},{""key"":""AString89351"",""value"":""00:00:00""}]" "[{""key"":-5474557666971701248,""value"":""00:00:00""},{""key"":-3812638179211989504,""value"":""00:00:00""}]" "[{""key"":""AString76783"",""value"":""00:00:00""},{""key"":""AString89351"",""value"":""00:00:00""}]" "[{""key"":-5474557666971701248,""value"":""00:00:00""},{""key"":-3812638179211989504,""value"":""00:00:00""}]" "[{""key"":""AString76783"",""value"":""00:00:00""},{""key"":""AString89351"",""value"":""00:00:00""}]" "[{""key"":-5474557666971701248,""value"":""00:00:00""},{""key"":-3812638179211989504,""value"":""00:00:00""}]" "[{""key"":""AString76783"",""value"":""00:00:00""},{""key"":""AString89351"",""value"":""00:00:00""}]" "[{""key"":-5474557666971701248,""value"":""00:00:00""},{""key"":-3812638179211989504,""value"":""00:00:00""}]" "[{""key"":""AString76783"",""value"":""2021-01-02 00:00:00""},{""key"":""AString89351"",""value"":""2021-01-02 00:00:00""}]" "[{""key"":-5474557666971701248,""value"":""2021-01-02 00:00:00""},{""key"":-3812638179211989504,""value"":""2021-01-02 00:00:00""}]" "[{""key"":""AString76783"",""value"":""2021-01-02 00:00:00""},{""key"":""AString89351"",""value"":""2021-01-02 00:00:00""}]" "[{""key"":-5474557666971701248,""value"":""2021-01-02 00:00:00""},{""key"":-3812638179211989504,""value"":""2021-01-02 00:00:00""}]" "[{""key"":""AString76783"",""value"":""2021-01-02 00:00:00""},{""key"":""AString89351"",""value"":""2021-01-02 00:00:00""}]" "[{""key"":-5474557666971701248,""value"":""2021-01-02 00:00:00""},{""key"":-3812638179211989504,""value"":""2021-01-02 00:00:00""}]" "[{""key"":""AString76783"",""value"":""2021-01-02 00:00:00""},{""key"":""AString89351"",""value"":""2021-01-02 00:00:00""}]" "[{""key"":-5474557666971701248,""value"":""2021-01-02 00:00:00""},{""key"":-3812638179211989504,""value"":""2021-01-02 00:00:00""}]" "[{""key"":""AString76783"",""value"":63063},{""key"":""AString89351"",""value"":34191}]" "[{""key"":-5474557666971701248,""value"":63063},{""key"":-3812638179211989504,""value"":34191}]" "[{""key"":""AString76783"",""value"":506910753},{""key"":""AString89351"",""value"":2112817056}]" "[{""key"":-5474557666971701248,""value"":506910753},{""key"":-3812638179211989504,""value"":2112817056}]" "[{""key"":""AString76783"",""value"":14697929703826477056},{""key"":""AString89351"",""value"":3812638179211989504}]" "[{""key"":-5474557666971701248,""value"":14697929703826477056},{""key"":-3812638179211989504,""value"":3812638179211989504}]" "[{""key"":""AString76783"",""value"":78},{""key"":""AString89351"",""value"":21}]" "[{""key"":-5474557666971701248,""value"":78},{""key"":-3812638179211989504,""value"":21}]" "[{""key"":""AString76783"",""value"":""00000000-0000-0000-0000-000000000001""},{""key"":""AString89351"",""value"":""00000000-0000-0000-0000-000000000001""}]" "[{""key"":-5474557666971701248,""value"":""00000000-0000-0000-0000-000000000001""},{""key"":-3812638179211989504,""value"":""00000000-0000-0000-0000-000000000001""}]" "[{""key"":""AString76783"",""value"":""826476783.5364813824""},{""key"":""AString89351"",""value"":""211989351.8533091328""}]" "[{""key"":-5474557666971701248,""value"":""826476783.5364813824""},{""key"":-3812638179211989504,""value"":""211989351.8533091328""}]" "{""binary"":""7049Ug=="",""boolean"":true,""date32"":""2023-07-16"",""date64"":""2023-07-16"",""daytimeinterval"":{""days"":6783,""milliseconds"":6783},""decimal"":""826476783.5364813824"",""duration_ms"":""6476783ms"",""duration_ns"":""6476783ns"",""duration_s"":""6476783s"",""duration_us"":""6476783us"",""float32"":783.936,""float64"":783.936,""inet"":""192.0.2.0/24"",""int16"":-30727,""int32"":-1781555540,""int64"":-5474557666971701248,""int8"":-123,""json"":{""test"":[""a"",""b"",3]},""json_array"":[{""test"":""test""},123,{""test_number"":456}],""largebinary"":""7049Ug=="",""largestring"":""AString76783"",""mac"":""aa:bb:cc:dd:ee:ff"",""monthdaynanointerval"":{""months"":6783,""days"":6783,""nanoseconds"":6783},""monthinterval"":{""months"":1},""string"":""AString76783"",""time32ms"":""00:00:00"",""time32s"":""00:00:00"",""time64ns"":""00:00:00"",""time64us"":""00:00:00"",""timestamp_ms"":""2021-01-02 00:00:00"",""timestamp_ns"":""2021-01-02 00:00:00"",""timestamp_s"":""2021-01-02 00:00:00"",""timestamp_us"":""2021-01-02 00:00:00"",""uint16"":63063,""uint32"":506910753,""uint64"":14697929703826477056,""uint8"":78,""uuid"":""00000000-0000-0000-0000-000000000001""}" "{""inner"":{""binary"":""7049Ug=="",""boolean"":true,""date32"":""2023-07-16"",""date64"":""2023-07-16"",""daytimeinterval"":{""days"":6783,""milliseconds"":6783},""decimal"":""826476783.5364813824"",""duration_ms"":""6476783ms"",""duration_ns"":""6476783ns"",""duration_s"":""6476783s"",""duration_us"":""6476783us"",""float32"":783.936,""float64"":783.936,""inet"":""192.0.2.0/24"",""int16"":-30727,""int32"":-1781555540,""int64"":-5474557666971701248,""int8"":-123,""json"":{""test"":[""a"",""b"",3]},""json_array"":[{""test"":""test""},123,{""test_number"":456}],""largebinary"":""7049Ug=="",""largestring"":""AString76783"",""mac"":""aa:bb:cc:dd:ee:ff"",""monthdaynanointerval"":{""months"":6783,""days"":6783,""nanoseconds"":6783},""monthinterval"":{""months"":1},""string"":""AString76783"",""time32ms"":""00:00:00"",""time32s"":""00:00:00"",""time64ns"":""00:00:00"",""time64us"":""00:00:00"",""timestamp_ms"":""2021-01-02 00:00:00"",""timestamp_ns"":""2021-01-02 00:00:00"",""timestamp_s"":""2021-01-02 00:00:00"",""timestamp_us"":""2021-01-02 00:00:00"",""uint16"":63063,""uint32"":506910753,""uint64"":14697929703826477056,""uint8"":78,""uuid"":""00000000-0000-0000-0000-000000000001""}}" -test-source 2021-01-01 00:00:00 00000000-0000-0000-0000-000000000001 +test-source 2021-01-01 00:00:00 00000000-0000-0000-0000-000000000001 00000000-0000-0000-0000-000000000001 AQIDBA== true 2023-04-24 2023-04-24 "{""days"":1,""milliseconds"":1}" 123456789ms 123456789ns 123456789s 123456789us 1.1 1.1 192.0.2.0/24 -1 -1 -1 -1 AQIDBA== AString aa:bb:cc:dd:ee:ff "{""months"":1,""days"":1,""nanoseconds"":1}" 1 AString 00:00:00.000 00:00:00 00:00:00.000000000 00:00:00.000000 2021-01-02 00:00:00 2021-01-02 00:00:00 2021-01-02 00:00:00 2021-01-02 00:00:00 1 1 1 1 00000000-0000-0000-0000-000000000001 "{""test"":[""a"",""b"",3]}" "[{""test"":""test""},123,{""test_number"":456}]" [true,null,true] "[""2023-04-24"",null,""2023-04-24""]" "[""2023-04-24"",null,""2023-04-24""]" "[{""days"":1,""milliseconds"":1},null,{""days"":1,""milliseconds"":1}]" "[""123456789ms"",null,""123456789ms""]" "[""123456789ns"",null,""123456789ns""]" "[""123456789s"",null,""123456789s""]" "[""123456789us"",null,""123456789us""]" [1.1,null,1.1] [1.1,null,1.1] "[""192.0.2.0/24"",null,""192.0.2.0/24""]" [-1,null,-1] [-1,null,-1] [-1,null,-1] [-1,null,-1] "[""AString"",null,""AString""]" "[""aa:bb:cc:dd:ee:ff"",null,""aa:bb:cc:dd:ee:ff""]" "[{""months"":1,""days"":1,""nanoseconds"":1},null,{""months"":1,""days"":1,""nanoseconds"":1}]" "[{""months"":1},null,{""months"":1}]" "[""AString"",null,""AString""]" "[""00:00:00"",null,""00:00:00""]" "[""00:00:00"",null,""00:00:00""]" "[""00:00:00"",null,""00:00:00""]" "[""00:00:00"",null,""00:00:00""]" "[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]" "[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]" "[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]" "[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]" [1,null,1] [1,null,1] [1,null,1] [1,null,1] "[""00000000-0000-0000-0000-000000000001"",null,""00000000-0000-0000-0000-000000000001""]" "{""binary"":""AQIDBA=="",""boolean"":true,""date32"":""2023-04-24"",""date64"":""2023-04-24"",""daytimeinterval"":{""days"":1,""milliseconds"":1},""duration_ms"":""123456789ms"",""duration_ns"":""123456789ns"",""duration_s"":""123456789s"",""duration_us"":""123456789us"",""float32"":1.1,""float64"":1.1,""inet"":""192.0.2.0/24"",""int16"":-1,""int32"":-1,""int64"":-1,""int8"":-1,""json"":{""test"":[""a"",""b"",3]},""json_array"":[{""test"":""test""},123,{""test_number"":456}],""largebinary"":""AQIDBA=="",""largestring"":""AString"",""mac"":""aa:bb:cc:dd:ee:ff"",""monthdaynanointerval"":{""months"":1,""days"":1,""nanoseconds"":1},""monthinterval"":{""months"":1},""string"":""AString"",""time32ms"":""00:00:00"",""time32s"":""00:00:00"",""time64ns"":""00:00:00"",""time64us"":""00:00:00"",""timestamp_ms"":""2021-01-02 00:00:00"",""timestamp_ns"":""2021-01-02 00:00:00"",""timestamp_s"":""2021-01-02 00:00:00"",""timestamp_us"":""2021-01-02 00:00:00"",""uint16"":1,""uint32"":1,""uint64"":1,""uint8"":1,""uuid"":""00000000-0000-0000-0000-000000000001""}" "{""inner"":{""binary"":""AQIDBA=="",""boolean"":true,""date32"":""2023-04-24"",""date64"":""2023-04-24"",""daytimeinterval"":{""days"":1,""milliseconds"":1},""duration_ms"":""123456789ms"",""duration_ns"":""123456789ns"",""duration_s"":""123456789s"",""duration_us"":""123456789us"",""float32"":1.1,""float64"":1.1,""inet"":""192.0.2.0/24"",""int16"":-1,""int32"":-1,""int64"":-1,""int8"":-1,""json"":{""test"":[""a"",""b"",3]},""json_array"":[{""test"":""test""},123,{""test_number"":456}],""largebinary"":""AQIDBA=="",""largestring"":""AString"",""mac"":""aa:bb:cc:dd:ee:ff"",""monthdaynanointerval"":{""months"":1,""days"":1,""nanoseconds"":1},""monthinterval"":{""months"":1},""string"":""AString"",""time32ms"":""00:00:00"",""time32s"":""00:00:00"",""time64ns"":""00:00:00"",""time64us"":""00:00:00"",""timestamp_ms"":""2021-01-02 00:00:00"",""timestamp_ns"":""2021-01-02 00:00:00"",""timestamp_s"":""2021-01-02 00:00:00"",""timestamp_us"":""2021-01-02 00:00:00"",""uint16"":1,""uint32"":1,""uint64"":1,""uint8"":1,""uuid"":""00000000-0000-0000-0000-000000000001""}}" +test-source 2021-01-01 00:00:00 00000000-0000-0000-0000-000000000001 diff --git a/csv/testdata/TestWriteRead-with_delimiter_headers.csv b/csv/testdata/TestWriteRead-with_delimiter_headers.csv index b0fdbfcb..117c955f 100644 --- a/csv/testdata/TestWriteRead-with_delimiter_headers.csv +++ b/csv/testdata/TestWriteRead-with_delimiter_headers.csv @@ -1,4 +1,4 @@ -_cq_source_name _cq_sync_time _cq_id _cq_parent_id binary boolean date32 date64 daytimeinterval duration_ms duration_ns duration_s duration_us float32 float64 inet int16 int32 int64 int8 largebinary largestring mac monthdaynanointerval monthinterval string time32ms time32s time64ns time64us timestamp_ms timestamp_ns timestamp_s timestamp_us uint16 uint32 uint64 uint8 uuid decimal json json_array boolean_list date32_list date64_list daytimeinterval_list duration_ms_list duration_ns_list duration_s_list duration_us_list float32_list float64_list inet_list int16_list int32_list int64_list int8_list largestring_list mac_list monthdaynanointerval_list monthinterval_list string_list time32ms_list time32s_list time64ns_list time64us_list timestamp_ms_list timestamp_ns_list timestamp_s_list timestamp_us_list uint16_list uint32_list uint64_list uint8_list uuid_list decimal_list string_boolean_map int_boolean_map string_date32_map int_date32_map string_date64_map int_date64_map string_daytimeinterval_map int_daytimeinterval_map string_duration_ms_map int_duration_ms_map string_duration_ns_map int_duration_ns_map string_duration_s_map int_duration_s_map string_duration_us_map int_duration_us_map string_float32_map int_float32_map string_float64_map int_float64_map string_inet_map int_inet_map string_int16_map int_int16_map string_int32_map int_int32_map string_int64_map int_int64_map string_int8_map int_int8_map string_largestring_map int_largestring_map string_mac_map int_mac_map string_monthdaynanointerval_map int_monthdaynanointerval_map string_monthinterval_map int_monthinterval_map string_string_map int_string_map string_time32ms_map int_time32ms_map string_time32s_map int_time32s_map string_time64ns_map int_time64ns_map string_time64us_map int_time64us_map string_timestamp_ms_map int_timestamp_ms_map string_timestamp_ns_map int_timestamp_ns_map string_timestamp_s_map int_timestamp_s_map string_timestamp_us_map int_timestamp_us_map string_uint16_map int_uint16_map string_uint32_map int_uint32_map string_uint64_map int_uint64_map string_uint8_map int_uint8_map string_uuid_map int_uuid_map string_decimal_map int_decimal_map struct nested_struct -test-source 2021-01-01 00:00:00 00000000-0000-0000-0000-000000000001 00000000-0000-0000-0000-000000000001 7049Ug== true 2023-07-16 2023-07-16 "{""days"":6783,""milliseconds"":6783}" 6476783ms 6476783ns 6476783s 6476783us 783.936 783.936 192.0.2.0/24 -30727 -1781555540 -5474557666971701248 -123 7049Ug== AString76783 aa:bb:cc:dd:ee:ff "{""months"":6783,""days"":6783,""nanoseconds"":6783}" 1 AString76783 00:00:00.000 00:00:00 00:00:00.000000000 00:00:00.000000 2021-01-02 00:00:00 2021-01-02 00:00:00 2021-01-02 00:00:00 2021-01-02 00:00:00 63063 506910753 14697929703826477056 78 00000000-0000-0000-0000-000000000001 826476783.5364813824 "{""test"":[""a"",""b"",3]}" "[{""test"":""test""},123,{""test_number"":456}]" [true,null,true] "[""2023-07-16"",null,""2023-07-16""]" "[""2023-07-16"",null,""2023-07-16""]" "[{""days"":6783,""milliseconds"":6783},null,{""days"":6783,""milliseconds"":6783}]" "[""6476783ms"",null,""6476783ms""]" "[""6476783ns"",null,""6476783ns""]" "[""6476783s"",null,""6476783s""]" "[""6476783us"",null,""6476783us""]" [783.936,null,783.936] [783.936,null,783.936] "[""192.0.2.0/24"",null,""192.0.2.0/24""]" [-30727,null,-30727] [-1781555540,null,-1781555540] [-5474557666971701248,null,-5474557666971701248] [-123,null,-123] "[""AString76783"",null,""AString76783""]" "[""aa:bb:cc:dd:ee:ff"",null,""aa:bb:cc:dd:ee:ff""]" "[{""months"":6783,""days"":6783,""nanoseconds"":6783},null,{""months"":6783,""days"":6783,""nanoseconds"":6783}]" "[{""months"":1},null,{""months"":1}]" "[""AString76783"",null,""AString76783""]" "[""00:00:00"",null,""00:00:00""]" "[""00:00:00"",null,""00:00:00""]" "[""00:00:00"",null,""00:00:00""]" "[""00:00:00"",null,""00:00:00""]" "[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]" "[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]" "[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]" "[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]" [63063,null,63063] [506910753,null,506910753] [14697929703826477056,null,14697929703826477056] [78,null,78] "[""00000000-0000-0000-0000-000000000001"",null,""00000000-0000-0000-0000-000000000001""]" "[""826476783.5364813824"",null,""826476783.5364813824""]" "[{""key"":""AString76783"",""value"":true},{""key"":""AString89351"",""value"":true}]" "[{""key"":-5474557666971701248,""value"":true},{""key"":-3812638179211989504,""value"":true}]" "[{""key"":""AString76783"",""value"":""2023-07-16""},{""key"":""AString89351"",""value"":""2023-06-14""}]" "[{""key"":-5474557666971701248,""value"":""2023-07-16""},{""key"":-3812638179211989504,""value"":""2023-06-14""}]" "[{""key"":""AString76783"",""value"":""2023-07-16""},{""key"":""AString89351"",""value"":""2023-06-14""}]" "[{""key"":-5474557666971701248,""value"":""2023-07-16""},{""key"":-3812638179211989504,""value"":""2023-06-14""}]" "[{""key"":""AString76783"",""value"":{""days"":6783,""milliseconds"":6783}},{""key"":""AString89351"",""value"":{""days"":9351,""milliseconds"":9351}}]" "[{""key"":-5474557666971701248,""value"":{""days"":6783,""milliseconds"":6783}},{""key"":-3812638179211989504,""value"":{""days"":9351,""milliseconds"":9351}}]" "[{""key"":""AString76783"",""value"":""6476783ms""},{""key"":""AString89351"",""value"":""1989351ms""}]" "[{""key"":-5474557666971701248,""value"":""6476783ms""},{""key"":-3812638179211989504,""value"":""1989351ms""}]" "[{""key"":""AString76783"",""value"":""6476783ns""},{""key"":""AString89351"",""value"":""1989351ns""}]" "[{""key"":-5474557666971701248,""value"":""6476783ns""},{""key"":-3812638179211989504,""value"":""1989351ns""}]" "[{""key"":""AString76783"",""value"":""6476783s""},{""key"":""AString89351"",""value"":""1989351s""}]" "[{""key"":-5474557666971701248,""value"":""6476783s""},{""key"":-3812638179211989504,""value"":""1989351s""}]" "[{""key"":""AString76783"",""value"":""6476783us""},{""key"":""AString89351"",""value"":""1989351us""}]" "[{""key"":-5474557666971701248,""value"":""6476783us""},{""key"":-3812638179211989504,""value"":""1989351us""}]" "[{""key"":""AString76783"",""value"":783.936},{""key"":""AString89351"",""value"":351.121}]" "[{""key"":-5474557666971701248,""value"":783.936},{""key"":-3812638179211989504,""value"":351.121}]" "[{""key"":""AString76783"",""value"":783.936},{""key"":""AString89351"",""value"":351.121}]" "[{""key"":-5474557666971701248,""value"":783.936},{""key"":-3812638179211989504,""value"":351.121}]" "[{""key"":""AString76783"",""value"":""192.0.2.0/24""},{""key"":""AString89351"",""value"":""192.0.2.0/24""}]" "[{""key"":-5474557666971701248,""value"":""192.0.2.0/24""},{""key"":-3812638179211989504,""value"":""192.0.2.0/24""}]" "[{""key"":""AString76783"",""value"":-30727},{""key"":""AString89351"",""value"":-25508}]" "[{""key"":-5474557666971701248,""value"":-30727},{""key"":-3812638179211989504,""value"":-25508}]" "[{""key"":""AString76783"",""value"":-1781555540},{""key"":""AString89351"",""value"":-853032410}]" "[{""key"":-5474557666971701248,""value"":-1781555540},{""key"":-3812638179211989504,""value"":-853032410}]" "[{""key"":""AString76783"",""value"":-5474557666971701248},{""key"":""AString89351"",""value"":-3812638179211989504}]" "[{""key"":-5474557666971701248,""value"":-5474557666971701248},{""key"":-3812638179211989504,""value"":-3812638179211989504}]" "[{""key"":""AString76783"",""value"":-123},{""key"":""AString89351"",""value"":-60}]" "[{""key"":-5474557666971701248,""value"":-123},{""key"":-3812638179211989504,""value"":-60}]" "[{""key"":""AString76783"",""value"":""AString76783""},{""key"":""AString89351"",""value"":""AString89351""}]" "[{""key"":-5474557666971701248,""value"":""AString76783""},{""key"":-3812638179211989504,""value"":""AString89351""}]" "[{""key"":""AString76783"",""value"":""aa:bb:cc:dd:ee:ff""},{""key"":""AString89351"",""value"":""aa:bb:cc:dd:ee:ff""}]" "[{""key"":-5474557666971701248,""value"":""aa:bb:cc:dd:ee:ff""},{""key"":-3812638179211989504,""value"":""aa:bb:cc:dd:ee:ff""}]" "[{""key"":""AString76783"",""value"":{""months"":6783,""days"":6783,""nanoseconds"":6783}},{""key"":""AString89351"",""value"":{""months"":9351,""days"":9351,""nanoseconds"":9351}}]" "[{""key"":-5474557666971701248,""value"":{""months"":6783,""days"":6783,""nanoseconds"":6783}},{""key"":-3812638179211989504,""value"":{""months"":9351,""days"":9351,""nanoseconds"":9351}}]" "[{""key"":""AString76783"",""value"":{""months"":1}},{""key"":""AString89351"",""value"":{""months"":1}}]" "[{""key"":-5474557666971701248,""value"":{""months"":1}},{""key"":-3812638179211989504,""value"":{""months"":1}}]" "[{""key"":""AString76783"",""value"":""AString76783""},{""key"":""AString89351"",""value"":""AString89351""}]" "[{""key"":-5474557666971701248,""value"":""AString76783""},{""key"":-3812638179211989504,""value"":""AString89351""}]" "[{""key"":""AString76783"",""value"":""00:00:00""},{""key"":""AString89351"",""value"":""00:00:00""}]" "[{""key"":-5474557666971701248,""value"":""00:00:00""},{""key"":-3812638179211989504,""value"":""00:00:00""}]" "[{""key"":""AString76783"",""value"":""00:00:00""},{""key"":""AString89351"",""value"":""00:00:00""}]" "[{""key"":-5474557666971701248,""value"":""00:00:00""},{""key"":-3812638179211989504,""value"":""00:00:00""}]" "[{""key"":""AString76783"",""value"":""00:00:00""},{""key"":""AString89351"",""value"":""00:00:00""}]" "[{""key"":-5474557666971701248,""value"":""00:00:00""},{""key"":-3812638179211989504,""value"":""00:00:00""}]" "[{""key"":""AString76783"",""value"":""00:00:00""},{""key"":""AString89351"",""value"":""00:00:00""}]" "[{""key"":-5474557666971701248,""value"":""00:00:00""},{""key"":-3812638179211989504,""value"":""00:00:00""}]" "[{""key"":""AString76783"",""value"":""2021-01-02 00:00:00""},{""key"":""AString89351"",""value"":""2021-01-02 00:00:00""}]" "[{""key"":-5474557666971701248,""value"":""2021-01-02 00:00:00""},{""key"":-3812638179211989504,""value"":""2021-01-02 00:00:00""}]" "[{""key"":""AString76783"",""value"":""2021-01-02 00:00:00""},{""key"":""AString89351"",""value"":""2021-01-02 00:00:00""}]" "[{""key"":-5474557666971701248,""value"":""2021-01-02 00:00:00""},{""key"":-3812638179211989504,""value"":""2021-01-02 00:00:00""}]" "[{""key"":""AString76783"",""value"":""2021-01-02 00:00:00""},{""key"":""AString89351"",""value"":""2021-01-02 00:00:00""}]" "[{""key"":-5474557666971701248,""value"":""2021-01-02 00:00:00""},{""key"":-3812638179211989504,""value"":""2021-01-02 00:00:00""}]" "[{""key"":""AString76783"",""value"":""2021-01-02 00:00:00""},{""key"":""AString89351"",""value"":""2021-01-02 00:00:00""}]" "[{""key"":-5474557666971701248,""value"":""2021-01-02 00:00:00""},{""key"":-3812638179211989504,""value"":""2021-01-02 00:00:00""}]" "[{""key"":""AString76783"",""value"":63063},{""key"":""AString89351"",""value"":34191}]" "[{""key"":-5474557666971701248,""value"":63063},{""key"":-3812638179211989504,""value"":34191}]" "[{""key"":""AString76783"",""value"":506910753},{""key"":""AString89351"",""value"":2112817056}]" "[{""key"":-5474557666971701248,""value"":506910753},{""key"":-3812638179211989504,""value"":2112817056}]" "[{""key"":""AString76783"",""value"":14697929703826477056},{""key"":""AString89351"",""value"":3812638179211989504}]" "[{""key"":-5474557666971701248,""value"":14697929703826477056},{""key"":-3812638179211989504,""value"":3812638179211989504}]" "[{""key"":""AString76783"",""value"":78},{""key"":""AString89351"",""value"":21}]" "[{""key"":-5474557666971701248,""value"":78},{""key"":-3812638179211989504,""value"":21}]" "[{""key"":""AString76783"",""value"":""00000000-0000-0000-0000-000000000001""},{""key"":""AString89351"",""value"":""00000000-0000-0000-0000-000000000001""}]" "[{""key"":-5474557666971701248,""value"":""00000000-0000-0000-0000-000000000001""},{""key"":-3812638179211989504,""value"":""00000000-0000-0000-0000-000000000001""}]" "[{""key"":""AString76783"",""value"":""826476783.5364813824""},{""key"":""AString89351"",""value"":""211989351.8533091328""}]" "[{""key"":-5474557666971701248,""value"":""826476783.5364813824""},{""key"":-3812638179211989504,""value"":""211989351.8533091328""}]" "{""binary"":""7049Ug=="",""boolean"":true,""date32"":""2023-07-16"",""date64"":""2023-07-16"",""daytimeinterval"":{""days"":6783,""milliseconds"":6783},""decimal"":""826476783.5364813824"",""duration_ms"":""6476783ms"",""duration_ns"":""6476783ns"",""duration_s"":""6476783s"",""duration_us"":""6476783us"",""float32"":783.936,""float64"":783.936,""inet"":""192.0.2.0/24"",""int16"":-30727,""int32"":-1781555540,""int64"":-5474557666971701248,""int8"":-123,""json"":{""test"":[""a"",""b"",3]},""json_array"":[{""test"":""test""},123,{""test_number"":456}],""largebinary"":""7049Ug=="",""largestring"":""AString76783"",""mac"":""aa:bb:cc:dd:ee:ff"",""monthdaynanointerval"":{""months"":6783,""days"":6783,""nanoseconds"":6783},""monthinterval"":{""months"":1},""string"":""AString76783"",""time32ms"":""00:00:00"",""time32s"":""00:00:00"",""time64ns"":""00:00:00"",""time64us"":""00:00:00"",""timestamp_ms"":""2021-01-02 00:00:00"",""timestamp_ns"":""2021-01-02 00:00:00"",""timestamp_s"":""2021-01-02 00:00:00"",""timestamp_us"":""2021-01-02 00:00:00"",""uint16"":63063,""uint32"":506910753,""uint64"":14697929703826477056,""uint8"":78,""uuid"":""00000000-0000-0000-0000-000000000001""}" "{""inner"":{""binary"":""7049Ug=="",""boolean"":true,""date32"":""2023-07-16"",""date64"":""2023-07-16"",""daytimeinterval"":{""days"":6783,""milliseconds"":6783},""decimal"":""826476783.5364813824"",""duration_ms"":""6476783ms"",""duration_ns"":""6476783ns"",""duration_s"":""6476783s"",""duration_us"":""6476783us"",""float32"":783.936,""float64"":783.936,""inet"":""192.0.2.0/24"",""int16"":-30727,""int32"":-1781555540,""int64"":-5474557666971701248,""int8"":-123,""json"":{""test"":[""a"",""b"",3]},""json_array"":[{""test"":""test""},123,{""test_number"":456}],""largebinary"":""7049Ug=="",""largestring"":""AString76783"",""mac"":""aa:bb:cc:dd:ee:ff"",""monthdaynanointerval"":{""months"":6783,""days"":6783,""nanoseconds"":6783},""monthinterval"":{""months"":1},""string"":""AString76783"",""time32ms"":""00:00:00"",""time32s"":""00:00:00"",""time64ns"":""00:00:00"",""time64us"":""00:00:00"",""timestamp_ms"":""2021-01-02 00:00:00"",""timestamp_ns"":""2021-01-02 00:00:00"",""timestamp_s"":""2021-01-02 00:00:00"",""timestamp_us"":""2021-01-02 00:00:00"",""uint16"":63063,""uint32"":506910753,""uint64"":14697929703826477056,""uint8"":78,""uuid"":""00000000-0000-0000-0000-000000000001""}}" -test-source 2021-01-01 00:00:00 00000000-0000-0000-0000-000000000001 +_cq_source_name _cq_sync_time _cq_id _cq_parent_id binary boolean date32 date64 daytimeinterval duration_ms duration_ns duration_s duration_us float32 float64 inet int16 int32 int64 int8 largebinary largestring mac monthdaynanointerval monthinterval string time32ms time32s time64ns time64us timestamp_ms timestamp_ns timestamp_s timestamp_us uint16 uint32 uint64 uint8 uuid json json_array boolean_list date32_list date64_list daytimeinterval_list duration_ms_list duration_ns_list duration_s_list duration_us_list float32_list float64_list inet_list int16_list int32_list int64_list int8_list largestring_list mac_list monthdaynanointerval_list monthinterval_list string_list time32ms_list time32s_list time64ns_list time64us_list timestamp_ms_list timestamp_ns_list timestamp_s_list timestamp_us_list uint16_list uint32_list uint64_list uint8_list uuid_list struct nested_struct +test-source 2021-01-01 00:00:00 00000000-0000-0000-0000-000000000001 00000000-0000-0000-0000-000000000001 AQIDBA== true 2023-04-24 2023-04-24 "{""days"":1,""milliseconds"":1}" 123456789ms 123456789ns 123456789s 123456789us 1.1 1.1 192.0.2.0/24 -1 -1 -1 -1 AQIDBA== AString aa:bb:cc:dd:ee:ff "{""months"":1,""days"":1,""nanoseconds"":1}" 1 AString 00:00:00.000 00:00:00 00:00:00.000000000 00:00:00.000000 2021-01-02 00:00:00 2021-01-02 00:00:00 2021-01-02 00:00:00 2021-01-02 00:00:00 1 1 1 1 00000000-0000-0000-0000-000000000001 "{""test"":[""a"",""b"",3]}" "[{""test"":""test""},123,{""test_number"":456}]" [true,null,true] "[""2023-04-24"",null,""2023-04-24""]" "[""2023-04-24"",null,""2023-04-24""]" "[{""days"":1,""milliseconds"":1},null,{""days"":1,""milliseconds"":1}]" "[""123456789ms"",null,""123456789ms""]" "[""123456789ns"",null,""123456789ns""]" "[""123456789s"",null,""123456789s""]" "[""123456789us"",null,""123456789us""]" [1.1,null,1.1] [1.1,null,1.1] "[""192.0.2.0/24"",null,""192.0.2.0/24""]" [-1,null,-1] [-1,null,-1] [-1,null,-1] [-1,null,-1] "[""AString"",null,""AString""]" "[""aa:bb:cc:dd:ee:ff"",null,""aa:bb:cc:dd:ee:ff""]" "[{""months"":1,""days"":1,""nanoseconds"":1},null,{""months"":1,""days"":1,""nanoseconds"":1}]" "[{""months"":1},null,{""months"":1}]" "[""AString"",null,""AString""]" "[""00:00:00"",null,""00:00:00""]" "[""00:00:00"",null,""00:00:00""]" "[""00:00:00"",null,""00:00:00""]" "[""00:00:00"",null,""00:00:00""]" "[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]" "[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]" "[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]" "[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]" [1,null,1] [1,null,1] [1,null,1] [1,null,1] "[""00000000-0000-0000-0000-000000000001"",null,""00000000-0000-0000-0000-000000000001""]" "{""binary"":""AQIDBA=="",""boolean"":true,""date32"":""2023-04-24"",""date64"":""2023-04-24"",""daytimeinterval"":{""days"":1,""milliseconds"":1},""duration_ms"":""123456789ms"",""duration_ns"":""123456789ns"",""duration_s"":""123456789s"",""duration_us"":""123456789us"",""float32"":1.1,""float64"":1.1,""inet"":""192.0.2.0/24"",""int16"":-1,""int32"":-1,""int64"":-1,""int8"":-1,""json"":{""test"":[""a"",""b"",3]},""json_array"":[{""test"":""test""},123,{""test_number"":456}],""largebinary"":""AQIDBA=="",""largestring"":""AString"",""mac"":""aa:bb:cc:dd:ee:ff"",""monthdaynanointerval"":{""months"":1,""days"":1,""nanoseconds"":1},""monthinterval"":{""months"":1},""string"":""AString"",""time32ms"":""00:00:00"",""time32s"":""00:00:00"",""time64ns"":""00:00:00"",""time64us"":""00:00:00"",""timestamp_ms"":""2021-01-02 00:00:00"",""timestamp_ns"":""2021-01-02 00:00:00"",""timestamp_s"":""2021-01-02 00:00:00"",""timestamp_us"":""2021-01-02 00:00:00"",""uint16"":1,""uint32"":1,""uint64"":1,""uint8"":1,""uuid"":""00000000-0000-0000-0000-000000000001""}" "{""inner"":{""binary"":""AQIDBA=="",""boolean"":true,""date32"":""2023-04-24"",""date64"":""2023-04-24"",""daytimeinterval"":{""days"":1,""milliseconds"":1},""duration_ms"":""123456789ms"",""duration_ns"":""123456789ns"",""duration_s"":""123456789s"",""duration_us"":""123456789us"",""float32"":1.1,""float64"":1.1,""inet"":""192.0.2.0/24"",""int16"":-1,""int32"":-1,""int64"":-1,""int8"":-1,""json"":{""test"":[""a"",""b"",3]},""json_array"":[{""test"":""test""},123,{""test_number"":456}],""largebinary"":""AQIDBA=="",""largestring"":""AString"",""mac"":""aa:bb:cc:dd:ee:ff"",""monthdaynanointerval"":{""months"":1,""days"":1,""nanoseconds"":1},""monthinterval"":{""months"":1},""string"":""AString"",""time32ms"":""00:00:00"",""time32s"":""00:00:00"",""time64ns"":""00:00:00"",""time64us"":""00:00:00"",""timestamp_ms"":""2021-01-02 00:00:00"",""timestamp_ns"":""2021-01-02 00:00:00"",""timestamp_s"":""2021-01-02 00:00:00"",""timestamp_us"":""2021-01-02 00:00:00"",""uint16"":1,""uint32"":1,""uint64"":1,""uint8"":1,""uuid"":""00000000-0000-0000-0000-000000000001""}}" +test-source 2021-01-01 00:00:00 00000000-0000-0000-0000-000000000001 diff --git a/csv/testdata/TestWriteRead-with_headers.csv b/csv/testdata/TestWriteRead-with_headers.csv index 10247f36..97e430e5 100644 --- a/csv/testdata/TestWriteRead-with_headers.csv +++ b/csv/testdata/TestWriteRead-with_headers.csv @@ -1,4 +1,4 @@ -_cq_source_name,_cq_sync_time,_cq_id,_cq_parent_id,binary,boolean,date32,date64,daytimeinterval,duration_ms,duration_ns,duration_s,duration_us,float32,float64,inet,int16,int32,int64,int8,largebinary,largestring,mac,monthdaynanointerval,monthinterval,string,time32ms,time32s,time64ns,time64us,timestamp_ms,timestamp_ns,timestamp_s,timestamp_us,uint16,uint32,uint64,uint8,uuid,decimal,json,json_array,boolean_list,date32_list,date64_list,daytimeinterval_list,duration_ms_list,duration_ns_list,duration_s_list,duration_us_list,float32_list,float64_list,inet_list,int16_list,int32_list,int64_list,int8_list,largestring_list,mac_list,monthdaynanointerval_list,monthinterval_list,string_list,time32ms_list,time32s_list,time64ns_list,time64us_list,timestamp_ms_list,timestamp_ns_list,timestamp_s_list,timestamp_us_list,uint16_list,uint32_list,uint64_list,uint8_list,uuid_list,decimal_list,string_boolean_map,int_boolean_map,string_date32_map,int_date32_map,string_date64_map,int_date64_map,string_daytimeinterval_map,int_daytimeinterval_map,string_duration_ms_map,int_duration_ms_map,string_duration_ns_map,int_duration_ns_map,string_duration_s_map,int_duration_s_map,string_duration_us_map,int_duration_us_map,string_float32_map,int_float32_map,string_float64_map,int_float64_map,string_inet_map,int_inet_map,string_int16_map,int_int16_map,string_int32_map,int_int32_map,string_int64_map,int_int64_map,string_int8_map,int_int8_map,string_largestring_map,int_largestring_map,string_mac_map,int_mac_map,string_monthdaynanointerval_map,int_monthdaynanointerval_map,string_monthinterval_map,int_monthinterval_map,string_string_map,int_string_map,string_time32ms_map,int_time32ms_map,string_time32s_map,int_time32s_map,string_time64ns_map,int_time64ns_map,string_time64us_map,int_time64us_map,string_timestamp_ms_map,int_timestamp_ms_map,string_timestamp_ns_map,int_timestamp_ns_map,string_timestamp_s_map,int_timestamp_s_map,string_timestamp_us_map,int_timestamp_us_map,string_uint16_map,int_uint16_map,string_uint32_map,int_uint32_map,string_uint64_map,int_uint64_map,string_uint8_map,int_uint8_map,string_uuid_map,int_uuid_map,string_decimal_map,int_decimal_map,struct,nested_struct -test-source,2021-01-01 00:00:00,00000000-0000-0000-0000-000000000001,00000000-0000-0000-0000-000000000001,7049Ug==,true,2023-07-16,2023-07-16,"{""days"":6783,""milliseconds"":6783}",6476783ms,6476783ns,6476783s,6476783us,783.936,783.936,192.0.2.0/24,-30727,-1781555540,-5474557666971701248,-123,7049Ug==,AString76783,aa:bb:cc:dd:ee:ff,"{""months"":6783,""days"":6783,""nanoseconds"":6783}",1,AString76783,00:00:00.000,00:00:00,00:00:00.000000000,00:00:00.000000,2021-01-02 00:00:00,2021-01-02 00:00:00,2021-01-02 00:00:00,2021-01-02 00:00:00,63063,506910753,14697929703826477056,78,00000000-0000-0000-0000-000000000001,826476783.5364813824,"{""test"":[""a"",""b"",3]}","[{""test"":""test""},123,{""test_number"":456}]","[true,null,true]","[""2023-07-16"",null,""2023-07-16""]","[""2023-07-16"",null,""2023-07-16""]","[{""days"":6783,""milliseconds"":6783},null,{""days"":6783,""milliseconds"":6783}]","[""6476783ms"",null,""6476783ms""]","[""6476783ns"",null,""6476783ns""]","[""6476783s"",null,""6476783s""]","[""6476783us"",null,""6476783us""]","[783.936,null,783.936]","[783.936,null,783.936]","[""192.0.2.0/24"",null,""192.0.2.0/24""]","[-30727,null,-30727]","[-1781555540,null,-1781555540]","[-5474557666971701248,null,-5474557666971701248]","[-123,null,-123]","[""AString76783"",null,""AString76783""]","[""aa:bb:cc:dd:ee:ff"",null,""aa:bb:cc:dd:ee:ff""]","[{""months"":6783,""days"":6783,""nanoseconds"":6783},null,{""months"":6783,""days"":6783,""nanoseconds"":6783}]","[{""months"":1},null,{""months"":1}]","[""AString76783"",null,""AString76783""]","[""00:00:00"",null,""00:00:00""]","[""00:00:00"",null,""00:00:00""]","[""00:00:00"",null,""00:00:00""]","[""00:00:00"",null,""00:00:00""]","[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]","[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]","[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]","[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]","[63063,null,63063]","[506910753,null,506910753]","[14697929703826477056,null,14697929703826477056]","[78,null,78]","[""00000000-0000-0000-0000-000000000001"",null,""00000000-0000-0000-0000-000000000001""]","[""826476783.5364813824"",null,""826476783.5364813824""]","[{""key"":""AString76783"",""value"":true},{""key"":""AString89351"",""value"":true}]","[{""key"":-5474557666971701248,""value"":true},{""key"":-3812638179211989504,""value"":true}]","[{""key"":""AString76783"",""value"":""2023-07-16""},{""key"":""AString89351"",""value"":""2023-06-14""}]","[{""key"":-5474557666971701248,""value"":""2023-07-16""},{""key"":-3812638179211989504,""value"":""2023-06-14""}]","[{""key"":""AString76783"",""value"":""2023-07-16""},{""key"":""AString89351"",""value"":""2023-06-14""}]","[{""key"":-5474557666971701248,""value"":""2023-07-16""},{""key"":-3812638179211989504,""value"":""2023-06-14""}]","[{""key"":""AString76783"",""value"":{""days"":6783,""milliseconds"":6783}},{""key"":""AString89351"",""value"":{""days"":9351,""milliseconds"":9351}}]","[{""key"":-5474557666971701248,""value"":{""days"":6783,""milliseconds"":6783}},{""key"":-3812638179211989504,""value"":{""days"":9351,""milliseconds"":9351}}]","[{""key"":""AString76783"",""value"":""6476783ms""},{""key"":""AString89351"",""value"":""1989351ms""}]","[{""key"":-5474557666971701248,""value"":""6476783ms""},{""key"":-3812638179211989504,""value"":""1989351ms""}]","[{""key"":""AString76783"",""value"":""6476783ns""},{""key"":""AString89351"",""value"":""1989351ns""}]","[{""key"":-5474557666971701248,""value"":""6476783ns""},{""key"":-3812638179211989504,""value"":""1989351ns""}]","[{""key"":""AString76783"",""value"":""6476783s""},{""key"":""AString89351"",""value"":""1989351s""}]","[{""key"":-5474557666971701248,""value"":""6476783s""},{""key"":-3812638179211989504,""value"":""1989351s""}]","[{""key"":""AString76783"",""value"":""6476783us""},{""key"":""AString89351"",""value"":""1989351us""}]","[{""key"":-5474557666971701248,""value"":""6476783us""},{""key"":-3812638179211989504,""value"":""1989351us""}]","[{""key"":""AString76783"",""value"":783.936},{""key"":""AString89351"",""value"":351.121}]","[{""key"":-5474557666971701248,""value"":783.936},{""key"":-3812638179211989504,""value"":351.121}]","[{""key"":""AString76783"",""value"":783.936},{""key"":""AString89351"",""value"":351.121}]","[{""key"":-5474557666971701248,""value"":783.936},{""key"":-3812638179211989504,""value"":351.121}]","[{""key"":""AString76783"",""value"":""192.0.2.0/24""},{""key"":""AString89351"",""value"":""192.0.2.0/24""}]","[{""key"":-5474557666971701248,""value"":""192.0.2.0/24""},{""key"":-3812638179211989504,""value"":""192.0.2.0/24""}]","[{""key"":""AString76783"",""value"":-30727},{""key"":""AString89351"",""value"":-25508}]","[{""key"":-5474557666971701248,""value"":-30727},{""key"":-3812638179211989504,""value"":-25508}]","[{""key"":""AString76783"",""value"":-1781555540},{""key"":""AString89351"",""value"":-853032410}]","[{""key"":-5474557666971701248,""value"":-1781555540},{""key"":-3812638179211989504,""value"":-853032410}]","[{""key"":""AString76783"",""value"":-5474557666971701248},{""key"":""AString89351"",""value"":-3812638179211989504}]","[{""key"":-5474557666971701248,""value"":-5474557666971701248},{""key"":-3812638179211989504,""value"":-3812638179211989504}]","[{""key"":""AString76783"",""value"":-123},{""key"":""AString89351"",""value"":-60}]","[{""key"":-5474557666971701248,""value"":-123},{""key"":-3812638179211989504,""value"":-60}]","[{""key"":""AString76783"",""value"":""AString76783""},{""key"":""AString89351"",""value"":""AString89351""}]","[{""key"":-5474557666971701248,""value"":""AString76783""},{""key"":-3812638179211989504,""value"":""AString89351""}]","[{""key"":""AString76783"",""value"":""aa:bb:cc:dd:ee:ff""},{""key"":""AString89351"",""value"":""aa:bb:cc:dd:ee:ff""}]","[{""key"":-5474557666971701248,""value"":""aa:bb:cc:dd:ee:ff""},{""key"":-3812638179211989504,""value"":""aa:bb:cc:dd:ee:ff""}]","[{""key"":""AString76783"",""value"":{""months"":6783,""days"":6783,""nanoseconds"":6783}},{""key"":""AString89351"",""value"":{""months"":9351,""days"":9351,""nanoseconds"":9351}}]","[{""key"":-5474557666971701248,""value"":{""months"":6783,""days"":6783,""nanoseconds"":6783}},{""key"":-3812638179211989504,""value"":{""months"":9351,""days"":9351,""nanoseconds"":9351}}]","[{""key"":""AString76783"",""value"":{""months"":1}},{""key"":""AString89351"",""value"":{""months"":1}}]","[{""key"":-5474557666971701248,""value"":{""months"":1}},{""key"":-3812638179211989504,""value"":{""months"":1}}]","[{""key"":""AString76783"",""value"":""AString76783""},{""key"":""AString89351"",""value"":""AString89351""}]","[{""key"":-5474557666971701248,""value"":""AString76783""},{""key"":-3812638179211989504,""value"":""AString89351""}]","[{""key"":""AString76783"",""value"":""00:00:00""},{""key"":""AString89351"",""value"":""00:00:00""}]","[{""key"":-5474557666971701248,""value"":""00:00:00""},{""key"":-3812638179211989504,""value"":""00:00:00""}]","[{""key"":""AString76783"",""value"":""00:00:00""},{""key"":""AString89351"",""value"":""00:00:00""}]","[{""key"":-5474557666971701248,""value"":""00:00:00""},{""key"":-3812638179211989504,""value"":""00:00:00""}]","[{""key"":""AString76783"",""value"":""00:00:00""},{""key"":""AString89351"",""value"":""00:00:00""}]","[{""key"":-5474557666971701248,""value"":""00:00:00""},{""key"":-3812638179211989504,""value"":""00:00:00""}]","[{""key"":""AString76783"",""value"":""00:00:00""},{""key"":""AString89351"",""value"":""00:00:00""}]","[{""key"":-5474557666971701248,""value"":""00:00:00""},{""key"":-3812638179211989504,""value"":""00:00:00""}]","[{""key"":""AString76783"",""value"":""2021-01-02 00:00:00""},{""key"":""AString89351"",""value"":""2021-01-02 00:00:00""}]","[{""key"":-5474557666971701248,""value"":""2021-01-02 00:00:00""},{""key"":-3812638179211989504,""value"":""2021-01-02 00:00:00""}]","[{""key"":""AString76783"",""value"":""2021-01-02 00:00:00""},{""key"":""AString89351"",""value"":""2021-01-02 00:00:00""}]","[{""key"":-5474557666971701248,""value"":""2021-01-02 00:00:00""},{""key"":-3812638179211989504,""value"":""2021-01-02 00:00:00""}]","[{""key"":""AString76783"",""value"":""2021-01-02 00:00:00""},{""key"":""AString89351"",""value"":""2021-01-02 00:00:00""}]","[{""key"":-5474557666971701248,""value"":""2021-01-02 00:00:00""},{""key"":-3812638179211989504,""value"":""2021-01-02 00:00:00""}]","[{""key"":""AString76783"",""value"":""2021-01-02 00:00:00""},{""key"":""AString89351"",""value"":""2021-01-02 00:00:00""}]","[{""key"":-5474557666971701248,""value"":""2021-01-02 00:00:00""},{""key"":-3812638179211989504,""value"":""2021-01-02 00:00:00""}]","[{""key"":""AString76783"",""value"":63063},{""key"":""AString89351"",""value"":34191}]","[{""key"":-5474557666971701248,""value"":63063},{""key"":-3812638179211989504,""value"":34191}]","[{""key"":""AString76783"",""value"":506910753},{""key"":""AString89351"",""value"":2112817056}]","[{""key"":-5474557666971701248,""value"":506910753},{""key"":-3812638179211989504,""value"":2112817056}]","[{""key"":""AString76783"",""value"":14697929703826477056},{""key"":""AString89351"",""value"":3812638179211989504}]","[{""key"":-5474557666971701248,""value"":14697929703826477056},{""key"":-3812638179211989504,""value"":3812638179211989504}]","[{""key"":""AString76783"",""value"":78},{""key"":""AString89351"",""value"":21}]","[{""key"":-5474557666971701248,""value"":78},{""key"":-3812638179211989504,""value"":21}]","[{""key"":""AString76783"",""value"":""00000000-0000-0000-0000-000000000001""},{""key"":""AString89351"",""value"":""00000000-0000-0000-0000-000000000001""}]","[{""key"":-5474557666971701248,""value"":""00000000-0000-0000-0000-000000000001""},{""key"":-3812638179211989504,""value"":""00000000-0000-0000-0000-000000000001""}]","[{""key"":""AString76783"",""value"":""826476783.5364813824""},{""key"":""AString89351"",""value"":""211989351.8533091328""}]","[{""key"":-5474557666971701248,""value"":""826476783.5364813824""},{""key"":-3812638179211989504,""value"":""211989351.8533091328""}]","{""binary"":""7049Ug=="",""boolean"":true,""date32"":""2023-07-16"",""date64"":""2023-07-16"",""daytimeinterval"":{""days"":6783,""milliseconds"":6783},""decimal"":""826476783.5364813824"",""duration_ms"":""6476783ms"",""duration_ns"":""6476783ns"",""duration_s"":""6476783s"",""duration_us"":""6476783us"",""float32"":783.936,""float64"":783.936,""inet"":""192.0.2.0/24"",""int16"":-30727,""int32"":-1781555540,""int64"":-5474557666971701248,""int8"":-123,""json"":{""test"":[""a"",""b"",3]},""json_array"":[{""test"":""test""},123,{""test_number"":456}],""largebinary"":""7049Ug=="",""largestring"":""AString76783"",""mac"":""aa:bb:cc:dd:ee:ff"",""monthdaynanointerval"":{""months"":6783,""days"":6783,""nanoseconds"":6783},""monthinterval"":{""months"":1},""string"":""AString76783"",""time32ms"":""00:00:00"",""time32s"":""00:00:00"",""time64ns"":""00:00:00"",""time64us"":""00:00:00"",""timestamp_ms"":""2021-01-02 00:00:00"",""timestamp_ns"":""2021-01-02 00:00:00"",""timestamp_s"":""2021-01-02 00:00:00"",""timestamp_us"":""2021-01-02 00:00:00"",""uint16"":63063,""uint32"":506910753,""uint64"":14697929703826477056,""uint8"":78,""uuid"":""00000000-0000-0000-0000-000000000001""}","{""inner"":{""binary"":""7049Ug=="",""boolean"":true,""date32"":""2023-07-16"",""date64"":""2023-07-16"",""daytimeinterval"":{""days"":6783,""milliseconds"":6783},""decimal"":""826476783.5364813824"",""duration_ms"":""6476783ms"",""duration_ns"":""6476783ns"",""duration_s"":""6476783s"",""duration_us"":""6476783us"",""float32"":783.936,""float64"":783.936,""inet"":""192.0.2.0/24"",""int16"":-30727,""int32"":-1781555540,""int64"":-5474557666971701248,""int8"":-123,""json"":{""test"":[""a"",""b"",3]},""json_array"":[{""test"":""test""},123,{""test_number"":456}],""largebinary"":""7049Ug=="",""largestring"":""AString76783"",""mac"":""aa:bb:cc:dd:ee:ff"",""monthdaynanointerval"":{""months"":6783,""days"":6783,""nanoseconds"":6783},""monthinterval"":{""months"":1},""string"":""AString76783"",""time32ms"":""00:00:00"",""time32s"":""00:00:00"",""time64ns"":""00:00:00"",""time64us"":""00:00:00"",""timestamp_ms"":""2021-01-02 00:00:00"",""timestamp_ns"":""2021-01-02 00:00:00"",""timestamp_s"":""2021-01-02 00:00:00"",""timestamp_us"":""2021-01-02 00:00:00"",""uint16"":63063,""uint32"":506910753,""uint64"":14697929703826477056,""uint8"":78,""uuid"":""00000000-0000-0000-0000-000000000001""}}" -test-source,2021-01-01 00:00:00,00000000-0000-0000-0000-000000000001,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +_cq_source_name,_cq_sync_time,_cq_id,_cq_parent_id,binary,boolean,date32,date64,daytimeinterval,duration_ms,duration_ns,duration_s,duration_us,float32,float64,inet,int16,int32,int64,int8,largebinary,largestring,mac,monthdaynanointerval,monthinterval,string,time32ms,time32s,time64ns,time64us,timestamp_ms,timestamp_ns,timestamp_s,timestamp_us,uint16,uint32,uint64,uint8,uuid,json,json_array,boolean_list,date32_list,date64_list,daytimeinterval_list,duration_ms_list,duration_ns_list,duration_s_list,duration_us_list,float32_list,float64_list,inet_list,int16_list,int32_list,int64_list,int8_list,largestring_list,mac_list,monthdaynanointerval_list,monthinterval_list,string_list,time32ms_list,time32s_list,time64ns_list,time64us_list,timestamp_ms_list,timestamp_ns_list,timestamp_s_list,timestamp_us_list,uint16_list,uint32_list,uint64_list,uint8_list,uuid_list,struct,nested_struct +test-source,2021-01-01 00:00:00,00000000-0000-0000-0000-000000000001,00000000-0000-0000-0000-000000000001,AQIDBA==,true,2023-04-24,2023-04-24,"{""days"":1,""milliseconds"":1}",123456789ms,123456789ns,123456789s,123456789us,1.1,1.1,192.0.2.0/24,-1,-1,-1,-1,AQIDBA==,AString,aa:bb:cc:dd:ee:ff,"{""months"":1,""days"":1,""nanoseconds"":1}",1,AString,00:00:00.000,00:00:00,00:00:00.000000000,00:00:00.000000,2021-01-02 00:00:00,2021-01-02 00:00:00,2021-01-02 00:00:00,2021-01-02 00:00:00,1,1,1,1,00000000-0000-0000-0000-000000000001,"{""test"":[""a"",""b"",3]}","[{""test"":""test""},123,{""test_number"":456}]","[true,null,true]","[""2023-04-24"",null,""2023-04-24""]","[""2023-04-24"",null,""2023-04-24""]","[{""days"":1,""milliseconds"":1},null,{""days"":1,""milliseconds"":1}]","[""123456789ms"",null,""123456789ms""]","[""123456789ns"",null,""123456789ns""]","[""123456789s"",null,""123456789s""]","[""123456789us"",null,""123456789us""]","[1.1,null,1.1]","[1.1,null,1.1]","[""192.0.2.0/24"",null,""192.0.2.0/24""]","[-1,null,-1]","[-1,null,-1]","[-1,null,-1]","[-1,null,-1]","[""AString"",null,""AString""]","[""aa:bb:cc:dd:ee:ff"",null,""aa:bb:cc:dd:ee:ff""]","[{""months"":1,""days"":1,""nanoseconds"":1},null,{""months"":1,""days"":1,""nanoseconds"":1}]","[{""months"":1},null,{""months"":1}]","[""AString"",null,""AString""]","[""00:00:00"",null,""00:00:00""]","[""00:00:00"",null,""00:00:00""]","[""00:00:00"",null,""00:00:00""]","[""00:00:00"",null,""00:00:00""]","[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]","[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]","[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]","[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]","[1,null,1]","[1,null,1]","[1,null,1]","[1,null,1]","[""00000000-0000-0000-0000-000000000001"",null,""00000000-0000-0000-0000-000000000001""]","{""binary"":""AQIDBA=="",""boolean"":true,""date32"":""2023-04-24"",""date64"":""2023-04-24"",""daytimeinterval"":{""days"":1,""milliseconds"":1},""duration_ms"":""123456789ms"",""duration_ns"":""123456789ns"",""duration_s"":""123456789s"",""duration_us"":""123456789us"",""float32"":1.1,""float64"":1.1,""inet"":""192.0.2.0/24"",""int16"":-1,""int32"":-1,""int64"":-1,""int8"":-1,""json"":{""test"":[""a"",""b"",3]},""json_array"":[{""test"":""test""},123,{""test_number"":456}],""largebinary"":""AQIDBA=="",""largestring"":""AString"",""mac"":""aa:bb:cc:dd:ee:ff"",""monthdaynanointerval"":{""months"":1,""days"":1,""nanoseconds"":1},""monthinterval"":{""months"":1},""string"":""AString"",""time32ms"":""00:00:00"",""time32s"":""00:00:00"",""time64ns"":""00:00:00"",""time64us"":""00:00:00"",""timestamp_ms"":""2021-01-02 00:00:00"",""timestamp_ns"":""2021-01-02 00:00:00"",""timestamp_s"":""2021-01-02 00:00:00"",""timestamp_us"":""2021-01-02 00:00:00"",""uint16"":1,""uint32"":1,""uint64"":1,""uint8"":1,""uuid"":""00000000-0000-0000-0000-000000000001""}","{""inner"":{""binary"":""AQIDBA=="",""boolean"":true,""date32"":""2023-04-24"",""date64"":""2023-04-24"",""daytimeinterval"":{""days"":1,""milliseconds"":1},""duration_ms"":""123456789ms"",""duration_ns"":""123456789ns"",""duration_s"":""123456789s"",""duration_us"":""123456789us"",""float32"":1.1,""float64"":1.1,""inet"":""192.0.2.0/24"",""int16"":-1,""int32"":-1,""int64"":-1,""int8"":-1,""json"":{""test"":[""a"",""b"",3]},""json_array"":[{""test"":""test""},123,{""test_number"":456}],""largebinary"":""AQIDBA=="",""largestring"":""AString"",""mac"":""aa:bb:cc:dd:ee:ff"",""monthdaynanointerval"":{""months"":1,""days"":1,""nanoseconds"":1},""monthinterval"":{""months"":1},""string"":""AString"",""time32ms"":""00:00:00"",""time32s"":""00:00:00"",""time64ns"":""00:00:00"",""time64us"":""00:00:00"",""timestamp_ms"":""2021-01-02 00:00:00"",""timestamp_ns"":""2021-01-02 00:00:00"",""timestamp_s"":""2021-01-02 00:00:00"",""timestamp_us"":""2021-01-02 00:00:00"",""uint16"":1,""uint32"":1,""uint64"":1,""uint8"":1,""uuid"":""00000000-0000-0000-0000-000000000001""}}" +test-source,2021-01-01 00:00:00,00000000-0000-0000-0000-000000000001,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, diff --git a/csv/write.go b/csv/write.go index 1d0f14c1..cc0895be 100644 --- a/csv/write.go +++ b/csv/write.go @@ -9,8 +9,8 @@ import ( "github.com/apache/arrow/go/v13/arrow/array" "github.com/apache/arrow/go/v13/arrow/csv" "github.com/apache/arrow/go/v13/arrow/memory" - "github.com/cloudquery/filetypes/v3/types" - "github.com/cloudquery/plugin-sdk/v3/schema" + "github.com/cloudquery/filetypes/v4/types" + "github.com/cloudquery/plugin-sdk/v4/schema" ) type Handle struct { diff --git a/csv/write_read_test.go b/csv/write_read_test.go index d5abfc76..0b4af948 100644 --- a/csv/write_read_test.go +++ b/csv/write_read_test.go @@ -9,9 +9,9 @@ import ( "github.com/apache/arrow/go/v13/arrow" "github.com/bradleyjkemp/cupaloy/v2" - "github.com/cloudquery/filetypes/v3/types" - "github.com/cloudquery/plugin-sdk/v3/plugins/destination" - "github.com/cloudquery/plugin-sdk/v3/schema" + "github.com/cloudquery/filetypes/v4/types" + "github.com/cloudquery/plugin-sdk/v4/plugin" + "github.com/cloudquery/plugin-sdk/v4/schema" "github.com/google/uuid" ) @@ -74,7 +74,7 @@ func TestWriteRead(t *testing.T) { }() totalCount := 0 for got := range ch { - if diff := destination.RecordDiff(records[totalCount], got); diff != "" { + if diff := plugin.RecordDiff(records[totalCount], got); diff != "" { t.Errorf("got diff: %s", diff) } totalCount++ diff --git a/go.mod b/go.mod index 29c061d1..0bc505c3 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,11 @@ -module github.com/cloudquery/filetypes/v3 +module github.com/cloudquery/filetypes/v4 go 1.19 require ( - github.com/apache/arrow/go/v13 v13.0.0-20230601070034-e07e22c5580a + github.com/apache/arrow/go/v13 v13.0.0-20230601214540-018e7d3f9c4b github.com/bradleyjkemp/cupaloy/v2 v2.8.0 - github.com/cloudquery/plugin-sdk/v3 v3.10.6 + github.com/cloudquery/plugin-sdk/v4 v4.0.0 github.com/goccy/go-json v0.10.2 github.com/google/uuid v1.3.0 github.com/stretchr/testify v1.8.4 @@ -14,13 +14,13 @@ require ( // TODO: remove once all updates are merged replace github.com/apache/arrow/go/v13 => github.com/cloudquery/arrow/go/v13 v13.0.0-20230619001325-b0832beeeade +replace github.com/cloudquery/plugin-sdk/v4 => github.com/cloudquery/plugin-sdk/v4 v4.0.0-20230617094031-84a6a9b7e057 + require ( github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect github.com/andybalholm/brotli v1.0.5 // indirect github.com/apache/thrift v0.18.1 // indirect - github.com/cloudquery/plugin-pb-go v1.0.9 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/ghodss/yaml v1.0.0 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/flatbuffers v23.1.21+incompatible // indirect @@ -47,6 +47,5 @@ require ( google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect google.golang.org/grpc v1.55.0 // indirect google.golang.org/protobuf v1.30.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 3d2d6d8d..96d86bf5 100644 --- a/go.sum +++ b/go.sum @@ -8,16 +8,12 @@ github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oM github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= github.com/cloudquery/arrow/go/v13 v13.0.0-20230619001325-b0832beeeade h1:4vkgraWxeK51HYQsGjQ0TTuVxTAxaUh9GLfXK1MiOJE= github.com/cloudquery/arrow/go/v13 v13.0.0-20230619001325-b0832beeeade/go.mod h1:W69eByFNO0ZR30q1/7Sr9d83zcVZmF2MiP3fFYAWJOc= -github.com/cloudquery/plugin-pb-go v1.0.9 h1:1apMGcRdBCc1N6/0IQD3UtHm7XHMCPyV6mNnklOHETA= -github.com/cloudquery/plugin-pb-go v1.0.9/go.mod h1:iQwZ/jiKmoSu2H8U5otCX9gCMJtAOkb3HAnGIKvg0oU= -github.com/cloudquery/plugin-sdk/v3 v3.10.6 h1:KqTsLZ6OA1h8BUMeMcU6BAD6TBW6ojgQaC4zDZMgvu0= -github.com/cloudquery/plugin-sdk/v3 v3.10.6/go.mod h1:QhBaVgiNyQ3P6uAzJWOYpYykHXL+WDZffwg1riTwv60= +github.com/cloudquery/plugin-sdk/v4 v4.0.0-20230617094031-84a6a9b7e057 h1:etJDrg7YcQmJvGP/thomMgGSzlIRKRbqeb9XiSsdBuk= +github.com/cloudquery/plugin-sdk/v4 v4.0.0-20230617094031-84a6a9b7e057/go.mod h1:C/OPmi3Jkded0V53qE4uDrjiKWGrn8pnb9jWcilimBw= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -103,8 +99,6 @@ google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/json/read.go b/json/read.go index d3b5b0a8..b212bfe8 100644 --- a/json/read.go +++ b/json/read.go @@ -7,7 +7,7 @@ import ( "github.com/apache/arrow/go/v13/arrow" "github.com/apache/arrow/go/v13/arrow/array" "github.com/apache/arrow/go/v13/arrow/memory" - "github.com/cloudquery/plugin-sdk/v3/schema" + "github.com/cloudquery/plugin-sdk/v4/schema" ) const maxJSONSize = 1024 * 1024 * 20 diff --git a/json/testdata/TestWriteRead.jsonl b/json/testdata/TestWriteRead.jsonl index 566c6f41..aaf04aaf 100644 --- a/json/testdata/TestWriteRead.jsonl +++ b/json/testdata/TestWriteRead.jsonl @@ -1,3 +1,3 @@ -{"_cq_id":"00000000-0000-0000-0000-000000000001","_cq_parent_id":"00000000-0000-0000-0000-000000000001","_cq_source_name":"test-source","_cq_sync_time":"2021-01-01 00:00:00","binary":"7049Ug==","boolean":true,"boolean_list":[true,null,true],"date32":"2023-07-16","date32_list":["2023-07-16",null,"2023-07-16"],"date64":"2023-07-16","date64_list":["2023-07-16",null,"2023-07-16"],"daytimeinterval":{"days":6783,"milliseconds":6783},"daytimeinterval_list":[{"days":6783,"milliseconds":6783},null,{"days":6783,"milliseconds":6783}],"decimal":"826476783.5364813824","decimal_list":["826476783.5364813824",null,"826476783.5364813824"],"duration_ms":"6476783ms","duration_ms_list":["6476783ms",null,"6476783ms"],"duration_ns":"6476783ns","duration_ns_list":["6476783ns",null,"6476783ns"],"duration_s":"6476783s","duration_s_list":["6476783s",null,"6476783s"],"duration_us":"6476783us","duration_us_list":["6476783us",null,"6476783us"],"float32":783.936,"float32_list":[783.936,null,783.936],"float64":783.936,"float64_list":[783.936,null,783.936],"inet":"192.0.2.0/24","inet_list":["192.0.2.0/24",null,"192.0.2.0/24"],"int16":-30727,"int16_list":[-30727,null,-30727],"int32":-1781555540,"int32_list":[-1781555540,null,-1781555540],"int64":-5474557666971701248,"int64_list":[-5474557666971701248,null,-5474557666971701248],"int8":-123,"int8_list":[-123,null,-123],"int_boolean_map":[{"key":-5474557666971701248,"value":true},{"key":-3812638179211989504,"value":true}],"int_date32_map":[{"key":-5474557666971701248,"value":"2023-07-16"},{"key":-3812638179211989504,"value":"2023-06-14"}],"int_date64_map":[{"key":-5474557666971701248,"value":"2023-07-16"},{"key":-3812638179211989504,"value":"2023-06-14"}],"int_daytimeinterval_map":[{"key":-5474557666971701248,"value":{"days":6783,"milliseconds":6783}},{"key":-3812638179211989504,"value":{"days":9351,"milliseconds":9351}}],"int_decimal_map":[{"key":-5474557666971701248,"value":"826476783.5364813824"},{"key":-3812638179211989504,"value":"211989351.8533091328"}],"int_duration_ms_map":[{"key":-5474557666971701248,"value":"6476783ms"},{"key":-3812638179211989504,"value":"1989351ms"}],"int_duration_ns_map":[{"key":-5474557666971701248,"value":"6476783ns"},{"key":-3812638179211989504,"value":"1989351ns"}],"int_duration_s_map":[{"key":-5474557666971701248,"value":"6476783s"},{"key":-3812638179211989504,"value":"1989351s"}],"int_duration_us_map":[{"key":-5474557666971701248,"value":"6476783us"},{"key":-3812638179211989504,"value":"1989351us"}],"int_float32_map":[{"key":-5474557666971701248,"value":783.936},{"key":-3812638179211989504,"value":351.121}],"int_float64_map":[{"key":-5474557666971701248,"value":783.936},{"key":-3812638179211989504,"value":351.121}],"int_inet_map":[{"key":-5474557666971701248,"value":"192.0.2.0/24"},{"key":-3812638179211989504,"value":"192.0.2.0/24"}],"int_int16_map":[{"key":-5474557666971701248,"value":-30727},{"key":-3812638179211989504,"value":-25508}],"int_int32_map":[{"key":-5474557666971701248,"value":-1781555540},{"key":-3812638179211989504,"value":-853032410}],"int_int64_map":[{"key":-5474557666971701248,"value":-5474557666971701248},{"key":-3812638179211989504,"value":-3812638179211989504}],"int_int8_map":[{"key":-5474557666971701248,"value":-123},{"key":-3812638179211989504,"value":-60}],"int_largestring_map":[{"key":-5474557666971701248,"value":"AString76783"},{"key":-3812638179211989504,"value":"AString89351"}],"int_mac_map":[{"key":-5474557666971701248,"value":"aa:bb:cc:dd:ee:ff"},{"key":-3812638179211989504,"value":"aa:bb:cc:dd:ee:ff"}],"int_monthdaynanointerval_map":[{"key":-5474557666971701248,"value":{"months":6783,"days":6783,"nanoseconds":6783}},{"key":-3812638179211989504,"value":{"months":9351,"days":9351,"nanoseconds":9351}}],"int_monthinterval_map":[{"key":-5474557666971701248,"value":{"months":1}},{"key":-3812638179211989504,"value":{"months":1}}],"int_string_map":[{"key":-5474557666971701248,"value":"AString76783"},{"key":-3812638179211989504,"value":"AString89351"}],"int_time32ms_map":[{"key":-5474557666971701248,"value":"00:00:00"},{"key":-3812638179211989504,"value":"00:00:00"}],"int_time32s_map":[{"key":-5474557666971701248,"value":"00:00:00"},{"key":-3812638179211989504,"value":"00:00:00"}],"int_time64ns_map":[{"key":-5474557666971701248,"value":"00:00:00"},{"key":-3812638179211989504,"value":"00:00:00"}],"int_time64us_map":[{"key":-5474557666971701248,"value":"00:00:00"},{"key":-3812638179211989504,"value":"00:00:00"}],"int_timestamp_ms_map":[{"key":-5474557666971701248,"value":"2021-01-02 00:00:00"},{"key":-3812638179211989504,"value":"2021-01-02 00:00:00"}],"int_timestamp_ns_map":[{"key":-5474557666971701248,"value":"2021-01-02 00:00:00"},{"key":-3812638179211989504,"value":"2021-01-02 00:00:00"}],"int_timestamp_s_map":[{"key":-5474557666971701248,"value":"2021-01-02 00:00:00"},{"key":-3812638179211989504,"value":"2021-01-02 00:00:00"}],"int_timestamp_us_map":[{"key":-5474557666971701248,"value":"2021-01-02 00:00:00"},{"key":-3812638179211989504,"value":"2021-01-02 00:00:00"}],"int_uint16_map":[{"key":-5474557666971701248,"value":63063},{"key":-3812638179211989504,"value":34191}],"int_uint32_map":[{"key":-5474557666971701248,"value":506910753},{"key":-3812638179211989504,"value":2112817056}],"int_uint64_map":[{"key":-5474557666971701248,"value":14697929703826477056},{"key":-3812638179211989504,"value":3812638179211989504}],"int_uint8_map":[{"key":-5474557666971701248,"value":78},{"key":-3812638179211989504,"value":21}],"int_uuid_map":[{"key":-5474557666971701248,"value":"00000000-0000-0000-0000-000000000001"},{"key":-3812638179211989504,"value":"00000000-0000-0000-0000-000000000001"}],"json":{"test":["a","b",3]},"json_array":[{"test":"test"},123,{"test_number":456}],"largebinary":"7049Ug==","largestring":"AString76783","largestring_list":["AString76783",null,"AString76783"],"mac":"aa:bb:cc:dd:ee:ff","mac_list":["aa:bb:cc:dd:ee:ff",null,"aa:bb:cc:dd:ee:ff"],"monthdaynanointerval":{"months":6783,"days":6783,"nanoseconds":6783},"monthdaynanointerval_list":[{"months":6783,"days":6783,"nanoseconds":6783},null,{"months":6783,"days":6783,"nanoseconds":6783}],"monthinterval":{"months":1},"monthinterval_list":[{"months":1},null,{"months":1}],"nested_struct":{"inner":{"binary":"7049Ug==","boolean":true,"date32":"2023-07-16","date64":"2023-07-16","daytimeinterval":{"days":6783,"milliseconds":6783},"decimal":"826476783.5364813824","duration_ms":"6476783ms","duration_ns":"6476783ns","duration_s":"6476783s","duration_us":"6476783us","float32":783.936,"float64":783.936,"inet":"192.0.2.0/24","int16":-30727,"int32":-1781555540,"int64":-5474557666971701248,"int8":-123,"json":{"test":["a","b",3]},"json_array":[{"test":"test"},123,{"test_number":456}],"largebinary":"7049Ug==","largestring":"AString76783","mac":"aa:bb:cc:dd:ee:ff","monthdaynanointerval":{"months":6783,"days":6783,"nanoseconds":6783},"monthinterval":{"months":1},"string":"AString76783","time32ms":"00:00:00","time32s":"00:00:00","time64ns":"00:00:00","time64us":"00:00:00","timestamp_ms":"2021-01-02 00:00:00","timestamp_ns":"2021-01-02 00:00:00","timestamp_s":"2021-01-02 00:00:00","timestamp_us":"2021-01-02 00:00:00","uint16":63063,"uint32":506910753,"uint64":14697929703826477056,"uint8":78,"uuid":"00000000-0000-0000-0000-000000000001"}},"string":"AString76783","string_boolean_map":[{"key":"AString76783","value":true},{"key":"AString89351","value":true}],"string_date32_map":[{"key":"AString76783","value":"2023-07-16"},{"key":"AString89351","value":"2023-06-14"}],"string_date64_map":[{"key":"AString76783","value":"2023-07-16"},{"key":"AString89351","value":"2023-06-14"}],"string_daytimeinterval_map":[{"key":"AString76783","value":{"days":6783,"milliseconds":6783}},{"key":"AString89351","value":{"days":9351,"milliseconds":9351}}],"string_decimal_map":[{"key":"AString76783","value":"826476783.5364813824"},{"key":"AString89351","value":"211989351.8533091328"}],"string_duration_ms_map":[{"key":"AString76783","value":"6476783ms"},{"key":"AString89351","value":"1989351ms"}],"string_duration_ns_map":[{"key":"AString76783","value":"6476783ns"},{"key":"AString89351","value":"1989351ns"}],"string_duration_s_map":[{"key":"AString76783","value":"6476783s"},{"key":"AString89351","value":"1989351s"}],"string_duration_us_map":[{"key":"AString76783","value":"6476783us"},{"key":"AString89351","value":"1989351us"}],"string_float32_map":[{"key":"AString76783","value":783.936},{"key":"AString89351","value":351.121}],"string_float64_map":[{"key":"AString76783","value":783.936},{"key":"AString89351","value":351.121}],"string_inet_map":[{"key":"AString76783","value":"192.0.2.0/24"},{"key":"AString89351","value":"192.0.2.0/24"}],"string_int16_map":[{"key":"AString76783","value":-30727},{"key":"AString89351","value":-25508}],"string_int32_map":[{"key":"AString76783","value":-1781555540},{"key":"AString89351","value":-853032410}],"string_int64_map":[{"key":"AString76783","value":-5474557666971701248},{"key":"AString89351","value":-3812638179211989504}],"string_int8_map":[{"key":"AString76783","value":-123},{"key":"AString89351","value":-60}],"string_largestring_map":[{"key":"AString76783","value":"AString76783"},{"key":"AString89351","value":"AString89351"}],"string_list":["AString76783",null,"AString76783"],"string_mac_map":[{"key":"AString76783","value":"aa:bb:cc:dd:ee:ff"},{"key":"AString89351","value":"aa:bb:cc:dd:ee:ff"}],"string_monthdaynanointerval_map":[{"key":"AString76783","value":{"months":6783,"days":6783,"nanoseconds":6783}},{"key":"AString89351","value":{"months":9351,"days":9351,"nanoseconds":9351}}],"string_monthinterval_map":[{"key":"AString76783","value":{"months":1}},{"key":"AString89351","value":{"months":1}}],"string_string_map":[{"key":"AString76783","value":"AString76783"},{"key":"AString89351","value":"AString89351"}],"string_time32ms_map":[{"key":"AString76783","value":"00:00:00"},{"key":"AString89351","value":"00:00:00"}],"string_time32s_map":[{"key":"AString76783","value":"00:00:00"},{"key":"AString89351","value":"00:00:00"}],"string_time64ns_map":[{"key":"AString76783","value":"00:00:00"},{"key":"AString89351","value":"00:00:00"}],"string_time64us_map":[{"key":"AString76783","value":"00:00:00"},{"key":"AString89351","value":"00:00:00"}],"string_timestamp_ms_map":[{"key":"AString76783","value":"2021-01-02 00:00:00"},{"key":"AString89351","value":"2021-01-02 00:00:00"}],"string_timestamp_ns_map":[{"key":"AString76783","value":"2021-01-02 00:00:00"},{"key":"AString89351","value":"2021-01-02 00:00:00"}],"string_timestamp_s_map":[{"key":"AString76783","value":"2021-01-02 00:00:00"},{"key":"AString89351","value":"2021-01-02 00:00:00"}],"string_timestamp_us_map":[{"key":"AString76783","value":"2021-01-02 00:00:00"},{"key":"AString89351","value":"2021-01-02 00:00:00"}],"string_uint16_map":[{"key":"AString76783","value":63063},{"key":"AString89351","value":34191}],"string_uint32_map":[{"key":"AString76783","value":506910753},{"key":"AString89351","value":2112817056}],"string_uint64_map":[{"key":"AString76783","value":14697929703826477056},{"key":"AString89351","value":3812638179211989504}],"string_uint8_map":[{"key":"AString76783","value":78},{"key":"AString89351","value":21}],"string_uuid_map":[{"key":"AString76783","value":"00000000-0000-0000-0000-000000000001"},{"key":"AString89351","value":"00000000-0000-0000-0000-000000000001"}],"struct":{"binary":"7049Ug==","boolean":true,"date32":"2023-07-16","date64":"2023-07-16","daytimeinterval":{"days":6783,"milliseconds":6783},"decimal":"826476783.5364813824","duration_ms":"6476783ms","duration_ns":"6476783ns","duration_s":"6476783s","duration_us":"6476783us","float32":783.936,"float64":783.936,"inet":"192.0.2.0/24","int16":-30727,"int32":-1781555540,"int64":-5474557666971701248,"int8":-123,"json":{"test":["a","b",3]},"json_array":[{"test":"test"},123,{"test_number":456}],"largebinary":"7049Ug==","largestring":"AString76783","mac":"aa:bb:cc:dd:ee:ff","monthdaynanointerval":{"months":6783,"days":6783,"nanoseconds":6783},"monthinterval":{"months":1},"string":"AString76783","time32ms":"00:00:00","time32s":"00:00:00","time64ns":"00:00:00","time64us":"00:00:00","timestamp_ms":"2021-01-02 00:00:00","timestamp_ns":"2021-01-02 00:00:00","timestamp_s":"2021-01-02 00:00:00","timestamp_us":"2021-01-02 00:00:00","uint16":63063,"uint32":506910753,"uint64":14697929703826477056,"uint8":78,"uuid":"00000000-0000-0000-0000-000000000001"},"time32ms":"00:00:00","time32ms_list":["00:00:00",null,"00:00:00"],"time32s":"00:00:00","time32s_list":["00:00:00",null,"00:00:00"],"time64ns":"00:00:00","time64ns_list":["00:00:00",null,"00:00:00"],"time64us":"00:00:00","time64us_list":["00:00:00",null,"00:00:00"],"timestamp_ms":"2021-01-02 00:00:00","timestamp_ms_list":["2021-01-02 00:00:00",null,"2021-01-02 00:00:00"],"timestamp_ns":"2021-01-02 00:00:00","timestamp_ns_list":["2021-01-02 00:00:00",null,"2021-01-02 00:00:00"],"timestamp_s":"2021-01-02 00:00:00","timestamp_s_list":["2021-01-02 00:00:00",null,"2021-01-02 00:00:00"],"timestamp_us":"2021-01-02 00:00:00","timestamp_us_list":["2021-01-02 00:00:00",null,"2021-01-02 00:00:00"],"uint16":63063,"uint16_list":[63063,null,63063],"uint32":506910753,"uint32_list":[506910753,null,506910753],"uint64":14697929703826477056,"uint64_list":[14697929703826477056,null,14697929703826477056],"uint8":78,"uint8_list":[78,null,78],"uuid":"00000000-0000-0000-0000-000000000001","uuid_list":["00000000-0000-0000-0000-000000000001",null,"00000000-0000-0000-0000-000000000001"]} -{"_cq_id":"00000000-0000-0000-0000-000000000001","_cq_parent_id":null,"_cq_source_name":"test-source","_cq_sync_time":"2021-01-01 00:00:00","binary":null,"boolean":null,"boolean_list":null,"date32":null,"date32_list":null,"date64":null,"date64_list":null,"daytimeinterval":null,"daytimeinterval_list":null,"decimal":null,"decimal_list":null,"duration_ms":null,"duration_ms_list":null,"duration_ns":null,"duration_ns_list":null,"duration_s":null,"duration_s_list":null,"duration_us":null,"duration_us_list":null,"float32":null,"float32_list":null,"float64":null,"float64_list":null,"inet":null,"inet_list":null,"int16":null,"int16_list":null,"int32":null,"int32_list":null,"int64":null,"int64_list":null,"int8":null,"int8_list":null,"int_boolean_map":null,"int_date32_map":null,"int_date64_map":null,"int_daytimeinterval_map":null,"int_decimal_map":null,"int_duration_ms_map":null,"int_duration_ns_map":null,"int_duration_s_map":null,"int_duration_us_map":null,"int_float32_map":null,"int_float64_map":null,"int_inet_map":null,"int_int16_map":null,"int_int32_map":null,"int_int64_map":null,"int_int8_map":null,"int_largestring_map":null,"int_mac_map":null,"int_monthdaynanointerval_map":null,"int_monthinterval_map":null,"int_string_map":null,"int_time32ms_map":null,"int_time32s_map":null,"int_time64ns_map":null,"int_time64us_map":null,"int_timestamp_ms_map":null,"int_timestamp_ns_map":null,"int_timestamp_s_map":null,"int_timestamp_us_map":null,"int_uint16_map":null,"int_uint32_map":null,"int_uint64_map":null,"int_uint8_map":null,"int_uuid_map":null,"json":null,"json_array":null,"largebinary":null,"largestring":null,"largestring_list":null,"mac":null,"mac_list":null,"monthdaynanointerval":null,"monthdaynanointerval_list":null,"monthinterval":null,"monthinterval_list":null,"nested_struct":null,"string":null,"string_boolean_map":null,"string_date32_map":null,"string_date64_map":null,"string_daytimeinterval_map":null,"string_decimal_map":null,"string_duration_ms_map":null,"string_duration_ns_map":null,"string_duration_s_map":null,"string_duration_us_map":null,"string_float32_map":null,"string_float64_map":null,"string_inet_map":null,"string_int16_map":null,"string_int32_map":null,"string_int64_map":null,"string_int8_map":null,"string_largestring_map":null,"string_list":null,"string_mac_map":null,"string_monthdaynanointerval_map":null,"string_monthinterval_map":null,"string_string_map":null,"string_time32ms_map":null,"string_time32s_map":null,"string_time64ns_map":null,"string_time64us_map":null,"string_timestamp_ms_map":null,"string_timestamp_ns_map":null,"string_timestamp_s_map":null,"string_timestamp_us_map":null,"string_uint16_map":null,"string_uint32_map":null,"string_uint64_map":null,"string_uint8_map":null,"string_uuid_map":null,"struct":null,"time32ms":null,"time32ms_list":null,"time32s":null,"time32s_list":null,"time64ns":null,"time64ns_list":null,"time64us":null,"time64us_list":null,"timestamp_ms":null,"timestamp_ms_list":null,"timestamp_ns":null,"timestamp_ns_list":null,"timestamp_s":null,"timestamp_s_list":null,"timestamp_us":null,"timestamp_us_list":null,"uint16":null,"uint16_list":null,"uint32":null,"uint32_list":null,"uint64":null,"uint64_list":null,"uint8":null,"uint8_list":null,"uuid":null,"uuid_list":null} +{"_cq_id":"00000000-0000-0000-0000-000000000001","_cq_parent_id":"00000000-0000-0000-0000-000000000001","_cq_source_name":"test-source","_cq_sync_time":"2021-01-01 00:00:00","binary":"AQIDBA==","boolean":true,"boolean_list":[true,null,true],"date32":"2023-04-24","date32_list":["2023-04-24",null,"2023-04-24"],"date64":"2023-04-24","date64_list":["2023-04-24",null,"2023-04-24"],"daytimeinterval":{"days":1,"milliseconds":1},"daytimeinterval_list":[{"days":1,"milliseconds":1},null,{"days":1,"milliseconds":1}],"duration_ms":"123456789ms","duration_ms_list":["123456789ms",null,"123456789ms"],"duration_ns":"123456789ns","duration_ns_list":["123456789ns",null,"123456789ns"],"duration_s":"123456789s","duration_s_list":["123456789s",null,"123456789s"],"duration_us":"123456789us","duration_us_list":["123456789us",null,"123456789us"],"float32":1.1,"float32_list":[1.1,null,1.1],"float64":1.1,"float64_list":[1.1,null,1.1],"inet":"192.0.2.0/24","inet_list":["192.0.2.0/24",null,"192.0.2.0/24"],"int16":-1,"int16_list":[-1,null,-1],"int32":-1,"int32_list":[-1,null,-1],"int64":-1,"int64_list":[-1,null,-1],"int8":-1,"int8_list":[-1,null,-1],"json":{"test":["a","b",3]},"json_array":[{"test":"test"},123,{"test_number":456}],"largebinary":"AQIDBA==","largestring":"AString","largestring_list":["AString",null,"AString"],"mac":"aa:bb:cc:dd:ee:ff","mac_list":["aa:bb:cc:dd:ee:ff",null,"aa:bb:cc:dd:ee:ff"],"monthdaynanointerval":{"months":1,"days":1,"nanoseconds":1},"monthdaynanointerval_list":[{"months":1,"days":1,"nanoseconds":1},null,{"months":1,"days":1,"nanoseconds":1}],"monthinterval":{"months":1},"monthinterval_list":[{"months":1},null,{"months":1}],"nested_struct":{"inner":{"binary":"AQIDBA==","boolean":true,"date32":"2023-04-24","date64":"2023-04-24","daytimeinterval":{"days":1,"milliseconds":1},"duration_ms":"123456789ms","duration_ns":"123456789ns","duration_s":"123456789s","duration_us":"123456789us","float32":1.1,"float64":1.1,"inet":"192.0.2.0/24","int16":-1,"int32":-1,"int64":-1,"int8":-1,"json":{"test":["a","b",3]},"json_array":[{"test":"test"},123,{"test_number":456}],"largebinary":"AQIDBA==","largestring":"AString","mac":"aa:bb:cc:dd:ee:ff","monthdaynanointerval":{"months":1,"days":1,"nanoseconds":1},"monthinterval":{"months":1},"string":"AString","time32ms":"00:00:00","time32s":"00:00:00","time64ns":"00:00:00","time64us":"00:00:00","timestamp_ms":"2021-01-02 00:00:00","timestamp_ns":"2021-01-02 00:00:00","timestamp_s":"2021-01-02 00:00:00","timestamp_us":"2021-01-02 00:00:00","uint16":1,"uint32":1,"uint64":1,"uint8":1,"uuid":"00000000-0000-0000-0000-000000000001"}},"string":"AString","string_list":["AString",null,"AString"],"struct":{"binary":"AQIDBA==","boolean":true,"date32":"2023-04-24","date64":"2023-04-24","daytimeinterval":{"days":1,"milliseconds":1},"duration_ms":"123456789ms","duration_ns":"123456789ns","duration_s":"123456789s","duration_us":"123456789us","float32":1.1,"float64":1.1,"inet":"192.0.2.0/24","int16":-1,"int32":-1,"int64":-1,"int8":-1,"json":{"test":["a","b",3]},"json_array":[{"test":"test"},123,{"test_number":456}],"largebinary":"AQIDBA==","largestring":"AString","mac":"aa:bb:cc:dd:ee:ff","monthdaynanointerval":{"months":1,"days":1,"nanoseconds":1},"monthinterval":{"months":1},"string":"AString","time32ms":"00:00:00","time32s":"00:00:00","time64ns":"00:00:00","time64us":"00:00:00","timestamp_ms":"2021-01-02 00:00:00","timestamp_ns":"2021-01-02 00:00:00","timestamp_s":"2021-01-02 00:00:00","timestamp_us":"2021-01-02 00:00:00","uint16":1,"uint32":1,"uint64":1,"uint8":1,"uuid":"00000000-0000-0000-0000-000000000001"},"time32ms":"00:00:00","time32ms_list":["00:00:00",null,"00:00:00"],"time32s":"00:00:00","time32s_list":["00:00:00",null,"00:00:00"],"time64ns":"00:00:00","time64ns_list":["00:00:00",null,"00:00:00"],"time64us":"00:00:00","time64us_list":["00:00:00",null,"00:00:00"],"timestamp_ms":"2021-01-02 00:00:00","timestamp_ms_list":["2021-01-02 00:00:00",null,"2021-01-02 00:00:00"],"timestamp_ns":"2021-01-02 00:00:00","timestamp_ns_list":["2021-01-02 00:00:00",null,"2021-01-02 00:00:00"],"timestamp_s":"2021-01-02 00:00:00","timestamp_s_list":["2021-01-02 00:00:00",null,"2021-01-02 00:00:00"],"timestamp_us":"2021-01-02 00:00:00","timestamp_us_list":["2021-01-02 00:00:00",null,"2021-01-02 00:00:00"],"uint16":1,"uint16_list":[1,null,1],"uint32":1,"uint32_list":[1,null,1],"uint64":1,"uint64_list":[1,null,1],"uint8":1,"uint8_list":[1,null,1],"uuid":"00000000-0000-0000-0000-000000000001","uuid_list":["00000000-0000-0000-0000-000000000001",null,"00000000-0000-0000-0000-000000000001"]} +{"_cq_id":"00000000-0000-0000-0000-000000000001","_cq_parent_id":null,"_cq_source_name":"test-source","_cq_sync_time":"2021-01-01 00:00:00","binary":null,"boolean":null,"boolean_list":null,"date32":null,"date32_list":null,"date64":null,"date64_list":null,"daytimeinterval":null,"daytimeinterval_list":null,"duration_ms":null,"duration_ms_list":null,"duration_ns":null,"duration_ns_list":null,"duration_s":null,"duration_s_list":null,"duration_us":null,"duration_us_list":null,"float32":null,"float32_list":null,"float64":null,"float64_list":null,"inet":null,"inet_list":null,"int16":null,"int16_list":null,"int32":null,"int32_list":null,"int64":null,"int64_list":null,"int8":null,"int8_list":null,"json":null,"json_array":null,"largebinary":null,"largestring":null,"largestring_list":null,"mac":null,"mac_list":null,"monthdaynanointerval":null,"monthdaynanointerval_list":null,"monthinterval":null,"monthinterval_list":null,"nested_struct":null,"string":null,"string_list":null,"struct":null,"time32ms":null,"time32ms_list":null,"time32s":null,"time32s_list":null,"time64ns":null,"time64ns_list":null,"time64us":null,"time64us_list":null,"timestamp_ms":null,"timestamp_ms_list":null,"timestamp_ns":null,"timestamp_ns_list":null,"timestamp_s":null,"timestamp_s_list":null,"timestamp_us":null,"timestamp_us_list":null,"uint16":null,"uint16_list":null,"uint32":null,"uint32_list":null,"uint64":null,"uint64_list":null,"uint8":null,"uint8_list":null,"uuid":null,"uuid_list":null} diff --git a/json/write.go b/json/write.go index d5dfc2d8..ff6c5b2e 100644 --- a/json/write.go +++ b/json/write.go @@ -5,8 +5,8 @@ import ( "github.com/apache/arrow/go/v13/arrow" "github.com/apache/arrow/go/v13/arrow/array" - "github.com/cloudquery/filetypes/v3/types" - "github.com/cloudquery/plugin-sdk/v3/schema" + "github.com/cloudquery/filetypes/v4/types" + "github.com/cloudquery/plugin-sdk/v4/schema" "github.com/goccy/go-json" ) diff --git a/json/write_read_test.go b/json/write_read_test.go index 6d78c078..dc8b940b 100644 --- a/json/write_read_test.go +++ b/json/write_read_test.go @@ -9,9 +9,9 @@ import ( "github.com/apache/arrow/go/v13/arrow" "github.com/bradleyjkemp/cupaloy/v2" - "github.com/cloudquery/filetypes/v3/types" - "github.com/cloudquery/plugin-sdk/v3/plugins/destination" - "github.com/cloudquery/plugin-sdk/v3/schema" + "github.com/cloudquery/filetypes/v4/types" + "github.com/cloudquery/plugin-sdk/v4/plugin" + "github.com/cloudquery/plugin-sdk/v4/schema" "github.com/google/uuid" ) @@ -82,7 +82,7 @@ func TestWriteRead(t *testing.T) { }() totalCount := 0 for got := range ch { - if diff := destination.RecordDiff(records[totalCount], got); diff != "" { + if diff := plugin.RecordDiff(records[totalCount], got); diff != "" { t.Fatalf("got diff: %s", diff) } totalCount++ diff --git a/parquet/read.go b/parquet/read.go index 793d6020..558d4243 100644 --- a/parquet/read.go +++ b/parquet/read.go @@ -11,7 +11,7 @@ import ( "github.com/apache/arrow/go/v13/parquet" "github.com/apache/arrow/go/v13/parquet/file" "github.com/apache/arrow/go/v13/parquet/pqarrow" - "github.com/cloudquery/plugin-sdk/v3/schema" + "github.com/cloudquery/plugin-sdk/v4/schema" ) func (*Client) Read(f parquet.ReaderAtSeeker, table *schema.Table, _ string, res chan<- arrow.Record) error { diff --git a/parquet/write.go b/parquet/write.go index e71b7c5b..20f17b4b 100644 --- a/parquet/write.go +++ b/parquet/write.go @@ -9,9 +9,9 @@ import ( "github.com/apache/arrow/go/v13/parquet" "github.com/apache/arrow/go/v13/parquet/compress" "github.com/apache/arrow/go/v13/parquet/pqarrow" - ftypes "github.com/cloudquery/filetypes/v3/types" - "github.com/cloudquery/plugin-sdk/v3/schema" - "github.com/cloudquery/plugin-sdk/v3/types" + ftypes "github.com/cloudquery/filetypes/v4/types" + "github.com/cloudquery/plugin-sdk/v4/schema" + "github.com/cloudquery/plugin-sdk/v4/types" ) type Handle struct { diff --git a/parquet/write_read_test.go b/parquet/write_read_test.go index 0a5aa535..04b2e327 100644 --- a/parquet/write_read_test.go +++ b/parquet/write_read_test.go @@ -9,9 +9,9 @@ import ( "github.com/apache/arrow/go/v13/arrow" "github.com/apache/arrow/go/v13/arrow/array" - "github.com/cloudquery/filetypes/v3/types" - "github.com/cloudquery/plugin-sdk/v3/plugins/destination" - "github.com/cloudquery/plugin-sdk/v3/schema" + "github.com/cloudquery/filetypes/v4/types" + "github.com/cloudquery/plugin-sdk/v4/plugin" + "github.com/cloudquery/plugin-sdk/v4/schema" ) func TestWriteRead(t *testing.T) { @@ -55,7 +55,7 @@ func TestWriteRead(t *testing.T) { for got := range ch { curr := records[totalCount] if !array.RecordApproxEqual(curr, got) { - t.Fatalf("got diff (record %d): %s\n", totalCount, destination.RecordDiff(records[totalCount], got)) + t.Fatalf("got diff (record %d): %s\n", totalCount, plugin.RecordDiff(records[totalCount], got)) } totalCount++ } diff --git a/read.go b/read.go index 517f60ab..33b97f74 100644 --- a/read.go +++ b/read.go @@ -4,7 +4,7 @@ import ( "io" "github.com/apache/arrow/go/v13/arrow" - "github.com/cloudquery/plugin-sdk/v3/schema" + "github.com/cloudquery/plugin-sdk/v4/schema" ) type ReaderAtSeeker interface { diff --git a/spec.go b/spec.go index c9fe75ec..6f922dd1 100644 --- a/spec.go +++ b/spec.go @@ -5,9 +5,9 @@ import ( "encoding/json" "fmt" - "github.com/cloudquery/filetypes/v3/csv" - jsonFile "github.com/cloudquery/filetypes/v3/json" - "github.com/cloudquery/filetypes/v3/parquet" + "github.com/cloudquery/filetypes/v4/csv" + jsonFile "github.com/cloudquery/filetypes/v4/json" + "github.com/cloudquery/filetypes/v4/parquet" ) type FormatType string diff --git a/spec_test.go b/spec_test.go index 8baefc47..54e593ca 100644 --- a/spec_test.go +++ b/spec_test.go @@ -3,8 +3,8 @@ package filetypes import ( "testing" - "github.com/cloudquery/filetypes/v3/csv" - "github.com/cloudquery/filetypes/v3/json" + "github.com/cloudquery/filetypes/v4/csv" + "github.com/cloudquery/filetypes/v4/json" "github.com/stretchr/testify/assert" ) diff --git a/types/types.go b/types/types.go index 77ae7a7c..022d5ac6 100644 --- a/types/types.go +++ b/types/types.go @@ -4,7 +4,7 @@ import ( "io" "github.com/apache/arrow/go/v13/arrow" - "github.com/cloudquery/plugin-sdk/v3/schema" + "github.com/cloudquery/plugin-sdk/v4/schema" ) type FileType interface { diff --git a/write.go b/write.go index fbb2eb9f..2633591e 100644 --- a/write.go +++ b/write.go @@ -4,8 +4,8 @@ import ( "io" "github.com/apache/arrow/go/v13/arrow" - "github.com/cloudquery/filetypes/v3/types" - "github.com/cloudquery/plugin-sdk/v3/schema" + "github.com/cloudquery/filetypes/v4/types" + "github.com/cloudquery/plugin-sdk/v4/schema" ) func (cl *Client) WriteTableBatchFile(w io.Writer, table *schema.Table, records []arrow.Record) error { From 8685218ab3f4caae8c477d905d74fbc21792987a Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Mon, 19 Jun 2023 14:25:36 +0100 Subject: [PATCH 02/16] Remove sourceName from read --- csv/read.go | 2 +- csv/write_read_test.go | 2 +- json/read.go | 2 +- json/write_read_test.go | 2 +- parquet/read.go | 2 +- parquet/write_read_test.go | 2 +- read.go | 8 ++++---- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/csv/read.go b/csv/read.go index a9534204..ecc44d9f 100644 --- a/csv/read.go +++ b/csv/read.go @@ -11,7 +11,7 @@ import ( "github.com/cloudquery/plugin-sdk/v4/schema" ) -func (cl *Client) Read(r io.Reader, table *schema.Table, _ string, res chan<- arrow.Record) error { +func (cl *Client) Read(r io.Reader, table *schema.Table, res chan<- arrow.Record) error { arrowSchema := table.ToArrowSchema() newSchema := convertSchema(arrowSchema) reader := csv.NewReader(r, newSchema, diff --git a/csv/write_read_test.go b/csv/write_read_test.go index 0b4af948..6ba53bb0 100644 --- a/csv/write_read_test.go +++ b/csv/write_read_test.go @@ -69,7 +69,7 @@ func TestWriteRead(t *testing.T) { ch := make(chan arrow.Record) var readErr error go func() { - readErr = cl.Read(byteReader, table, "test-source", ch) + readErr = cl.Read(byteReader, table, ch) close(ch) }() totalCount := 0 diff --git a/json/read.go b/json/read.go index b212bfe8..90d5e692 100644 --- a/json/read.go +++ b/json/read.go @@ -12,7 +12,7 @@ import ( const maxJSONSize = 1024 * 1024 * 20 -func (*Client) Read(r io.Reader, table *schema.Table, _ string, res chan<- arrow.Record) error { +func (*Client) Read(r io.Reader, table *schema.Table, res chan<- arrow.Record) error { scanner := bufio.NewScanner(r) scanner.Buffer(make([]byte, maxJSONSize), maxJSONSize) rb := array.NewRecordBuilder(memory.DefaultAllocator, table.ToArrowSchema()) diff --git a/json/write_read_test.go b/json/write_read_test.go index dc8b940b..4b6a2cef 100644 --- a/json/write_read_test.go +++ b/json/write_read_test.go @@ -77,7 +77,7 @@ func TestWriteRead(t *testing.T) { ch := make(chan arrow.Record) var readErr error go func() { - readErr = cl.Read(byteReader, table, "test-source", ch) + readErr = cl.Read(byteReader, table, ch) close(ch) }() totalCount := 0 diff --git a/parquet/read.go b/parquet/read.go index 558d4243..8128ba1f 100644 --- a/parquet/read.go +++ b/parquet/read.go @@ -14,7 +14,7 @@ import ( "github.com/cloudquery/plugin-sdk/v4/schema" ) -func (*Client) Read(f parquet.ReaderAtSeeker, table *schema.Table, _ string, res chan<- arrow.Record) error { +func (*Client) Read(f parquet.ReaderAtSeeker, table *schema.Table, res chan<- arrow.Record) error { ctx := context.Background() rdr, err := file.NewParquetReader(f) if err != nil { diff --git a/parquet/write_read_test.go b/parquet/write_read_test.go index 04b2e327..5f0dc5f1 100644 --- a/parquet/write_read_test.go +++ b/parquet/write_read_test.go @@ -48,7 +48,7 @@ func TestWriteRead(t *testing.T) { ch := make(chan arrow.Record) var readErr error go func() { - readErr = cl.Read(byteReader, table, "test-source", ch) + readErr = cl.Read(byteReader, table, ch) close(ch) }() totalCount := 0 diff --git a/read.go b/read.go index 33b97f74..7992319b 100644 --- a/read.go +++ b/read.go @@ -13,18 +13,18 @@ type ReaderAtSeeker interface { io.Seeker } -func (cl *Client) Read(f ReaderAtSeeker, table *schema.Table, sourceName string, res chan<- arrow.Record) error { +func (cl *Client) Read(f ReaderAtSeeker, table *schema.Table, res chan<- arrow.Record) error { switch cl.spec.Format { case FormatTypeCSV: - if err := cl.csv.Read(f, table, sourceName, res); err != nil { + if err := cl.csv.Read(f, table, res); err != nil { return err } case FormatTypeJSON: - if err := cl.json.Read(f, table, sourceName, res); err != nil { + if err := cl.json.Read(f, table, res); err != nil { return err } case FormatTypeParquet: - if err := cl.parquet.Read(f, table, sourceName, res); err != nil { + if err := cl.parquet.Read(f, table, res); err != nil { return err } default: From c3bb3e796571303692a7ef3e7b8c71d2b04b9192 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Mon, 19 Jun 2023 18:15:58 +0100 Subject: [PATCH 03/16] Remove unnecessary test --- json/write_read_test.go | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/json/write_read_test.go b/json/write_read_test.go index 4b6a2cef..e1064fde 100644 --- a/json/write_read_test.go +++ b/json/write_read_test.go @@ -15,27 +15,6 @@ import ( "github.com/google/uuid" ) -func TestWrite(t *testing.T) { - var b bytes.Buffer - table := schema.TestTable("test", schema.TestSourceOptions{}) - sourceName := "test-source" - syncTime := time.Now().UTC().Round(time.Second) - opts := schema.GenTestDataOptions{ - SourceName: sourceName, - SyncTime: syncTime, - MaxRows: 1, - } - records := schema.GenTestData(table, opts) - cl, err := NewClient() - if err != nil { - t.Fatal(err) - } - if err := types.WriteAll(cl, &b, table, records); err != nil { - t.Fatal(err) - } - t.Log(b.String()) -} - func TestWriteRead(t *testing.T) { table := schema.TestTable("test", schema.TestSourceOptions{}) sourceName := "test-source" From 481690d18bb02992a05b13ff749dc51810b3385f Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Tue, 20 Jun 2023 11:41:24 +0100 Subject: [PATCH 04/16] go mod tidy --- go.mod | 1 - go.sum | 10 ++-------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 9c21b7a4..8d720fa1 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,6 @@ require ( github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect github.com/andybalholm/brotli v1.0.5 // indirect github.com/apache/thrift v0.18.1 // indirect - github.com/cloudquery/plugin-pb-go v1.1.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect diff --git a/go.sum b/go.sum index 834102a7..26518311 100644 --- a/go.sum +++ b/go.sum @@ -6,16 +6,10 @@ github.com/apache/thrift v0.18.1 h1:lNhK/1nqjbwbiOPDBPFJVKxgDEGSepKuTh6OLiXW8kg= github.com/apache/thrift v0.18.1/go.mod h1:rdQn/dCcDKEWjjylUeueum4vQEjG2v8v2PqriUnbr+I= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= -github.com/cloudquery/arrow/go/v13 v13.0.0-20230619001325-b0832beeeade h1:4vkgraWxeK51HYQsGjQ0TTuVxTAxaUh9GLfXK1MiOJE= -github.com/cloudquery/arrow/go/v13 v13.0.0-20230619001325-b0832beeeade/go.mod h1:W69eByFNO0ZR30q1/7Sr9d83zcVZmF2MiP3fFYAWJOc= -github.com/cloudquery/plugin-sdk/v4 v4.0.0-20230617094031-84a6a9b7e057 h1:etJDrg7YcQmJvGP/thomMgGSzlIRKRbqeb9XiSsdBuk= -github.com/cloudquery/plugin-sdk/v4 v4.0.0-20230617094031-84a6a9b7e057/go.mod h1:C/OPmi3Jkded0V53qE4uDrjiKWGrn8pnb9jWcilimBw= github.com/cloudquery/arrow/go/v13 v13.0.0-20230620001150-43638cb1b099 h1:C+UgdZ4xg4pxttG8nWBdrfspkmFrwCEI9rgTYssOLRg= github.com/cloudquery/arrow/go/v13 v13.0.0-20230620001150-43638cb1b099/go.mod h1:W69eByFNO0ZR30q1/7Sr9d83zcVZmF2MiP3fFYAWJOc= -github.com/cloudquery/plugin-pb-go v1.1.0 h1:F1r/x4aF5aO1hmgBk8rqAp2oejeYMMughTPaYosQLDk= -github.com/cloudquery/plugin-pb-go v1.1.0/go.mod h1:327Dd56bQ357KNIbhZNGDoJ7jPYXsCZWZ4Tj955gU7M= -github.com/cloudquery/plugin-sdk/v3 v3.10.6 h1:KqTsLZ6OA1h8BUMeMcU6BAD6TBW6ojgQaC4zDZMgvu0= -github.com/cloudquery/plugin-sdk/v3 v3.10.6/go.mod h1:QhBaVgiNyQ3P6uAzJWOYpYykHXL+WDZffwg1riTwv60= +github.com/cloudquery/plugin-sdk/v4 v4.0.0-20230617094031-84a6a9b7e057 h1:etJDrg7YcQmJvGP/thomMgGSzlIRKRbqeb9XiSsdBuk= +github.com/cloudquery/plugin-sdk/v4 v4.0.0-20230617094031-84a6a9b7e057/go.mod h1:C/OPmi3Jkded0V53qE4uDrjiKWGrn8pnb9jWcilimBw= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= From cd5daa29f05cda2392402d678f4b907d252434f9 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Fri, 23 Jun 2023 10:43:34 +0100 Subject: [PATCH 05/16] csv writer: strip `cq:extension:` metadata from columns, late flush --- csv/write.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/csv/write.go b/csv/write.go index cc0895be..91618cf7 100644 --- a/csv/write.go +++ b/csv/write.go @@ -4,6 +4,7 @@ import ( // "encoding/csv" "fmt" "io" + "strings" "github.com/apache/arrow/go/v13/arrow" "github.com/apache/arrow/go/v13/arrow/array" @@ -39,9 +40,10 @@ func (h *Handle) WriteContent(records []arrow.Record) error { if err := h.w.Write(castRec); err != nil { return fmt.Errorf("failed to write record to csv: %w", err) } - if err := h.w.Flush(); err != nil { - return fmt.Errorf("failed to flush csv writer: %w", err) - } + } + + if err := h.w.Flush(); err != nil { + return fmt.Errorf("failed to flush csv writer: %w", err) } return nil } @@ -58,6 +60,7 @@ func convertSchema(sch *arrow.Schema) *arrow.Schema { if !isTypeSupported(f.Type) { fields[i].Type = arrow.BinaryTypes.String } + fields[i].Metadata = stripCQExtensionMetadata(fields[i].Metadata) } md := sch.Metadata() @@ -108,3 +111,13 @@ func castToString(rec arrow.Record) arrow.Record { } return array.NewRecord(newSchema, cols, rec.NumRows()) } + +func stripCQExtensionMetadata(md arrow.Metadata) arrow.Metadata { + m := md.ToMap() + for k := range m { + if !strings.HasPrefix(k, "cq:extension:") { + delete(m, k) + } + } + return arrow.MetadataFrom(m) +} From 9f08f9ebb6ded681816c35b682271c2a6aad3cee Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Fri, 23 Jun 2023 10:46:58 +0100 Subject: [PATCH 06/16] update sdk, go mod tidy --- go.mod | 2 +- go.sum | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 51cbfdfb..65d2c989 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( // TODO: remove once all updates are merged replace github.com/apache/arrow/go/v13 => github.com/cloudquery/arrow/go/v13 v13.0.0-20230622001328-95d31999d687 -replace github.com/cloudquery/plugin-sdk/v4 => github.com/cloudquery/plugin-sdk/v4 v4.0.0-20230617094031-84a6a9b7e057 +replace github.com/cloudquery/plugin-sdk/v4 => github.com/cloudquery/plugin-sdk/v4 v4.0.0-20230622195407-5823d970ad6d require ( github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect diff --git a/go.sum b/go.sum index 4bced09e..5272bdb4 100644 --- a/go.sum +++ b/go.sum @@ -6,16 +6,10 @@ github.com/apache/thrift v0.18.1 h1:lNhK/1nqjbwbiOPDBPFJVKxgDEGSepKuTh6OLiXW8kg= github.com/apache/thrift v0.18.1/go.mod h1:rdQn/dCcDKEWjjylUeueum4vQEjG2v8v2PqriUnbr+I= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= -github.com/cloudquery/arrow/go/v13 v13.0.0-20230620001150-43638cb1b099 h1:C+UgdZ4xg4pxttG8nWBdrfspkmFrwCEI9rgTYssOLRg= -github.com/cloudquery/arrow/go/v13 v13.0.0-20230620001150-43638cb1b099/go.mod h1:W69eByFNO0ZR30q1/7Sr9d83zcVZmF2MiP3fFYAWJOc= -github.com/cloudquery/plugin-sdk/v4 v4.0.0-20230617094031-84a6a9b7e057 h1:etJDrg7YcQmJvGP/thomMgGSzlIRKRbqeb9XiSsdBuk= -github.com/cloudquery/plugin-sdk/v4 v4.0.0-20230617094031-84a6a9b7e057/go.mod h1:C/OPmi3Jkded0V53qE4uDrjiKWGrn8pnb9jWcilimBw= github.com/cloudquery/arrow/go/v13 v13.0.0-20230622001328-95d31999d687 h1:VM+2p8PDI94sYsofensnFY4mIG/8JNZmMLfQsscQ9Xk= github.com/cloudquery/arrow/go/v13 v13.0.0-20230622001328-95d31999d687/go.mod h1:W69eByFNO0ZR30q1/7Sr9d83zcVZmF2MiP3fFYAWJOc= -github.com/cloudquery/plugin-pb-go v1.1.0 h1:F1r/x4aF5aO1hmgBk8rqAp2oejeYMMughTPaYosQLDk= -github.com/cloudquery/plugin-pb-go v1.1.0/go.mod h1:327Dd56bQ357KNIbhZNGDoJ7jPYXsCZWZ4Tj955gU7M= -github.com/cloudquery/plugin-sdk/v3 v3.10.6 h1:KqTsLZ6OA1h8BUMeMcU6BAD6TBW6ojgQaC4zDZMgvu0= -github.com/cloudquery/plugin-sdk/v3 v3.10.6/go.mod h1:QhBaVgiNyQ3P6uAzJWOYpYykHXL+WDZffwg1riTwv60= +github.com/cloudquery/plugin-sdk/v4 v4.0.0-20230622195407-5823d970ad6d h1:M6cH4jq3eJxPW0u2cyy3UtH/vGEQQoSFaTTNvm9Rg2c= +github.com/cloudquery/plugin-sdk/v4 v4.0.0-20230622195407-5823d970ad6d/go.mod h1:6KcdvApEzLQ1nRBJlfDm1OqzPoAHE2HbMGOciGqAYaM= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= From 2d509d123f1e9abb5d7aaf8cad67af5b6a39ed52 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Fri, 23 Jun 2023 11:33:48 +0100 Subject: [PATCH 07/16] update sdk ref --- go.mod | 4 ++-- go.sum | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 65d2c989..c23cb9f2 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/cloudquery/filetypes/v4 go 1.19 require ( - github.com/apache/arrow/go/v13 v13.0.0-20230601214540-018e7d3f9c4b + github.com/apache/arrow/go/v13 v13.0.0-20230622042343-ec413b7763fe github.com/bradleyjkemp/cupaloy/v2 v2.8.0 github.com/cloudquery/plugin-sdk/v4 v4.0.0 github.com/goccy/go-json v0.10.2 @@ -14,7 +14,7 @@ require ( // TODO: remove once all updates are merged replace github.com/apache/arrow/go/v13 => github.com/cloudquery/arrow/go/v13 v13.0.0-20230622001328-95d31999d687 -replace github.com/cloudquery/plugin-sdk/v4 => github.com/cloudquery/plugin-sdk/v4 v4.0.0-20230622195407-5823d970ad6d +replace github.com/cloudquery/plugin-sdk/v4 => github.com/cloudquery/plugin-sdk/v4 v4.0.0-20230623101720-d4b06c321647 require ( github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect diff --git a/go.sum b/go.sum index 5272bdb4..1650cf79 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oM github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= github.com/cloudquery/arrow/go/v13 v13.0.0-20230622001328-95d31999d687 h1:VM+2p8PDI94sYsofensnFY4mIG/8JNZmMLfQsscQ9Xk= github.com/cloudquery/arrow/go/v13 v13.0.0-20230622001328-95d31999d687/go.mod h1:W69eByFNO0ZR30q1/7Sr9d83zcVZmF2MiP3fFYAWJOc= -github.com/cloudquery/plugin-sdk/v4 v4.0.0-20230622195407-5823d970ad6d h1:M6cH4jq3eJxPW0u2cyy3UtH/vGEQQoSFaTTNvm9Rg2c= -github.com/cloudquery/plugin-sdk/v4 v4.0.0-20230622195407-5823d970ad6d/go.mod h1:6KcdvApEzLQ1nRBJlfDm1OqzPoAHE2HbMGOciGqAYaM= +github.com/cloudquery/plugin-sdk/v4 v4.0.0-20230623101720-d4b06c321647 h1:bf0A8wq7kW69dGriLhnuB3RdFYyd3aGpgLcic/Diib0= +github.com/cloudquery/plugin-sdk/v4 v4.0.0-20230623101720-d4b06c321647/go.mod h1:kRmb9XCAohUw5ZX8TjAM2Vhwy9XexYXPTILQtfz1XZk= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= From 532d83310a1f80368c888ac014f8671535860973 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Fri, 23 Jun 2023 12:10:58 +0100 Subject: [PATCH 08/16] update sdk ref --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c23cb9f2..a29313b0 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( // TODO: remove once all updates are merged replace github.com/apache/arrow/go/v13 => github.com/cloudquery/arrow/go/v13 v13.0.0-20230622001328-95d31999d687 -replace github.com/cloudquery/plugin-sdk/v4 => github.com/cloudquery/plugin-sdk/v4 v4.0.0-20230623101720-d4b06c321647 +replace github.com/cloudquery/plugin-sdk/v4 => github.com/cloudquery/plugin-sdk/v4 v4.0.0-20230623105555-75af24931824 require ( github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect diff --git a/go.sum b/go.sum index 1650cf79..75a92f38 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oM github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= github.com/cloudquery/arrow/go/v13 v13.0.0-20230622001328-95d31999d687 h1:VM+2p8PDI94sYsofensnFY4mIG/8JNZmMLfQsscQ9Xk= github.com/cloudquery/arrow/go/v13 v13.0.0-20230622001328-95d31999d687/go.mod h1:W69eByFNO0ZR30q1/7Sr9d83zcVZmF2MiP3fFYAWJOc= -github.com/cloudquery/plugin-sdk/v4 v4.0.0-20230623101720-d4b06c321647 h1:bf0A8wq7kW69dGriLhnuB3RdFYyd3aGpgLcic/Diib0= -github.com/cloudquery/plugin-sdk/v4 v4.0.0-20230623101720-d4b06c321647/go.mod h1:kRmb9XCAohUw5ZX8TjAM2Vhwy9XexYXPTILQtfz1XZk= +github.com/cloudquery/plugin-sdk/v4 v4.0.0-20230623105555-75af24931824 h1:0jxuSpwDAzo2JHzBv3eKsp12oyfOiH5KH4SkNUQXwIk= +github.com/cloudquery/plugin-sdk/v4 v4.0.0-20230623105555-75af24931824/go.mod h1:yWa5I2LH4o8fUh16pjwQmeXmEypw3i6RLB2OOv4IUS8= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= From 493fd14bedecd250860c2b719e8fa1528be892bc Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Fri, 23 Jun 2023 12:18:16 +0100 Subject: [PATCH 09/16] strip, not keep --- csv/write.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csv/write.go b/csv/write.go index 91618cf7..b4cf7911 100644 --- a/csv/write.go +++ b/csv/write.go @@ -115,7 +115,7 @@ func castToString(rec arrow.Record) arrow.Record { func stripCQExtensionMetadata(md arrow.Metadata) arrow.Metadata { m := md.ToMap() for k := range m { - if !strings.HasPrefix(k, "cq:extension:") { + if strings.HasPrefix(k, "cq:extension:") { delete(m, k) } } From 0d3cf18fb236764e47778d68d1b05e67707efc02 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Mon, 26 Jun 2023 09:11:58 +0100 Subject: [PATCH 10/16] update sdk ref --- go.mod | 2 +- go.sum | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 44da647c..5db2a0af 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( // TODO: remove once all updates are merged replace github.com/apache/arrow/go/v13 => github.com/cloudquery/arrow/go/v13 v13.0.0-20230626001500-065602842c3a -replace github.com/cloudquery/plugin-sdk/v4 => github.com/cloudquery/plugin-sdk/v4 v4.0.0-20230623105555-75af24931824 +replace github.com/cloudquery/plugin-sdk/v4 => github.com/cloudquery/plugin-sdk/v4 v4.0.0-20230625074510-d3cd5aad7dec require ( github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect diff --git a/go.sum b/go.sum index 861889ba..ca09faa9 100644 --- a/go.sum +++ b/go.sum @@ -6,16 +6,10 @@ github.com/apache/thrift v0.18.1 h1:lNhK/1nqjbwbiOPDBPFJVKxgDEGSepKuTh6OLiXW8kg= github.com/apache/thrift v0.18.1/go.mod h1:rdQn/dCcDKEWjjylUeueum4vQEjG2v8v2PqriUnbr+I= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= -github.com/cloudquery/arrow/go/v13 v13.0.0-20230622001328-95d31999d687 h1:VM+2p8PDI94sYsofensnFY4mIG/8JNZmMLfQsscQ9Xk= -github.com/cloudquery/arrow/go/v13 v13.0.0-20230622001328-95d31999d687/go.mod h1:W69eByFNO0ZR30q1/7Sr9d83zcVZmF2MiP3fFYAWJOc= -github.com/cloudquery/plugin-sdk/v4 v4.0.0-20230623105555-75af24931824 h1:0jxuSpwDAzo2JHzBv3eKsp12oyfOiH5KH4SkNUQXwIk= -github.com/cloudquery/plugin-sdk/v4 v4.0.0-20230623105555-75af24931824/go.mod h1:yWa5I2LH4o8fUh16pjwQmeXmEypw3i6RLB2OOv4IUS8= github.com/cloudquery/arrow/go/v13 v13.0.0-20230626001500-065602842c3a h1:O/FNq1+8YlWzHYNj2tokFQyja6GXsQBdkuvLMdpuaSw= github.com/cloudquery/arrow/go/v13 v13.0.0-20230626001500-065602842c3a/go.mod h1:W69eByFNO0ZR30q1/7Sr9d83zcVZmF2MiP3fFYAWJOc= -github.com/cloudquery/plugin-pb-go v1.3.3 h1:+4y6hIDm2NtV0ohzgrWzXMaaOTanh7nmDpDgkFtgBlg= -github.com/cloudquery/plugin-pb-go v1.3.3/go.mod h1:NbWAtT2BzJQ9+XUWwh3IKBg3MOeV9ZEpHoHNAQ/YDV8= -github.com/cloudquery/plugin-sdk/v3 v3.10.6 h1:KqTsLZ6OA1h8BUMeMcU6BAD6TBW6ojgQaC4zDZMgvu0= -github.com/cloudquery/plugin-sdk/v3 v3.10.6/go.mod h1:QhBaVgiNyQ3P6uAzJWOYpYykHXL+WDZffwg1riTwv60= +github.com/cloudquery/plugin-sdk/v4 v4.0.0-20230625074510-d3cd5aad7dec h1:gJbBpfIHovks+ERCCHExtlX42uSg82XWuPNFtLqCp2k= +github.com/cloudquery/plugin-sdk/v4 v4.0.0-20230625074510-d3cd5aad7dec/go.mod h1:RnLpmlnyTeBAouDH7th4h75YHg1TudKVwXMQsjCbuQk= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= From 1d7e0b40882eb33123eedf51eb4f26e1877218bb Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Mon, 26 Jun 2023 11:30:20 +0100 Subject: [PATCH 11/16] Add stream helper --- go.mod | 4 +- go.sum | 4 +- stream.go | 92 ++++++++++++++++++++++++++++++ stream_test.go | 149 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 244 insertions(+), 5 deletions(-) create mode 100644 stream.go create mode 100644 stream_test.go diff --git a/go.mod b/go.mod index 5db2a0af..c5b60c85 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.19 require ( github.com/apache/arrow/go/v13 v13.0.0-20230622042343-ec413b7763fe github.com/bradleyjkemp/cupaloy/v2 v2.8.0 - github.com/cloudquery/plugin-sdk/v4 v4.0.0 + github.com/cloudquery/plugin-sdk/v4 v4.0.0-rc1 github.com/goccy/go-json v0.10.2 github.com/google/uuid v1.3.0 github.com/stretchr/testify v1.8.4 @@ -14,8 +14,6 @@ require ( // TODO: remove once all updates are merged replace github.com/apache/arrow/go/v13 => github.com/cloudquery/arrow/go/v13 v13.0.0-20230626001500-065602842c3a -replace github.com/cloudquery/plugin-sdk/v4 => github.com/cloudquery/plugin-sdk/v4 v4.0.0-20230625074510-d3cd5aad7dec - require ( github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect github.com/andybalholm/brotli v1.0.5 // indirect diff --git a/go.sum b/go.sum index ca09faa9..67ae3d5a 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oM github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= github.com/cloudquery/arrow/go/v13 v13.0.0-20230626001500-065602842c3a h1:O/FNq1+8YlWzHYNj2tokFQyja6GXsQBdkuvLMdpuaSw= github.com/cloudquery/arrow/go/v13 v13.0.0-20230626001500-065602842c3a/go.mod h1:W69eByFNO0ZR30q1/7Sr9d83zcVZmF2MiP3fFYAWJOc= -github.com/cloudquery/plugin-sdk/v4 v4.0.0-20230625074510-d3cd5aad7dec h1:gJbBpfIHovks+ERCCHExtlX42uSg82XWuPNFtLqCp2k= -github.com/cloudquery/plugin-sdk/v4 v4.0.0-20230625074510-d3cd5aad7dec/go.mod h1:RnLpmlnyTeBAouDH7th4h75YHg1TudKVwXMQsjCbuQk= +github.com/cloudquery/plugin-sdk/v4 v4.0.0-rc1 h1:YX8QPlzm5kiwbYsM8zKVXk1yu7ONeooJD77ZoA4hI5o= +github.com/cloudquery/plugin-sdk/v4 v4.0.0-rc1/go.mod h1:QfZgX4QC3Z1SdjnDmtzGAETaPDOYVTSWTW6XZQWHjeQ= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/stream.go b/stream.go new file mode 100644 index 00000000..e6a3d230 --- /dev/null +++ b/stream.go @@ -0,0 +1,92 @@ +package filetypes + +import ( + "fmt" + "io" + + "github.com/apache/arrow/go/v13/arrow" + "github.com/cloudquery/filetypes/v4/types" + "github.com/cloudquery/plugin-sdk/v4/schema" +) + +// Stream helps with streaming uploads by handling header/footer and uploader logic. Use StartStream to start a stream and then Write to it. +type Stream struct { + h types.Handle + wc *writeCloser + done chan error +} + +type writeCloser struct { + *io.PipeWriter + closed bool +} + +func (w *writeCloser) Close() error { + w.closed = true + return w.PipeWriter.Close() +} + +// StartStream starts a streaming upload using the provided uploadFunc. +func (c *Client) StartStream(table *schema.Table, uploadFunc func(io.Reader) error) (*Stream, error) { + pr, pw := io.Pipe() + doneCh := make(chan error) + + go func() { + err := uploadFunc(pr) + _ = pr.CloseWithError(err) + doneCh <- err + close(doneCh) + }() + + wc := &writeCloser{PipeWriter: pw} + h, err := c.WriteHeader(wc, table) + if err != nil { + _ = pw.CloseWithError(err) + <-doneCh + return nil, err + } + + return &Stream{ + h: h, + wc: wc, + done: doneCh, + }, nil +} + +// Write to the stream opened with StartStream. +func (s *Stream) Write(records []arrow.Record) error { + if len(records) == 0 { + return nil + } + + return s.h.WriteContent(records) +} + +// Finish writing to the stream. +func (s *Stream) Finish() error { + return s.FinishWithError(nil) +} + +// FinishWithError aborts writing to the stream by closing the writer with the provided error and waiting for the uploader to finish. +func (s *Stream) FinishWithError(finishError error) error { + if finishError != nil { + _ = s.wc.CloseWithError(finishError) + return <-s.done + } + + if err := s.h.WriteFooter(); err != nil { + if !s.wc.closed { + _ = s.wc.CloseWithError(err) + } + return fmt.Errorf("failed to write footer: %w", <-s.done) + } + + // ParquetWriter likes to close the underlying writer, so we need to check if it's already closed + if !s.wc.closed { + if err := s.wc.Close(); err != nil { + return err + } + } + + return <-s.done +} diff --git a/stream_test.go b/stream_test.go new file mode 100644 index 00000000..b906a15f --- /dev/null +++ b/stream_test.go @@ -0,0 +1,149 @@ +package filetypes_test + +import ( + "bufio" + fmt "fmt" + "io" + "testing" + + "github.com/apache/arrow/go/v13/arrow" + "github.com/apache/arrow/go/v13/arrow/array" + "github.com/apache/arrow/go/v13/arrow/memory" + "github.com/cloudquery/filetypes/v4" + "github.com/cloudquery/plugin-sdk/v4/schema" + "github.com/stretchr/testify/assert" +) + +type uploadHelper struct { + t *testing.T + failAfter int + expect []byte + expectAt int +} + +var errTest = fmt.Errorf("test error") + +func (u *uploadHelper) Upload(r io.Reader) error { + s := bufio.NewScanner(r) + i := 0 + for s.Scan() { + if u.failAfter > 0 && i == u.failAfter { + return errTest + } + if u.expect != nil && i == u.expectAt { + if !assert.Equal(u.t, u.expect, s.Bytes()) { + return fmt.Errorf("assertion failed") + } + } + i++ + } + return s.Err() +} + +func TestHappyPath(t *testing.T) { + cl, err := filetypes.NewClient(&filetypes.FileSpec{ + Format: filetypes.FormatTypeJSON, + }) + if !assert.NoError(t, err) { + return + } + + table := &schema.Table{ + Name: "test", + Columns: []schema.Column{ + {Name: "name", Type: arrow.BinaryTypes.String}, + }, + } + + u := &uploadHelper{ + t: t, + expect: []byte(`{"name":"bar"}`), + expectAt: 1, + } + s, err := cl.StartStream(table, u.Upload) + if !assert.NoError(t, err) { + return + } + + bldr := array.NewRecordBuilder(memory.DefaultAllocator, table.ToArrowSchema()) + bldr.Field(0).(*array.StringBuilder).Append("foo") + bldr.Field(0).(*array.StringBuilder).Append("bar") + record := bldr.NewRecord() + + if !assert.NoError(t, s.Write([]arrow.Record{record})) { + return + } + + assert.NoError(t, s.Finish()) +} + +func TestWriteError(t *testing.T) { + cl, err := filetypes.NewClient(&filetypes.FileSpec{ + Format: filetypes.FormatTypeJSON, + }) + if !assert.NoError(t, err) { + return + } + + table := &schema.Table{ + Name: "test", + Columns: []schema.Column{ + {Name: "name", Type: arrow.BinaryTypes.String}, + }, + } + + u := &uploadHelper{ + t: t, + failAfter: 1, + } + s, err := cl.StartStream(table, u.Upload) + if !assert.NoError(t, err) { + return + } + + bldr := array.NewRecordBuilder(memory.DefaultAllocator, table.ToArrowSchema()) + bldr.Field(0).(*array.StringBuilder).Append("foo") + bldr.Field(0).(*array.StringBuilder).Append("bar") + record := bldr.NewRecord() + + if !assert.NoError(t, s.Write([]arrow.Record{record})) { + return + } + + assert.ErrorIs(t, s.Finish(), errTest) +} + +func TestCloseError(t *testing.T) { + cl, err := filetypes.NewClient(&filetypes.FileSpec{ + Format: filetypes.FormatTypeJSON, + }) + if !assert.NoError(t, err) { + return + } + + table := &schema.Table{ + Name: "test", + Columns: []schema.Column{ + {Name: "name", Type: arrow.BinaryTypes.String}, + }, + } + + u := &uploadHelper{ + t: t, + } + s, err := cl.StartStream(table, u.Upload) + if !assert.NoError(t, err) { + return + } + + bldr := array.NewRecordBuilder(memory.DefaultAllocator, table.ToArrowSchema()) + bldr.Field(0).(*array.StringBuilder).Append("foo") + bldr.Field(0).(*array.StringBuilder).Append("bar") + record := bldr.NewRecord() + + if !assert.NoError(t, s.Write([]arrow.Record{record})) { + return + } + + assert.ErrorIs(t, s.FinishWithError(errTest), errTest) +} From 4948ce1e42a841b174e46386283d70904dac64ad Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Mon, 26 Jun 2023 11:30:52 +0100 Subject: [PATCH 12/16] Tidy --- go.mod | 1 - go.sum | 4 ---- 2 files changed, 5 deletions(-) diff --git a/go.mod b/go.mod index f07dfcee..c5b60c85 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,6 @@ require ( github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect github.com/andybalholm/brotli v1.0.5 // indirect github.com/apache/thrift v0.18.1 // indirect - github.com/cloudquery/plugin-pb-go v1.3.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect diff --git a/go.sum b/go.sum index b2755669..67ae3d5a 100644 --- a/go.sum +++ b/go.sum @@ -10,10 +10,6 @@ github.com/cloudquery/arrow/go/v13 v13.0.0-20230626001500-065602842c3a h1:O/FNq1 github.com/cloudquery/arrow/go/v13 v13.0.0-20230626001500-065602842c3a/go.mod h1:W69eByFNO0ZR30q1/7Sr9d83zcVZmF2MiP3fFYAWJOc= github.com/cloudquery/plugin-sdk/v4 v4.0.0-rc1 h1:YX8QPlzm5kiwbYsM8zKVXk1yu7ONeooJD77ZoA4hI5o= github.com/cloudquery/plugin-sdk/v4 v4.0.0-rc1/go.mod h1:QfZgX4QC3Z1SdjnDmtzGAETaPDOYVTSWTW6XZQWHjeQ= -github.com/cloudquery/plugin-pb-go v1.3.4 h1:GKNVo9gpmvctyD11QaPVnSLYzE9lw1g1kzt4TSmP+2s= -github.com/cloudquery/plugin-pb-go v1.3.4/go.mod h1:NbWAtT2BzJQ9+XUWwh3IKBg3MOeV9ZEpHoHNAQ/YDV8= -github.com/cloudquery/plugin-sdk/v3 v3.10.6 h1:KqTsLZ6OA1h8BUMeMcU6BAD6TBW6ojgQaC4zDZMgvu0= -github.com/cloudquery/plugin-sdk/v3 v3.10.6/go.mod h1:QhBaVgiNyQ3P6uAzJWOYpYykHXL+WDZffwg1riTwv60= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= From f1b4bb9dac8702d7f390c060996638b2412b36fc Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Mon, 26 Jun 2023 11:42:22 +0100 Subject: [PATCH 13/16] update testdata --- csv/testdata/TestWriteRead-default.csv | 4 ++-- csv/testdata/TestWriteRead-with_delimiter.csv | 4 ++-- csv/testdata/TestWriteRead-with_delimiter_headers.csv | 6 +++--- csv/testdata/TestWriteRead-with_headers.csv | 6 +++--- json/testdata/TestWriteRead.jsonl | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/csv/testdata/TestWriteRead-default.csv b/csv/testdata/TestWriteRead-default.csv index c77a27b4..abfb1d8c 100644 --- a/csv/testdata/TestWriteRead-default.csv +++ b/csv/testdata/TestWriteRead-default.csv @@ -1,3 +1,3 @@ -test-source,2021-01-01 00:00:00,00000000-0000-0000-0000-000000000001,00000000-0000-0000-0000-000000000001,AQIDBA==,true,2023-04-24,2023-04-24,"{""days"":1,""milliseconds"":1}",123456789ms,123456789ns,123456789s,123456789us,1.1,1.1,192.0.2.0/24,-1,-1,-1,-1,AQIDBA==,AString,aa:bb:cc:dd:ee:ff,"{""months"":1,""days"":1,""nanoseconds"":1}",1,AString,00:00:00.000,00:00:00,00:00:00.000000000,00:00:00.000000,2021-01-02 00:00:00,2021-01-02 00:00:00,2021-01-02 00:00:00,2021-01-02 00:00:00,1,1,1,1,00000000-0000-0000-0000-000000000001,"{""test"":[""a"",""b"",3]}","[{""test"":""test""},123,{""test_number"":456}]","[true,null,true]","[""2023-04-24"",null,""2023-04-24""]","[""2023-04-24"",null,""2023-04-24""]","[{""days"":1,""milliseconds"":1},null,{""days"":1,""milliseconds"":1}]","[""123456789ms"",null,""123456789ms""]","[""123456789ns"",null,""123456789ns""]","[""123456789s"",null,""123456789s""]","[""123456789us"",null,""123456789us""]","[1.1,null,1.1]","[1.1,null,1.1]","[""192.0.2.0/24"",null,""192.0.2.0/24""]","[-1,null,-1]","[-1,null,-1]","[-1,null,-1]","[-1,null,-1]","[""AString"",null,""AString""]","[""aa:bb:cc:dd:ee:ff"",null,""aa:bb:cc:dd:ee:ff""]","[{""months"":1,""days"":1,""nanoseconds"":1},null,{""months"":1,""days"":1,""nanoseconds"":1}]","[{""months"":1},null,{""months"":1}]","[""AString"",null,""AString""]","[""00:00:00"",null,""00:00:00""]","[""00:00:00"",null,""00:00:00""]","[""00:00:00"",null,""00:00:00""]","[""00:00:00"",null,""00:00:00""]","[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]","[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]","[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]","[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]","[1,null,1]","[1,null,1]","[1,null,1]","[1,null,1]","[""00000000-0000-0000-0000-000000000001"",null,""00000000-0000-0000-0000-000000000001""]","{""binary"":""AQIDBA=="",""boolean"":true,""date32"":""2023-04-24"",""date64"":""2023-04-24"",""daytimeinterval"":{""days"":1,""milliseconds"":1},""duration_ms"":""123456789ms"",""duration_ns"":""123456789ns"",""duration_s"":""123456789s"",""duration_us"":""123456789us"",""float32"":1.1,""float64"":1.1,""inet"":""192.0.2.0/24"",""int16"":-1,""int32"":-1,""int64"":-1,""int8"":-1,""json"":{""test"":[""a"",""b"",3]},""json_array"":[{""test"":""test""},123,{""test_number"":456}],""largebinary"":""AQIDBA=="",""largestring"":""AString"",""mac"":""aa:bb:cc:dd:ee:ff"",""monthdaynanointerval"":{""months"":1,""days"":1,""nanoseconds"":1},""monthinterval"":{""months"":1},""string"":""AString"",""time32ms"":""00:00:00"",""time32s"":""00:00:00"",""time64ns"":""00:00:00"",""time64us"":""00:00:00"",""timestamp_ms"":""2021-01-02 00:00:00"",""timestamp_ns"":""2021-01-02 00:00:00"",""timestamp_s"":""2021-01-02 00:00:00"",""timestamp_us"":""2021-01-02 00:00:00"",""uint16"":1,""uint32"":1,""uint64"":1,""uint8"":1,""uuid"":""00000000-0000-0000-0000-000000000001""}","{""inner"":{""binary"":""AQIDBA=="",""boolean"":true,""date32"":""2023-04-24"",""date64"":""2023-04-24"",""daytimeinterval"":{""days"":1,""milliseconds"":1},""duration_ms"":""123456789ms"",""duration_ns"":""123456789ns"",""duration_s"":""123456789s"",""duration_us"":""123456789us"",""float32"":1.1,""float64"":1.1,""inet"":""192.0.2.0/24"",""int16"":-1,""int32"":-1,""int64"":-1,""int8"":-1,""json"":{""test"":[""a"",""b"",3]},""json_array"":[{""test"":""test""},123,{""test_number"":456}],""largebinary"":""AQIDBA=="",""largestring"":""AString"",""mac"":""aa:bb:cc:dd:ee:ff"",""monthdaynanointerval"":{""months"":1,""days"":1,""nanoseconds"":1},""monthinterval"":{""months"":1},""string"":""AString"",""time32ms"":""00:00:00"",""time32s"":""00:00:00"",""time64ns"":""00:00:00"",""time64us"":""00:00:00"",""timestamp_ms"":""2021-01-02 00:00:00"",""timestamp_ns"":""2021-01-02 00:00:00"",""timestamp_s"":""2021-01-02 00:00:00"",""timestamp_us"":""2021-01-02 00:00:00"",""uint16"":1,""uint32"":1,""uint64"":1,""uint8"":1,""uuid"":""00000000-0000-0000-0000-000000000001""}}" -test-source,2021-01-01 00:00:00,00000000-0000-0000-0000-000000000001,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +test-source,2021-01-01 00:00:00,00000000-0000-0000-0000-000000000001,00000000-0000-0000-0000-000000000001,7049Ug==,true,2023-07-16,2023-07-16,"{""days"":6783,""milliseconds"":6783}",6476783ms,6476783ns,6476783s,6476783us,783.936,783.936,192.0.2.0/24,-30727,-1781555540,-5474557666971701248,-123,7049Ug==,AString76783,aa:bb:cc:dd:ee:ff,"{""months"":6783,""days"":6783,""nanoseconds"":6783}",1,AString76783,00:00:00.000,00:00:00,00:00:00.000000000,00:00:00.000000,2021-01-02 00:00:00,2021-01-02 00:00:00,2021-01-02 00:00:00,2021-01-02 00:00:00,63063,506910753,14697929703826477056,78,00000000-0000-0000-0000-000000000001,826476783.5364813824,"{""test"":[""a"",""b"",3]}","[{""test"":""test""},123,{""test_number"":456}]","[true,null,true]","[""2023-07-16"",null,""2023-07-16""]","[""2023-07-16"",null,""2023-07-16""]","[{""days"":6783,""milliseconds"":6783},null,{""days"":6783,""milliseconds"":6783}]","[""6476783ms"",null,""6476783ms""]","[""6476783ns"",null,""6476783ns""]","[""6476783s"",null,""6476783s""]","[""6476783us"",null,""6476783us""]","[783.936,null,783.936]","[783.936,null,783.936]","[""192.0.2.0/24"",null,""192.0.2.0/24""]","[-30727,null,-30727]","[-1781555540,null,-1781555540]","[-5474557666971701248,null,-5474557666971701248]","[-123,null,-123]","[""AString76783"",null,""AString76783""]","[""aa:bb:cc:dd:ee:ff"",null,""aa:bb:cc:dd:ee:ff""]","[{""months"":6783,""days"":6783,""nanoseconds"":6783},null,{""months"":6783,""days"":6783,""nanoseconds"":6783}]","[{""months"":1},null,{""months"":1}]","[""AString76783"",null,""AString76783""]","[""00:00:00"",null,""00:00:00""]","[""00:00:00"",null,""00:00:00""]","[""00:00:00"",null,""00:00:00""]","[""00:00:00"",null,""00:00:00""]","[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]","[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]","[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]","[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]","[63063,null,63063]","[506910753,null,506910753]","[14697929703826477056,null,14697929703826477056]","[78,null,78]","[""00000000-0000-0000-0000-000000000001"",null,""00000000-0000-0000-0000-000000000001""]","[""826476783.5364813824"",null,""826476783.5364813824""]","[{""key"":""AString76783"",""value"":true},{""key"":""AString89351"",""value"":true}]","[{""key"":-5474557666971701248,""value"":true},{""key"":-3812638179211989504,""value"":true}]","[{""key"":""AString76783"",""value"":""2023-07-16""},{""key"":""AString89351"",""value"":""2023-06-14""}]","[{""key"":-5474557666971701248,""value"":""2023-07-16""},{""key"":-3812638179211989504,""value"":""2023-06-14""}]","[{""key"":""AString76783"",""value"":""2023-07-16""},{""key"":""AString89351"",""value"":""2023-06-14""}]","[{""key"":-5474557666971701248,""value"":""2023-07-16""},{""key"":-3812638179211989504,""value"":""2023-06-14""}]","[{""key"":""AString76783"",""value"":{""days"":6783,""milliseconds"":6783}},{""key"":""AString89351"",""value"":{""days"":9351,""milliseconds"":9351}}]","[{""key"":-5474557666971701248,""value"":{""days"":6783,""milliseconds"":6783}},{""key"":-3812638179211989504,""value"":{""days"":9351,""milliseconds"":9351}}]","[{""key"":""AString76783"",""value"":""6476783ms""},{""key"":""AString89351"",""value"":""1989351ms""}]","[{""key"":-5474557666971701248,""value"":""6476783ms""},{""key"":-3812638179211989504,""value"":""1989351ms""}]","[{""key"":""AString76783"",""value"":""6476783ns""},{""key"":""AString89351"",""value"":""1989351ns""}]","[{""key"":-5474557666971701248,""value"":""6476783ns""},{""key"":-3812638179211989504,""value"":""1989351ns""}]","[{""key"":""AString76783"",""value"":""6476783s""},{""key"":""AString89351"",""value"":""1989351s""}]","[{""key"":-5474557666971701248,""value"":""6476783s""},{""key"":-3812638179211989504,""value"":""1989351s""}]","[{""key"":""AString76783"",""value"":""6476783us""},{""key"":""AString89351"",""value"":""1989351us""}]","[{""key"":-5474557666971701248,""value"":""6476783us""},{""key"":-3812638179211989504,""value"":""1989351us""}]","[{""key"":""AString76783"",""value"":783.936},{""key"":""AString89351"",""value"":351.121}]","[{""key"":-5474557666971701248,""value"":783.936},{""key"":-3812638179211989504,""value"":351.121}]","[{""key"":""AString76783"",""value"":783.936},{""key"":""AString89351"",""value"":351.121}]","[{""key"":-5474557666971701248,""value"":783.936},{""key"":-3812638179211989504,""value"":351.121}]","[{""key"":""AString76783"",""value"":""192.0.2.0/24""},{""key"":""AString89351"",""value"":""192.0.2.0/24""}]","[{""key"":-5474557666971701248,""value"":""192.0.2.0/24""},{""key"":-3812638179211989504,""value"":""192.0.2.0/24""}]","[{""key"":""AString76783"",""value"":-30727},{""key"":""AString89351"",""value"":-25508}]","[{""key"":-5474557666971701248,""value"":-30727},{""key"":-3812638179211989504,""value"":-25508}]","[{""key"":""AString76783"",""value"":-1781555540},{""key"":""AString89351"",""value"":-853032410}]","[{""key"":-5474557666971701248,""value"":-1781555540},{""key"":-3812638179211989504,""value"":-853032410}]","[{""key"":""AString76783"",""value"":-5474557666971701248},{""key"":""AString89351"",""value"":-3812638179211989504}]","[{""key"":-5474557666971701248,""value"":-5474557666971701248},{""key"":-3812638179211989504,""value"":-3812638179211989504}]","[{""key"":""AString76783"",""value"":-123},{""key"":""AString89351"",""value"":-60}]","[{""key"":-5474557666971701248,""value"":-123},{""key"":-3812638179211989504,""value"":-60}]","[{""key"":""AString76783"",""value"":""AString76783""},{""key"":""AString89351"",""value"":""AString89351""}]","[{""key"":-5474557666971701248,""value"":""AString76783""},{""key"":-3812638179211989504,""value"":""AString89351""}]","[{""key"":""AString76783"",""value"":""aa:bb:cc:dd:ee:ff""},{""key"":""AString89351"",""value"":""aa:bb:cc:dd:ee:ff""}]","[{""key"":-5474557666971701248,""value"":""aa:bb:cc:dd:ee:ff""},{""key"":-3812638179211989504,""value"":""aa:bb:cc:dd:ee:ff""}]","[{""key"":""AString76783"",""value"":{""months"":6783,""days"":6783,""nanoseconds"":6783}},{""key"":""AString89351"",""value"":{""months"":9351,""days"":9351,""nanoseconds"":9351}}]","[{""key"":-5474557666971701248,""value"":{""months"":6783,""days"":6783,""nanoseconds"":6783}},{""key"":-3812638179211989504,""value"":{""months"":9351,""days"":9351,""nanoseconds"":9351}}]","[{""key"":""AString76783"",""value"":{""months"":1}},{""key"":""AString89351"",""value"":{""months"":1}}]","[{""key"":-5474557666971701248,""value"":{""months"":1}},{""key"":-3812638179211989504,""value"":{""months"":1}}]","[{""key"":""AString76783"",""value"":""AString76783""},{""key"":""AString89351"",""value"":""AString89351""}]","[{""key"":-5474557666971701248,""value"":""AString76783""},{""key"":-3812638179211989504,""value"":""AString89351""}]","[{""key"":""AString76783"",""value"":""00:00:00""},{""key"":""AString89351"",""value"":""00:00:00""}]","[{""key"":-5474557666971701248,""value"":""00:00:00""},{""key"":-3812638179211989504,""value"":""00:00:00""}]","[{""key"":""AString76783"",""value"":""00:00:00""},{""key"":""AString89351"",""value"":""00:00:00""}]","[{""key"":-5474557666971701248,""value"":""00:00:00""},{""key"":-3812638179211989504,""value"":""00:00:00""}]","[{""key"":""AString76783"",""value"":""00:00:00""},{""key"":""AString89351"",""value"":""00:00:00""}]","[{""key"":-5474557666971701248,""value"":""00:00:00""},{""key"":-3812638179211989504,""value"":""00:00:00""}]","[{""key"":""AString76783"",""value"":""00:00:00""},{""key"":""AString89351"",""value"":""00:00:00""}]","[{""key"":-5474557666971701248,""value"":""00:00:00""},{""key"":-3812638179211989504,""value"":""00:00:00""}]","[{""key"":""AString76783"",""value"":""2021-01-02 00:00:00""},{""key"":""AString89351"",""value"":""2021-01-02 00:00:00""}]","[{""key"":-5474557666971701248,""value"":""2021-01-02 00:00:00""},{""key"":-3812638179211989504,""value"":""2021-01-02 00:00:00""}]","[{""key"":""AString76783"",""value"":""2021-01-02 00:00:00""},{""key"":""AString89351"",""value"":""2021-01-02 00:00:00""}]","[{""key"":-5474557666971701248,""value"":""2021-01-02 00:00:00""},{""key"":-3812638179211989504,""value"":""2021-01-02 00:00:00""}]","[{""key"":""AString76783"",""value"":""2021-01-02 00:00:00""},{""key"":""AString89351"",""value"":""2021-01-02 00:00:00""}]","[{""key"":-5474557666971701248,""value"":""2021-01-02 00:00:00""},{""key"":-3812638179211989504,""value"":""2021-01-02 00:00:00""}]","[{""key"":""AString76783"",""value"":""2021-01-02 00:00:00""},{""key"":""AString89351"",""value"":""2021-01-02 00:00:00""}]","[{""key"":-5474557666971701248,""value"":""2021-01-02 00:00:00""},{""key"":-3812638179211989504,""value"":""2021-01-02 00:00:00""}]","[{""key"":""AString76783"",""value"":63063},{""key"":""AString89351"",""value"":34191}]","[{""key"":-5474557666971701248,""value"":63063},{""key"":-3812638179211989504,""value"":34191}]","[{""key"":""AString76783"",""value"":506910753},{""key"":""AString89351"",""value"":2112817056}]","[{""key"":-5474557666971701248,""value"":506910753},{""key"":-3812638179211989504,""value"":2112817056}]","[{""key"":""AString76783"",""value"":14697929703826477056},{""key"":""AString89351"",""value"":3812638179211989504}]","[{""key"":-5474557666971701248,""value"":14697929703826477056},{""key"":-3812638179211989504,""value"":3812638179211989504}]","[{""key"":""AString76783"",""value"":78},{""key"":""AString89351"",""value"":21}]","[{""key"":-5474557666971701248,""value"":78},{""key"":-3812638179211989504,""value"":21}]","[{""key"":""AString76783"",""value"":""00000000-0000-0000-0000-000000000001""},{""key"":""AString89351"",""value"":""00000000-0000-0000-0000-000000000001""}]","[{""key"":-5474557666971701248,""value"":""00000000-0000-0000-0000-000000000001""},{""key"":-3812638179211989504,""value"":""00000000-0000-0000-0000-000000000001""}]","[{""key"":""AString76783"",""value"":""826476783.5364813824""},{""key"":""AString89351"",""value"":""211989351.8533091328""}]","[{""key"":-5474557666971701248,""value"":""826476783.5364813824""},{""key"":-3812638179211989504,""value"":""211989351.8533091328""}]","{""binary"":""7049Ug=="",""boolean"":true,""date32"":""2023-07-16"",""date64"":""2023-07-16"",""daytimeinterval"":{""days"":6783,""milliseconds"":6783},""decimal"":""826476783.5364813824"",""duration_ms"":""6476783ms"",""duration_ns"":""6476783ns"",""duration_s"":""6476783s"",""duration_us"":""6476783us"",""float32"":783.936,""float64"":783.936,""inet"":""192.0.2.0/24"",""int16"":-30727,""int32"":-1781555540,""int64"":-5474557666971701248,""int8"":-123,""json"":{""test"":[""a"",""b"",3]},""json_array"":[{""test"":""test""},123,{""test_number"":456}],""largebinary"":""7049Ug=="",""largestring"":""AString76783"",""mac"":""aa:bb:cc:dd:ee:ff"",""monthdaynanointerval"":{""months"":6783,""days"":6783,""nanoseconds"":6783},""monthinterval"":{""months"":1},""string"":""AString76783"",""time32ms"":""00:00:00"",""time32s"":""00:00:00"",""time64ns"":""00:00:00"",""time64us"":""00:00:00"",""timestamp_ms"":""2021-01-02 00:00:00"",""timestamp_ns"":""2021-01-02 00:00:00"",""timestamp_s"":""2021-01-02 00:00:00"",""timestamp_us"":""2021-01-02 00:00:00"",""uint16"":63063,""uint32"":506910753,""uint64"":14697929703826477056,""uint8"":78,""uuid"":""00000000-0000-0000-0000-000000000001""}","{""inner"":{""binary"":""7049Ug=="",""boolean"":true,""date32"":""2023-07-16"",""date64"":""2023-07-16"",""daytimeinterval"":{""days"":6783,""milliseconds"":6783},""decimal"":""826476783.5364813824"",""duration_ms"":""6476783ms"",""duration_ns"":""6476783ns"",""duration_s"":""6476783s"",""duration_us"":""6476783us"",""float32"":783.936,""float64"":783.936,""inet"":""192.0.2.0/24"",""int16"":-30727,""int32"":-1781555540,""int64"":-5474557666971701248,""int8"":-123,""json"":{""test"":[""a"",""b"",3]},""json_array"":[{""test"":""test""},123,{""test_number"":456}],""largebinary"":""7049Ug=="",""largestring"":""AString76783"",""mac"":""aa:bb:cc:dd:ee:ff"",""monthdaynanointerval"":{""months"":6783,""days"":6783,""nanoseconds"":6783},""monthinterval"":{""months"":1},""string"":""AString76783"",""time32ms"":""00:00:00"",""time32s"":""00:00:00"",""time64ns"":""00:00:00"",""time64us"":""00:00:00"",""timestamp_ms"":""2021-01-02 00:00:00"",""timestamp_ns"":""2021-01-02 00:00:00"",""timestamp_s"":""2021-01-02 00:00:00"",""timestamp_us"":""2021-01-02 00:00:00"",""uint16"":63063,""uint32"":506910753,""uint64"":14697929703826477056,""uint8"":78,""uuid"":""00000000-0000-0000-0000-000000000001""}}" +test-source,2021-01-01 00:00:00,00000000-0000-0000-0000-000000000001,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, diff --git a/csv/testdata/TestWriteRead-with_delimiter.csv b/csv/testdata/TestWriteRead-with_delimiter.csv index 7f667976..b9655aac 100644 --- a/csv/testdata/TestWriteRead-with_delimiter.csv +++ b/csv/testdata/TestWriteRead-with_delimiter.csv @@ -1,3 +1,3 @@ -test-source 2021-01-01 00:00:00 00000000-0000-0000-0000-000000000001 00000000-0000-0000-0000-000000000001 AQIDBA== true 2023-04-24 2023-04-24 "{""days"":1,""milliseconds"":1}" 123456789ms 123456789ns 123456789s 123456789us 1.1 1.1 192.0.2.0/24 -1 -1 -1 -1 AQIDBA== AString aa:bb:cc:dd:ee:ff "{""months"":1,""days"":1,""nanoseconds"":1}" 1 AString 00:00:00.000 00:00:00 00:00:00.000000000 00:00:00.000000 2021-01-02 00:00:00 2021-01-02 00:00:00 2021-01-02 00:00:00 2021-01-02 00:00:00 1 1 1 1 00000000-0000-0000-0000-000000000001 "{""test"":[""a"",""b"",3]}" "[{""test"":""test""},123,{""test_number"":456}]" [true,null,true] "[""2023-04-24"",null,""2023-04-24""]" "[""2023-04-24"",null,""2023-04-24""]" "[{""days"":1,""milliseconds"":1},null,{""days"":1,""milliseconds"":1}]" "[""123456789ms"",null,""123456789ms""]" "[""123456789ns"",null,""123456789ns""]" "[""123456789s"",null,""123456789s""]" "[""123456789us"",null,""123456789us""]" [1.1,null,1.1] [1.1,null,1.1] "[""192.0.2.0/24"",null,""192.0.2.0/24""]" [-1,null,-1] [-1,null,-1] [-1,null,-1] [-1,null,-1] "[""AString"",null,""AString""]" "[""aa:bb:cc:dd:ee:ff"",null,""aa:bb:cc:dd:ee:ff""]" "[{""months"":1,""days"":1,""nanoseconds"":1},null,{""months"":1,""days"":1,""nanoseconds"":1}]" "[{""months"":1},null,{""months"":1}]" "[""AString"",null,""AString""]" "[""00:00:00"",null,""00:00:00""]" "[""00:00:00"",null,""00:00:00""]" "[""00:00:00"",null,""00:00:00""]" "[""00:00:00"",null,""00:00:00""]" "[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]" "[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]" "[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]" "[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]" [1,null,1] [1,null,1] [1,null,1] [1,null,1] "[""00000000-0000-0000-0000-000000000001"",null,""00000000-0000-0000-0000-000000000001""]" "{""binary"":""AQIDBA=="",""boolean"":true,""date32"":""2023-04-24"",""date64"":""2023-04-24"",""daytimeinterval"":{""days"":1,""milliseconds"":1},""duration_ms"":""123456789ms"",""duration_ns"":""123456789ns"",""duration_s"":""123456789s"",""duration_us"":""123456789us"",""float32"":1.1,""float64"":1.1,""inet"":""192.0.2.0/24"",""int16"":-1,""int32"":-1,""int64"":-1,""int8"":-1,""json"":{""test"":[""a"",""b"",3]},""json_array"":[{""test"":""test""},123,{""test_number"":456}],""largebinary"":""AQIDBA=="",""largestring"":""AString"",""mac"":""aa:bb:cc:dd:ee:ff"",""monthdaynanointerval"":{""months"":1,""days"":1,""nanoseconds"":1},""monthinterval"":{""months"":1},""string"":""AString"",""time32ms"":""00:00:00"",""time32s"":""00:00:00"",""time64ns"":""00:00:00"",""time64us"":""00:00:00"",""timestamp_ms"":""2021-01-02 00:00:00"",""timestamp_ns"":""2021-01-02 00:00:00"",""timestamp_s"":""2021-01-02 00:00:00"",""timestamp_us"":""2021-01-02 00:00:00"",""uint16"":1,""uint32"":1,""uint64"":1,""uint8"":1,""uuid"":""00000000-0000-0000-0000-000000000001""}" "{""inner"":{""binary"":""AQIDBA=="",""boolean"":true,""date32"":""2023-04-24"",""date64"":""2023-04-24"",""daytimeinterval"":{""days"":1,""milliseconds"":1},""duration_ms"":""123456789ms"",""duration_ns"":""123456789ns"",""duration_s"":""123456789s"",""duration_us"":""123456789us"",""float32"":1.1,""float64"":1.1,""inet"":""192.0.2.0/24"",""int16"":-1,""int32"":-1,""int64"":-1,""int8"":-1,""json"":{""test"":[""a"",""b"",3]},""json_array"":[{""test"":""test""},123,{""test_number"":456}],""largebinary"":""AQIDBA=="",""largestring"":""AString"",""mac"":""aa:bb:cc:dd:ee:ff"",""monthdaynanointerval"":{""months"":1,""days"":1,""nanoseconds"":1},""monthinterval"":{""months"":1},""string"":""AString"",""time32ms"":""00:00:00"",""time32s"":""00:00:00"",""time64ns"":""00:00:00"",""time64us"":""00:00:00"",""timestamp_ms"":""2021-01-02 00:00:00"",""timestamp_ns"":""2021-01-02 00:00:00"",""timestamp_s"":""2021-01-02 00:00:00"",""timestamp_us"":""2021-01-02 00:00:00"",""uint16"":1,""uint32"":1,""uint64"":1,""uint8"":1,""uuid"":""00000000-0000-0000-0000-000000000001""}}" -test-source 2021-01-01 00:00:00 00000000-0000-0000-0000-000000000001 +test-source 2021-01-01 00:00:00 00000000-0000-0000-0000-000000000001 00000000-0000-0000-0000-000000000001 7049Ug== true 2023-07-16 2023-07-16 "{""days"":6783,""milliseconds"":6783}" 6476783ms 6476783ns 6476783s 6476783us 783.936 783.936 192.0.2.0/24 -30727 -1781555540 -5474557666971701248 -123 7049Ug== AString76783 aa:bb:cc:dd:ee:ff "{""months"":6783,""days"":6783,""nanoseconds"":6783}" 1 AString76783 00:00:00.000 00:00:00 00:00:00.000000000 00:00:00.000000 2021-01-02 00:00:00 2021-01-02 00:00:00 2021-01-02 00:00:00 2021-01-02 00:00:00 63063 506910753 14697929703826477056 78 00000000-0000-0000-0000-000000000001 826476783.5364813824 "{""test"":[""a"",""b"",3]}" "[{""test"":""test""},123,{""test_number"":456}]" [true,null,true] "[""2023-07-16"",null,""2023-07-16""]" "[""2023-07-16"",null,""2023-07-16""]" "[{""days"":6783,""milliseconds"":6783},null,{""days"":6783,""milliseconds"":6783}]" "[""6476783ms"",null,""6476783ms""]" "[""6476783ns"",null,""6476783ns""]" "[""6476783s"",null,""6476783s""]" "[""6476783us"",null,""6476783us""]" [783.936,null,783.936] [783.936,null,783.936] "[""192.0.2.0/24"",null,""192.0.2.0/24""]" [-30727,null,-30727] [-1781555540,null,-1781555540] [-5474557666971701248,null,-5474557666971701248] [-123,null,-123] "[""AString76783"",null,""AString76783""]" "[""aa:bb:cc:dd:ee:ff"",null,""aa:bb:cc:dd:ee:ff""]" "[{""months"":6783,""days"":6783,""nanoseconds"":6783},null,{""months"":6783,""days"":6783,""nanoseconds"":6783}]" "[{""months"":1},null,{""months"":1}]" "[""AString76783"",null,""AString76783""]" "[""00:00:00"",null,""00:00:00""]" "[""00:00:00"",null,""00:00:00""]" "[""00:00:00"",null,""00:00:00""]" "[""00:00:00"",null,""00:00:00""]" "[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]" "[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]" "[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]" "[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]" [63063,null,63063] [506910753,null,506910753] [14697929703826477056,null,14697929703826477056] [78,null,78] "[""00000000-0000-0000-0000-000000000001"",null,""00000000-0000-0000-0000-000000000001""]" "[""826476783.5364813824"",null,""826476783.5364813824""]" "[{""key"":""AString76783"",""value"":true},{""key"":""AString89351"",""value"":true}]" "[{""key"":-5474557666971701248,""value"":true},{""key"":-3812638179211989504,""value"":true}]" "[{""key"":""AString76783"",""value"":""2023-07-16""},{""key"":""AString89351"",""value"":""2023-06-14""}]" "[{""key"":-5474557666971701248,""value"":""2023-07-16""},{""key"":-3812638179211989504,""value"":""2023-06-14""}]" "[{""key"":""AString76783"",""value"":""2023-07-16""},{""key"":""AString89351"",""value"":""2023-06-14""}]" "[{""key"":-5474557666971701248,""value"":""2023-07-16""},{""key"":-3812638179211989504,""value"":""2023-06-14""}]" "[{""key"":""AString76783"",""value"":{""days"":6783,""milliseconds"":6783}},{""key"":""AString89351"",""value"":{""days"":9351,""milliseconds"":9351}}]" "[{""key"":-5474557666971701248,""value"":{""days"":6783,""milliseconds"":6783}},{""key"":-3812638179211989504,""value"":{""days"":9351,""milliseconds"":9351}}]" "[{""key"":""AString76783"",""value"":""6476783ms""},{""key"":""AString89351"",""value"":""1989351ms""}]" "[{""key"":-5474557666971701248,""value"":""6476783ms""},{""key"":-3812638179211989504,""value"":""1989351ms""}]" "[{""key"":""AString76783"",""value"":""6476783ns""},{""key"":""AString89351"",""value"":""1989351ns""}]" "[{""key"":-5474557666971701248,""value"":""6476783ns""},{""key"":-3812638179211989504,""value"":""1989351ns""}]" "[{""key"":""AString76783"",""value"":""6476783s""},{""key"":""AString89351"",""value"":""1989351s""}]" "[{""key"":-5474557666971701248,""value"":""6476783s""},{""key"":-3812638179211989504,""value"":""1989351s""}]" "[{""key"":""AString76783"",""value"":""6476783us""},{""key"":""AString89351"",""value"":""1989351us""}]" "[{""key"":-5474557666971701248,""value"":""6476783us""},{""key"":-3812638179211989504,""value"":""1989351us""}]" "[{""key"":""AString76783"",""value"":783.936},{""key"":""AString89351"",""value"":351.121}]" "[{""key"":-5474557666971701248,""value"":783.936},{""key"":-3812638179211989504,""value"":351.121}]" "[{""key"":""AString76783"",""value"":783.936},{""key"":""AString89351"",""value"":351.121}]" "[{""key"":-5474557666971701248,""value"":783.936},{""key"":-3812638179211989504,""value"":351.121}]" "[{""key"":""AString76783"",""value"":""192.0.2.0/24""},{""key"":""AString89351"",""value"":""192.0.2.0/24""}]" "[{""key"":-5474557666971701248,""value"":""192.0.2.0/24""},{""key"":-3812638179211989504,""value"":""192.0.2.0/24""}]" "[{""key"":""AString76783"",""value"":-30727},{""key"":""AString89351"",""value"":-25508}]" "[{""key"":-5474557666971701248,""value"":-30727},{""key"":-3812638179211989504,""value"":-25508}]" "[{""key"":""AString76783"",""value"":-1781555540},{""key"":""AString89351"",""value"":-853032410}]" "[{""key"":-5474557666971701248,""value"":-1781555540},{""key"":-3812638179211989504,""value"":-853032410}]" "[{""key"":""AString76783"",""value"":-5474557666971701248},{""key"":""AString89351"",""value"":-3812638179211989504}]" "[{""key"":-5474557666971701248,""value"":-5474557666971701248},{""key"":-3812638179211989504,""value"":-3812638179211989504}]" "[{""key"":""AString76783"",""value"":-123},{""key"":""AString89351"",""value"":-60}]" "[{""key"":-5474557666971701248,""value"":-123},{""key"":-3812638179211989504,""value"":-60}]" "[{""key"":""AString76783"",""value"":""AString76783""},{""key"":""AString89351"",""value"":""AString89351""}]" "[{""key"":-5474557666971701248,""value"":""AString76783""},{""key"":-3812638179211989504,""value"":""AString89351""}]" "[{""key"":""AString76783"",""value"":""aa:bb:cc:dd:ee:ff""},{""key"":""AString89351"",""value"":""aa:bb:cc:dd:ee:ff""}]" "[{""key"":-5474557666971701248,""value"":""aa:bb:cc:dd:ee:ff""},{""key"":-3812638179211989504,""value"":""aa:bb:cc:dd:ee:ff""}]" "[{""key"":""AString76783"",""value"":{""months"":6783,""days"":6783,""nanoseconds"":6783}},{""key"":""AString89351"",""value"":{""months"":9351,""days"":9351,""nanoseconds"":9351}}]" "[{""key"":-5474557666971701248,""value"":{""months"":6783,""days"":6783,""nanoseconds"":6783}},{""key"":-3812638179211989504,""value"":{""months"":9351,""days"":9351,""nanoseconds"":9351}}]" "[{""key"":""AString76783"",""value"":{""months"":1}},{""key"":""AString89351"",""value"":{""months"":1}}]" "[{""key"":-5474557666971701248,""value"":{""months"":1}},{""key"":-3812638179211989504,""value"":{""months"":1}}]" "[{""key"":""AString76783"",""value"":""AString76783""},{""key"":""AString89351"",""value"":""AString89351""}]" "[{""key"":-5474557666971701248,""value"":""AString76783""},{""key"":-3812638179211989504,""value"":""AString89351""}]" "[{""key"":""AString76783"",""value"":""00:00:00""},{""key"":""AString89351"",""value"":""00:00:00""}]" "[{""key"":-5474557666971701248,""value"":""00:00:00""},{""key"":-3812638179211989504,""value"":""00:00:00""}]" "[{""key"":""AString76783"",""value"":""00:00:00""},{""key"":""AString89351"",""value"":""00:00:00""}]" "[{""key"":-5474557666971701248,""value"":""00:00:00""},{""key"":-3812638179211989504,""value"":""00:00:00""}]" "[{""key"":""AString76783"",""value"":""00:00:00""},{""key"":""AString89351"",""value"":""00:00:00""}]" "[{""key"":-5474557666971701248,""value"":""00:00:00""},{""key"":-3812638179211989504,""value"":""00:00:00""}]" "[{""key"":""AString76783"",""value"":""00:00:00""},{""key"":""AString89351"",""value"":""00:00:00""}]" "[{""key"":-5474557666971701248,""value"":""00:00:00""},{""key"":-3812638179211989504,""value"":""00:00:00""}]" "[{""key"":""AString76783"",""value"":""2021-01-02 00:00:00""},{""key"":""AString89351"",""value"":""2021-01-02 00:00:00""}]" "[{""key"":-5474557666971701248,""value"":""2021-01-02 00:00:00""},{""key"":-3812638179211989504,""value"":""2021-01-02 00:00:00""}]" "[{""key"":""AString76783"",""value"":""2021-01-02 00:00:00""},{""key"":""AString89351"",""value"":""2021-01-02 00:00:00""}]" "[{""key"":-5474557666971701248,""value"":""2021-01-02 00:00:00""},{""key"":-3812638179211989504,""value"":""2021-01-02 00:00:00""}]" "[{""key"":""AString76783"",""value"":""2021-01-02 00:00:00""},{""key"":""AString89351"",""value"":""2021-01-02 00:00:00""}]" "[{""key"":-5474557666971701248,""value"":""2021-01-02 00:00:00""},{""key"":-3812638179211989504,""value"":""2021-01-02 00:00:00""}]" "[{""key"":""AString76783"",""value"":""2021-01-02 00:00:00""},{""key"":""AString89351"",""value"":""2021-01-02 00:00:00""}]" "[{""key"":-5474557666971701248,""value"":""2021-01-02 00:00:00""},{""key"":-3812638179211989504,""value"":""2021-01-02 00:00:00""}]" "[{""key"":""AString76783"",""value"":63063},{""key"":""AString89351"",""value"":34191}]" "[{""key"":-5474557666971701248,""value"":63063},{""key"":-3812638179211989504,""value"":34191}]" "[{""key"":""AString76783"",""value"":506910753},{""key"":""AString89351"",""value"":2112817056}]" "[{""key"":-5474557666971701248,""value"":506910753},{""key"":-3812638179211989504,""value"":2112817056}]" "[{""key"":""AString76783"",""value"":14697929703826477056},{""key"":""AString89351"",""value"":3812638179211989504}]" "[{""key"":-5474557666971701248,""value"":14697929703826477056},{""key"":-3812638179211989504,""value"":3812638179211989504}]" "[{""key"":""AString76783"",""value"":78},{""key"":""AString89351"",""value"":21}]" "[{""key"":-5474557666971701248,""value"":78},{""key"":-3812638179211989504,""value"":21}]" "[{""key"":""AString76783"",""value"":""00000000-0000-0000-0000-000000000001""},{""key"":""AString89351"",""value"":""00000000-0000-0000-0000-000000000001""}]" "[{""key"":-5474557666971701248,""value"":""00000000-0000-0000-0000-000000000001""},{""key"":-3812638179211989504,""value"":""00000000-0000-0000-0000-000000000001""}]" "[{""key"":""AString76783"",""value"":""826476783.5364813824""},{""key"":""AString89351"",""value"":""211989351.8533091328""}]" "[{""key"":-5474557666971701248,""value"":""826476783.5364813824""},{""key"":-3812638179211989504,""value"":""211989351.8533091328""}]" "{""binary"":""7049Ug=="",""boolean"":true,""date32"":""2023-07-16"",""date64"":""2023-07-16"",""daytimeinterval"":{""days"":6783,""milliseconds"":6783},""decimal"":""826476783.5364813824"",""duration_ms"":""6476783ms"",""duration_ns"":""6476783ns"",""duration_s"":""6476783s"",""duration_us"":""6476783us"",""float32"":783.936,""float64"":783.936,""inet"":""192.0.2.0/24"",""int16"":-30727,""int32"":-1781555540,""int64"":-5474557666971701248,""int8"":-123,""json"":{""test"":[""a"",""b"",3]},""json_array"":[{""test"":""test""},123,{""test_number"":456}],""largebinary"":""7049Ug=="",""largestring"":""AString76783"",""mac"":""aa:bb:cc:dd:ee:ff"",""monthdaynanointerval"":{""months"":6783,""days"":6783,""nanoseconds"":6783},""monthinterval"":{""months"":1},""string"":""AString76783"",""time32ms"":""00:00:00"",""time32s"":""00:00:00"",""time64ns"":""00:00:00"",""time64us"":""00:00:00"",""timestamp_ms"":""2021-01-02 00:00:00"",""timestamp_ns"":""2021-01-02 00:00:00"",""timestamp_s"":""2021-01-02 00:00:00"",""timestamp_us"":""2021-01-02 00:00:00"",""uint16"":63063,""uint32"":506910753,""uint64"":14697929703826477056,""uint8"":78,""uuid"":""00000000-0000-0000-0000-000000000001""}" "{""inner"":{""binary"":""7049Ug=="",""boolean"":true,""date32"":""2023-07-16"",""date64"":""2023-07-16"",""daytimeinterval"":{""days"":6783,""milliseconds"":6783},""decimal"":""826476783.5364813824"",""duration_ms"":""6476783ms"",""duration_ns"":""6476783ns"",""duration_s"":""6476783s"",""duration_us"":""6476783us"",""float32"":783.936,""float64"":783.936,""inet"":""192.0.2.0/24"",""int16"":-30727,""int32"":-1781555540,""int64"":-5474557666971701248,""int8"":-123,""json"":{""test"":[""a"",""b"",3]},""json_array"":[{""test"":""test""},123,{""test_number"":456}],""largebinary"":""7049Ug=="",""largestring"":""AString76783"",""mac"":""aa:bb:cc:dd:ee:ff"",""monthdaynanointerval"":{""months"":6783,""days"":6783,""nanoseconds"":6783},""monthinterval"":{""months"":1},""string"":""AString76783"",""time32ms"":""00:00:00"",""time32s"":""00:00:00"",""time64ns"":""00:00:00"",""time64us"":""00:00:00"",""timestamp_ms"":""2021-01-02 00:00:00"",""timestamp_ns"":""2021-01-02 00:00:00"",""timestamp_s"":""2021-01-02 00:00:00"",""timestamp_us"":""2021-01-02 00:00:00"",""uint16"":63063,""uint32"":506910753,""uint64"":14697929703826477056,""uint8"":78,""uuid"":""00000000-0000-0000-0000-000000000001""}}" +test-source 2021-01-01 00:00:00 00000000-0000-0000-0000-000000000001 diff --git a/csv/testdata/TestWriteRead-with_delimiter_headers.csv b/csv/testdata/TestWriteRead-with_delimiter_headers.csv index 117c955f..b0fdbfcb 100644 --- a/csv/testdata/TestWriteRead-with_delimiter_headers.csv +++ b/csv/testdata/TestWriteRead-with_delimiter_headers.csv @@ -1,4 +1,4 @@ -_cq_source_name _cq_sync_time _cq_id _cq_parent_id binary boolean date32 date64 daytimeinterval duration_ms duration_ns duration_s duration_us float32 float64 inet int16 int32 int64 int8 largebinary largestring mac monthdaynanointerval monthinterval string time32ms time32s time64ns time64us timestamp_ms timestamp_ns timestamp_s timestamp_us uint16 uint32 uint64 uint8 uuid json json_array boolean_list date32_list date64_list daytimeinterval_list duration_ms_list duration_ns_list duration_s_list duration_us_list float32_list float64_list inet_list int16_list int32_list int64_list int8_list largestring_list mac_list monthdaynanointerval_list monthinterval_list string_list time32ms_list time32s_list time64ns_list time64us_list timestamp_ms_list timestamp_ns_list timestamp_s_list timestamp_us_list uint16_list uint32_list uint64_list uint8_list uuid_list struct nested_struct -test-source 2021-01-01 00:00:00 00000000-0000-0000-0000-000000000001 00000000-0000-0000-0000-000000000001 AQIDBA== true 2023-04-24 2023-04-24 "{""days"":1,""milliseconds"":1}" 123456789ms 123456789ns 123456789s 123456789us 1.1 1.1 192.0.2.0/24 -1 -1 -1 -1 AQIDBA== AString aa:bb:cc:dd:ee:ff "{""months"":1,""days"":1,""nanoseconds"":1}" 1 AString 00:00:00.000 00:00:00 00:00:00.000000000 00:00:00.000000 2021-01-02 00:00:00 2021-01-02 00:00:00 2021-01-02 00:00:00 2021-01-02 00:00:00 1 1 1 1 00000000-0000-0000-0000-000000000001 "{""test"":[""a"",""b"",3]}" "[{""test"":""test""},123,{""test_number"":456}]" [true,null,true] "[""2023-04-24"",null,""2023-04-24""]" "[""2023-04-24"",null,""2023-04-24""]" "[{""days"":1,""milliseconds"":1},null,{""days"":1,""milliseconds"":1}]" "[""123456789ms"",null,""123456789ms""]" "[""123456789ns"",null,""123456789ns""]" "[""123456789s"",null,""123456789s""]" "[""123456789us"",null,""123456789us""]" [1.1,null,1.1] [1.1,null,1.1] "[""192.0.2.0/24"",null,""192.0.2.0/24""]" [-1,null,-1] [-1,null,-1] [-1,null,-1] [-1,null,-1] "[""AString"",null,""AString""]" "[""aa:bb:cc:dd:ee:ff"",null,""aa:bb:cc:dd:ee:ff""]" "[{""months"":1,""days"":1,""nanoseconds"":1},null,{""months"":1,""days"":1,""nanoseconds"":1}]" "[{""months"":1},null,{""months"":1}]" "[""AString"",null,""AString""]" "[""00:00:00"",null,""00:00:00""]" "[""00:00:00"",null,""00:00:00""]" "[""00:00:00"",null,""00:00:00""]" "[""00:00:00"",null,""00:00:00""]" "[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]" "[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]" "[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]" "[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]" [1,null,1] [1,null,1] [1,null,1] [1,null,1] "[""00000000-0000-0000-0000-000000000001"",null,""00000000-0000-0000-0000-000000000001""]" "{""binary"":""AQIDBA=="",""boolean"":true,""date32"":""2023-04-24"",""date64"":""2023-04-24"",""daytimeinterval"":{""days"":1,""milliseconds"":1},""duration_ms"":""123456789ms"",""duration_ns"":""123456789ns"",""duration_s"":""123456789s"",""duration_us"":""123456789us"",""float32"":1.1,""float64"":1.1,""inet"":""192.0.2.0/24"",""int16"":-1,""int32"":-1,""int64"":-1,""int8"":-1,""json"":{""test"":[""a"",""b"",3]},""json_array"":[{""test"":""test""},123,{""test_number"":456}],""largebinary"":""AQIDBA=="",""largestring"":""AString"",""mac"":""aa:bb:cc:dd:ee:ff"",""monthdaynanointerval"":{""months"":1,""days"":1,""nanoseconds"":1},""monthinterval"":{""months"":1},""string"":""AString"",""time32ms"":""00:00:00"",""time32s"":""00:00:00"",""time64ns"":""00:00:00"",""time64us"":""00:00:00"",""timestamp_ms"":""2021-01-02 00:00:00"",""timestamp_ns"":""2021-01-02 00:00:00"",""timestamp_s"":""2021-01-02 00:00:00"",""timestamp_us"":""2021-01-02 00:00:00"",""uint16"":1,""uint32"":1,""uint64"":1,""uint8"":1,""uuid"":""00000000-0000-0000-0000-000000000001""}" "{""inner"":{""binary"":""AQIDBA=="",""boolean"":true,""date32"":""2023-04-24"",""date64"":""2023-04-24"",""daytimeinterval"":{""days"":1,""milliseconds"":1},""duration_ms"":""123456789ms"",""duration_ns"":""123456789ns"",""duration_s"":""123456789s"",""duration_us"":""123456789us"",""float32"":1.1,""float64"":1.1,""inet"":""192.0.2.0/24"",""int16"":-1,""int32"":-1,""int64"":-1,""int8"":-1,""json"":{""test"":[""a"",""b"",3]},""json_array"":[{""test"":""test""},123,{""test_number"":456}],""largebinary"":""AQIDBA=="",""largestring"":""AString"",""mac"":""aa:bb:cc:dd:ee:ff"",""monthdaynanointerval"":{""months"":1,""days"":1,""nanoseconds"":1},""monthinterval"":{""months"":1},""string"":""AString"",""time32ms"":""00:00:00"",""time32s"":""00:00:00"",""time64ns"":""00:00:00"",""time64us"":""00:00:00"",""timestamp_ms"":""2021-01-02 00:00:00"",""timestamp_ns"":""2021-01-02 00:00:00"",""timestamp_s"":""2021-01-02 00:00:00"",""timestamp_us"":""2021-01-02 00:00:00"",""uint16"":1,""uint32"":1,""uint64"":1,""uint8"":1,""uuid"":""00000000-0000-0000-0000-000000000001""}}" -test-source 2021-01-01 00:00:00 00000000-0000-0000-0000-000000000001 +_cq_source_name _cq_sync_time _cq_id _cq_parent_id binary boolean date32 date64 daytimeinterval duration_ms duration_ns duration_s duration_us float32 float64 inet int16 int32 int64 int8 largebinary largestring mac monthdaynanointerval monthinterval string time32ms time32s time64ns time64us timestamp_ms timestamp_ns timestamp_s timestamp_us uint16 uint32 uint64 uint8 uuid decimal json json_array boolean_list date32_list date64_list daytimeinterval_list duration_ms_list duration_ns_list duration_s_list duration_us_list float32_list float64_list inet_list int16_list int32_list int64_list int8_list largestring_list mac_list monthdaynanointerval_list monthinterval_list string_list time32ms_list time32s_list time64ns_list time64us_list timestamp_ms_list timestamp_ns_list timestamp_s_list timestamp_us_list uint16_list uint32_list uint64_list uint8_list uuid_list decimal_list string_boolean_map int_boolean_map string_date32_map int_date32_map string_date64_map int_date64_map string_daytimeinterval_map int_daytimeinterval_map string_duration_ms_map int_duration_ms_map string_duration_ns_map int_duration_ns_map string_duration_s_map int_duration_s_map string_duration_us_map int_duration_us_map string_float32_map int_float32_map string_float64_map int_float64_map string_inet_map int_inet_map string_int16_map int_int16_map string_int32_map int_int32_map string_int64_map int_int64_map string_int8_map int_int8_map string_largestring_map int_largestring_map string_mac_map int_mac_map string_monthdaynanointerval_map int_monthdaynanointerval_map string_monthinterval_map int_monthinterval_map string_string_map int_string_map string_time32ms_map int_time32ms_map string_time32s_map int_time32s_map string_time64ns_map int_time64ns_map string_time64us_map int_time64us_map string_timestamp_ms_map int_timestamp_ms_map string_timestamp_ns_map int_timestamp_ns_map string_timestamp_s_map int_timestamp_s_map string_timestamp_us_map int_timestamp_us_map string_uint16_map int_uint16_map string_uint32_map int_uint32_map string_uint64_map int_uint64_map string_uint8_map int_uint8_map string_uuid_map int_uuid_map string_decimal_map int_decimal_map struct nested_struct +test-source 2021-01-01 00:00:00 00000000-0000-0000-0000-000000000001 00000000-0000-0000-0000-000000000001 7049Ug== true 2023-07-16 2023-07-16 "{""days"":6783,""milliseconds"":6783}" 6476783ms 6476783ns 6476783s 6476783us 783.936 783.936 192.0.2.0/24 -30727 -1781555540 -5474557666971701248 -123 7049Ug== AString76783 aa:bb:cc:dd:ee:ff "{""months"":6783,""days"":6783,""nanoseconds"":6783}" 1 AString76783 00:00:00.000 00:00:00 00:00:00.000000000 00:00:00.000000 2021-01-02 00:00:00 2021-01-02 00:00:00 2021-01-02 00:00:00 2021-01-02 00:00:00 63063 506910753 14697929703826477056 78 00000000-0000-0000-0000-000000000001 826476783.5364813824 "{""test"":[""a"",""b"",3]}" "[{""test"":""test""},123,{""test_number"":456}]" [true,null,true] "[""2023-07-16"",null,""2023-07-16""]" "[""2023-07-16"",null,""2023-07-16""]" "[{""days"":6783,""milliseconds"":6783},null,{""days"":6783,""milliseconds"":6783}]" "[""6476783ms"",null,""6476783ms""]" "[""6476783ns"",null,""6476783ns""]" "[""6476783s"",null,""6476783s""]" "[""6476783us"",null,""6476783us""]" [783.936,null,783.936] [783.936,null,783.936] "[""192.0.2.0/24"",null,""192.0.2.0/24""]" [-30727,null,-30727] [-1781555540,null,-1781555540] [-5474557666971701248,null,-5474557666971701248] [-123,null,-123] "[""AString76783"",null,""AString76783""]" "[""aa:bb:cc:dd:ee:ff"",null,""aa:bb:cc:dd:ee:ff""]" "[{""months"":6783,""days"":6783,""nanoseconds"":6783},null,{""months"":6783,""days"":6783,""nanoseconds"":6783}]" "[{""months"":1},null,{""months"":1}]" "[""AString76783"",null,""AString76783""]" "[""00:00:00"",null,""00:00:00""]" "[""00:00:00"",null,""00:00:00""]" "[""00:00:00"",null,""00:00:00""]" "[""00:00:00"",null,""00:00:00""]" "[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]" "[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]" "[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]" "[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]" [63063,null,63063] [506910753,null,506910753] [14697929703826477056,null,14697929703826477056] [78,null,78] "[""00000000-0000-0000-0000-000000000001"",null,""00000000-0000-0000-0000-000000000001""]" "[""826476783.5364813824"",null,""826476783.5364813824""]" "[{""key"":""AString76783"",""value"":true},{""key"":""AString89351"",""value"":true}]" "[{""key"":-5474557666971701248,""value"":true},{""key"":-3812638179211989504,""value"":true}]" "[{""key"":""AString76783"",""value"":""2023-07-16""},{""key"":""AString89351"",""value"":""2023-06-14""}]" "[{""key"":-5474557666971701248,""value"":""2023-07-16""},{""key"":-3812638179211989504,""value"":""2023-06-14""}]" "[{""key"":""AString76783"",""value"":""2023-07-16""},{""key"":""AString89351"",""value"":""2023-06-14""}]" "[{""key"":-5474557666971701248,""value"":""2023-07-16""},{""key"":-3812638179211989504,""value"":""2023-06-14""}]" "[{""key"":""AString76783"",""value"":{""days"":6783,""milliseconds"":6783}},{""key"":""AString89351"",""value"":{""days"":9351,""milliseconds"":9351}}]" "[{""key"":-5474557666971701248,""value"":{""days"":6783,""milliseconds"":6783}},{""key"":-3812638179211989504,""value"":{""days"":9351,""milliseconds"":9351}}]" "[{""key"":""AString76783"",""value"":""6476783ms""},{""key"":""AString89351"",""value"":""1989351ms""}]" "[{""key"":-5474557666971701248,""value"":""6476783ms""},{""key"":-3812638179211989504,""value"":""1989351ms""}]" "[{""key"":""AString76783"",""value"":""6476783ns""},{""key"":""AString89351"",""value"":""1989351ns""}]" "[{""key"":-5474557666971701248,""value"":""6476783ns""},{""key"":-3812638179211989504,""value"":""1989351ns""}]" "[{""key"":""AString76783"",""value"":""6476783s""},{""key"":""AString89351"",""value"":""1989351s""}]" "[{""key"":-5474557666971701248,""value"":""6476783s""},{""key"":-3812638179211989504,""value"":""1989351s""}]" "[{""key"":""AString76783"",""value"":""6476783us""},{""key"":""AString89351"",""value"":""1989351us""}]" "[{""key"":-5474557666971701248,""value"":""6476783us""},{""key"":-3812638179211989504,""value"":""1989351us""}]" "[{""key"":""AString76783"",""value"":783.936},{""key"":""AString89351"",""value"":351.121}]" "[{""key"":-5474557666971701248,""value"":783.936},{""key"":-3812638179211989504,""value"":351.121}]" "[{""key"":""AString76783"",""value"":783.936},{""key"":""AString89351"",""value"":351.121}]" "[{""key"":-5474557666971701248,""value"":783.936},{""key"":-3812638179211989504,""value"":351.121}]" "[{""key"":""AString76783"",""value"":""192.0.2.0/24""},{""key"":""AString89351"",""value"":""192.0.2.0/24""}]" "[{""key"":-5474557666971701248,""value"":""192.0.2.0/24""},{""key"":-3812638179211989504,""value"":""192.0.2.0/24""}]" "[{""key"":""AString76783"",""value"":-30727},{""key"":""AString89351"",""value"":-25508}]" "[{""key"":-5474557666971701248,""value"":-30727},{""key"":-3812638179211989504,""value"":-25508}]" "[{""key"":""AString76783"",""value"":-1781555540},{""key"":""AString89351"",""value"":-853032410}]" "[{""key"":-5474557666971701248,""value"":-1781555540},{""key"":-3812638179211989504,""value"":-853032410}]" "[{""key"":""AString76783"",""value"":-5474557666971701248},{""key"":""AString89351"",""value"":-3812638179211989504}]" "[{""key"":-5474557666971701248,""value"":-5474557666971701248},{""key"":-3812638179211989504,""value"":-3812638179211989504}]" "[{""key"":""AString76783"",""value"":-123},{""key"":""AString89351"",""value"":-60}]" "[{""key"":-5474557666971701248,""value"":-123},{""key"":-3812638179211989504,""value"":-60}]" "[{""key"":""AString76783"",""value"":""AString76783""},{""key"":""AString89351"",""value"":""AString89351""}]" "[{""key"":-5474557666971701248,""value"":""AString76783""},{""key"":-3812638179211989504,""value"":""AString89351""}]" "[{""key"":""AString76783"",""value"":""aa:bb:cc:dd:ee:ff""},{""key"":""AString89351"",""value"":""aa:bb:cc:dd:ee:ff""}]" "[{""key"":-5474557666971701248,""value"":""aa:bb:cc:dd:ee:ff""},{""key"":-3812638179211989504,""value"":""aa:bb:cc:dd:ee:ff""}]" "[{""key"":""AString76783"",""value"":{""months"":6783,""days"":6783,""nanoseconds"":6783}},{""key"":""AString89351"",""value"":{""months"":9351,""days"":9351,""nanoseconds"":9351}}]" "[{""key"":-5474557666971701248,""value"":{""months"":6783,""days"":6783,""nanoseconds"":6783}},{""key"":-3812638179211989504,""value"":{""months"":9351,""days"":9351,""nanoseconds"":9351}}]" "[{""key"":""AString76783"",""value"":{""months"":1}},{""key"":""AString89351"",""value"":{""months"":1}}]" "[{""key"":-5474557666971701248,""value"":{""months"":1}},{""key"":-3812638179211989504,""value"":{""months"":1}}]" "[{""key"":""AString76783"",""value"":""AString76783""},{""key"":""AString89351"",""value"":""AString89351""}]" "[{""key"":-5474557666971701248,""value"":""AString76783""},{""key"":-3812638179211989504,""value"":""AString89351""}]" "[{""key"":""AString76783"",""value"":""00:00:00""},{""key"":""AString89351"",""value"":""00:00:00""}]" "[{""key"":-5474557666971701248,""value"":""00:00:00""},{""key"":-3812638179211989504,""value"":""00:00:00""}]" "[{""key"":""AString76783"",""value"":""00:00:00""},{""key"":""AString89351"",""value"":""00:00:00""}]" "[{""key"":-5474557666971701248,""value"":""00:00:00""},{""key"":-3812638179211989504,""value"":""00:00:00""}]" "[{""key"":""AString76783"",""value"":""00:00:00""},{""key"":""AString89351"",""value"":""00:00:00""}]" "[{""key"":-5474557666971701248,""value"":""00:00:00""},{""key"":-3812638179211989504,""value"":""00:00:00""}]" "[{""key"":""AString76783"",""value"":""00:00:00""},{""key"":""AString89351"",""value"":""00:00:00""}]" "[{""key"":-5474557666971701248,""value"":""00:00:00""},{""key"":-3812638179211989504,""value"":""00:00:00""}]" "[{""key"":""AString76783"",""value"":""2021-01-02 00:00:00""},{""key"":""AString89351"",""value"":""2021-01-02 00:00:00""}]" "[{""key"":-5474557666971701248,""value"":""2021-01-02 00:00:00""},{""key"":-3812638179211989504,""value"":""2021-01-02 00:00:00""}]" "[{""key"":""AString76783"",""value"":""2021-01-02 00:00:00""},{""key"":""AString89351"",""value"":""2021-01-02 00:00:00""}]" "[{""key"":-5474557666971701248,""value"":""2021-01-02 00:00:00""},{""key"":-3812638179211989504,""value"":""2021-01-02 00:00:00""}]" "[{""key"":""AString76783"",""value"":""2021-01-02 00:00:00""},{""key"":""AString89351"",""value"":""2021-01-02 00:00:00""}]" "[{""key"":-5474557666971701248,""value"":""2021-01-02 00:00:00""},{""key"":-3812638179211989504,""value"":""2021-01-02 00:00:00""}]" "[{""key"":""AString76783"",""value"":""2021-01-02 00:00:00""},{""key"":""AString89351"",""value"":""2021-01-02 00:00:00""}]" "[{""key"":-5474557666971701248,""value"":""2021-01-02 00:00:00""},{""key"":-3812638179211989504,""value"":""2021-01-02 00:00:00""}]" "[{""key"":""AString76783"",""value"":63063},{""key"":""AString89351"",""value"":34191}]" "[{""key"":-5474557666971701248,""value"":63063},{""key"":-3812638179211989504,""value"":34191}]" "[{""key"":""AString76783"",""value"":506910753},{""key"":""AString89351"",""value"":2112817056}]" "[{""key"":-5474557666971701248,""value"":506910753},{""key"":-3812638179211989504,""value"":2112817056}]" "[{""key"":""AString76783"",""value"":14697929703826477056},{""key"":""AString89351"",""value"":3812638179211989504}]" "[{""key"":-5474557666971701248,""value"":14697929703826477056},{""key"":-3812638179211989504,""value"":3812638179211989504}]" "[{""key"":""AString76783"",""value"":78},{""key"":""AString89351"",""value"":21}]" "[{""key"":-5474557666971701248,""value"":78},{""key"":-3812638179211989504,""value"":21}]" "[{""key"":""AString76783"",""value"":""00000000-0000-0000-0000-000000000001""},{""key"":""AString89351"",""value"":""00000000-0000-0000-0000-000000000001""}]" "[{""key"":-5474557666971701248,""value"":""00000000-0000-0000-0000-000000000001""},{""key"":-3812638179211989504,""value"":""00000000-0000-0000-0000-000000000001""}]" "[{""key"":""AString76783"",""value"":""826476783.5364813824""},{""key"":""AString89351"",""value"":""211989351.8533091328""}]" "[{""key"":-5474557666971701248,""value"":""826476783.5364813824""},{""key"":-3812638179211989504,""value"":""211989351.8533091328""}]" "{""binary"":""7049Ug=="",""boolean"":true,""date32"":""2023-07-16"",""date64"":""2023-07-16"",""daytimeinterval"":{""days"":6783,""milliseconds"":6783},""decimal"":""826476783.5364813824"",""duration_ms"":""6476783ms"",""duration_ns"":""6476783ns"",""duration_s"":""6476783s"",""duration_us"":""6476783us"",""float32"":783.936,""float64"":783.936,""inet"":""192.0.2.0/24"",""int16"":-30727,""int32"":-1781555540,""int64"":-5474557666971701248,""int8"":-123,""json"":{""test"":[""a"",""b"",3]},""json_array"":[{""test"":""test""},123,{""test_number"":456}],""largebinary"":""7049Ug=="",""largestring"":""AString76783"",""mac"":""aa:bb:cc:dd:ee:ff"",""monthdaynanointerval"":{""months"":6783,""days"":6783,""nanoseconds"":6783},""monthinterval"":{""months"":1},""string"":""AString76783"",""time32ms"":""00:00:00"",""time32s"":""00:00:00"",""time64ns"":""00:00:00"",""time64us"":""00:00:00"",""timestamp_ms"":""2021-01-02 00:00:00"",""timestamp_ns"":""2021-01-02 00:00:00"",""timestamp_s"":""2021-01-02 00:00:00"",""timestamp_us"":""2021-01-02 00:00:00"",""uint16"":63063,""uint32"":506910753,""uint64"":14697929703826477056,""uint8"":78,""uuid"":""00000000-0000-0000-0000-000000000001""}" "{""inner"":{""binary"":""7049Ug=="",""boolean"":true,""date32"":""2023-07-16"",""date64"":""2023-07-16"",""daytimeinterval"":{""days"":6783,""milliseconds"":6783},""decimal"":""826476783.5364813824"",""duration_ms"":""6476783ms"",""duration_ns"":""6476783ns"",""duration_s"":""6476783s"",""duration_us"":""6476783us"",""float32"":783.936,""float64"":783.936,""inet"":""192.0.2.0/24"",""int16"":-30727,""int32"":-1781555540,""int64"":-5474557666971701248,""int8"":-123,""json"":{""test"":[""a"",""b"",3]},""json_array"":[{""test"":""test""},123,{""test_number"":456}],""largebinary"":""7049Ug=="",""largestring"":""AString76783"",""mac"":""aa:bb:cc:dd:ee:ff"",""monthdaynanointerval"":{""months"":6783,""days"":6783,""nanoseconds"":6783},""monthinterval"":{""months"":1},""string"":""AString76783"",""time32ms"":""00:00:00"",""time32s"":""00:00:00"",""time64ns"":""00:00:00"",""time64us"":""00:00:00"",""timestamp_ms"":""2021-01-02 00:00:00"",""timestamp_ns"":""2021-01-02 00:00:00"",""timestamp_s"":""2021-01-02 00:00:00"",""timestamp_us"":""2021-01-02 00:00:00"",""uint16"":63063,""uint32"":506910753,""uint64"":14697929703826477056,""uint8"":78,""uuid"":""00000000-0000-0000-0000-000000000001""}}" +test-source 2021-01-01 00:00:00 00000000-0000-0000-0000-000000000001 diff --git a/csv/testdata/TestWriteRead-with_headers.csv b/csv/testdata/TestWriteRead-with_headers.csv index 97e430e5..10247f36 100644 --- a/csv/testdata/TestWriteRead-with_headers.csv +++ b/csv/testdata/TestWriteRead-with_headers.csv @@ -1,4 +1,4 @@ -_cq_source_name,_cq_sync_time,_cq_id,_cq_parent_id,binary,boolean,date32,date64,daytimeinterval,duration_ms,duration_ns,duration_s,duration_us,float32,float64,inet,int16,int32,int64,int8,largebinary,largestring,mac,monthdaynanointerval,monthinterval,string,time32ms,time32s,time64ns,time64us,timestamp_ms,timestamp_ns,timestamp_s,timestamp_us,uint16,uint32,uint64,uint8,uuid,json,json_array,boolean_list,date32_list,date64_list,daytimeinterval_list,duration_ms_list,duration_ns_list,duration_s_list,duration_us_list,float32_list,float64_list,inet_list,int16_list,int32_list,int64_list,int8_list,largestring_list,mac_list,monthdaynanointerval_list,monthinterval_list,string_list,time32ms_list,time32s_list,time64ns_list,time64us_list,timestamp_ms_list,timestamp_ns_list,timestamp_s_list,timestamp_us_list,uint16_list,uint32_list,uint64_list,uint8_list,uuid_list,struct,nested_struct -test-source,2021-01-01 00:00:00,00000000-0000-0000-0000-000000000001,00000000-0000-0000-0000-000000000001,AQIDBA==,true,2023-04-24,2023-04-24,"{""days"":1,""milliseconds"":1}",123456789ms,123456789ns,123456789s,123456789us,1.1,1.1,192.0.2.0/24,-1,-1,-1,-1,AQIDBA==,AString,aa:bb:cc:dd:ee:ff,"{""months"":1,""days"":1,""nanoseconds"":1}",1,AString,00:00:00.000,00:00:00,00:00:00.000000000,00:00:00.000000,2021-01-02 00:00:00,2021-01-02 00:00:00,2021-01-02 00:00:00,2021-01-02 00:00:00,1,1,1,1,00000000-0000-0000-0000-000000000001,"{""test"":[""a"",""b"",3]}","[{""test"":""test""},123,{""test_number"":456}]","[true,null,true]","[""2023-04-24"",null,""2023-04-24""]","[""2023-04-24"",null,""2023-04-24""]","[{""days"":1,""milliseconds"":1},null,{""days"":1,""milliseconds"":1}]","[""123456789ms"",null,""123456789ms""]","[""123456789ns"",null,""123456789ns""]","[""123456789s"",null,""123456789s""]","[""123456789us"",null,""123456789us""]","[1.1,null,1.1]","[1.1,null,1.1]","[""192.0.2.0/24"",null,""192.0.2.0/24""]","[-1,null,-1]","[-1,null,-1]","[-1,null,-1]","[-1,null,-1]","[""AString"",null,""AString""]","[""aa:bb:cc:dd:ee:ff"",null,""aa:bb:cc:dd:ee:ff""]","[{""months"":1,""days"":1,""nanoseconds"":1},null,{""months"":1,""days"":1,""nanoseconds"":1}]","[{""months"":1},null,{""months"":1}]","[""AString"",null,""AString""]","[""00:00:00"",null,""00:00:00""]","[""00:00:00"",null,""00:00:00""]","[""00:00:00"",null,""00:00:00""]","[""00:00:00"",null,""00:00:00""]","[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]","[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]","[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]","[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]","[1,null,1]","[1,null,1]","[1,null,1]","[1,null,1]","[""00000000-0000-0000-0000-000000000001"",null,""00000000-0000-0000-0000-000000000001""]","{""binary"":""AQIDBA=="",""boolean"":true,""date32"":""2023-04-24"",""date64"":""2023-04-24"",""daytimeinterval"":{""days"":1,""milliseconds"":1},""duration_ms"":""123456789ms"",""duration_ns"":""123456789ns"",""duration_s"":""123456789s"",""duration_us"":""123456789us"",""float32"":1.1,""float64"":1.1,""inet"":""192.0.2.0/24"",""int16"":-1,""int32"":-1,""int64"":-1,""int8"":-1,""json"":{""test"":[""a"",""b"",3]},""json_array"":[{""test"":""test""},123,{""test_number"":456}],""largebinary"":""AQIDBA=="",""largestring"":""AString"",""mac"":""aa:bb:cc:dd:ee:ff"",""monthdaynanointerval"":{""months"":1,""days"":1,""nanoseconds"":1},""monthinterval"":{""months"":1},""string"":""AString"",""time32ms"":""00:00:00"",""time32s"":""00:00:00"",""time64ns"":""00:00:00"",""time64us"":""00:00:00"",""timestamp_ms"":""2021-01-02 00:00:00"",""timestamp_ns"":""2021-01-02 00:00:00"",""timestamp_s"":""2021-01-02 00:00:00"",""timestamp_us"":""2021-01-02 00:00:00"",""uint16"":1,""uint32"":1,""uint64"":1,""uint8"":1,""uuid"":""00000000-0000-0000-0000-000000000001""}","{""inner"":{""binary"":""AQIDBA=="",""boolean"":true,""date32"":""2023-04-24"",""date64"":""2023-04-24"",""daytimeinterval"":{""days"":1,""milliseconds"":1},""duration_ms"":""123456789ms"",""duration_ns"":""123456789ns"",""duration_s"":""123456789s"",""duration_us"":""123456789us"",""float32"":1.1,""float64"":1.1,""inet"":""192.0.2.0/24"",""int16"":-1,""int32"":-1,""int64"":-1,""int8"":-1,""json"":{""test"":[""a"",""b"",3]},""json_array"":[{""test"":""test""},123,{""test_number"":456}],""largebinary"":""AQIDBA=="",""largestring"":""AString"",""mac"":""aa:bb:cc:dd:ee:ff"",""monthdaynanointerval"":{""months"":1,""days"":1,""nanoseconds"":1},""monthinterval"":{""months"":1},""string"":""AString"",""time32ms"":""00:00:00"",""time32s"":""00:00:00"",""time64ns"":""00:00:00"",""time64us"":""00:00:00"",""timestamp_ms"":""2021-01-02 00:00:00"",""timestamp_ns"":""2021-01-02 00:00:00"",""timestamp_s"":""2021-01-02 00:00:00"",""timestamp_us"":""2021-01-02 00:00:00"",""uint16"":1,""uint32"":1,""uint64"":1,""uint8"":1,""uuid"":""00000000-0000-0000-0000-000000000001""}}" -test-source,2021-01-01 00:00:00,00000000-0000-0000-0000-000000000001,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +_cq_source_name,_cq_sync_time,_cq_id,_cq_parent_id,binary,boolean,date32,date64,daytimeinterval,duration_ms,duration_ns,duration_s,duration_us,float32,float64,inet,int16,int32,int64,int8,largebinary,largestring,mac,monthdaynanointerval,monthinterval,string,time32ms,time32s,time64ns,time64us,timestamp_ms,timestamp_ns,timestamp_s,timestamp_us,uint16,uint32,uint64,uint8,uuid,decimal,json,json_array,boolean_list,date32_list,date64_list,daytimeinterval_list,duration_ms_list,duration_ns_list,duration_s_list,duration_us_list,float32_list,float64_list,inet_list,int16_list,int32_list,int64_list,int8_list,largestring_list,mac_list,monthdaynanointerval_list,monthinterval_list,string_list,time32ms_list,time32s_list,time64ns_list,time64us_list,timestamp_ms_list,timestamp_ns_list,timestamp_s_list,timestamp_us_list,uint16_list,uint32_list,uint64_list,uint8_list,uuid_list,decimal_list,string_boolean_map,int_boolean_map,string_date32_map,int_date32_map,string_date64_map,int_date64_map,string_daytimeinterval_map,int_daytimeinterval_map,string_duration_ms_map,int_duration_ms_map,string_duration_ns_map,int_duration_ns_map,string_duration_s_map,int_duration_s_map,string_duration_us_map,int_duration_us_map,string_float32_map,int_float32_map,string_float64_map,int_float64_map,string_inet_map,int_inet_map,string_int16_map,int_int16_map,string_int32_map,int_int32_map,string_int64_map,int_int64_map,string_int8_map,int_int8_map,string_largestring_map,int_largestring_map,string_mac_map,int_mac_map,string_monthdaynanointerval_map,int_monthdaynanointerval_map,string_monthinterval_map,int_monthinterval_map,string_string_map,int_string_map,string_time32ms_map,int_time32ms_map,string_time32s_map,int_time32s_map,string_time64ns_map,int_time64ns_map,string_time64us_map,int_time64us_map,string_timestamp_ms_map,int_timestamp_ms_map,string_timestamp_ns_map,int_timestamp_ns_map,string_timestamp_s_map,int_timestamp_s_map,string_timestamp_us_map,int_timestamp_us_map,string_uint16_map,int_uint16_map,string_uint32_map,int_uint32_map,string_uint64_map,int_uint64_map,string_uint8_map,int_uint8_map,string_uuid_map,int_uuid_map,string_decimal_map,int_decimal_map,struct,nested_struct +test-source,2021-01-01 00:00:00,00000000-0000-0000-0000-000000000001,00000000-0000-0000-0000-000000000001,7049Ug==,true,2023-07-16,2023-07-16,"{""days"":6783,""milliseconds"":6783}",6476783ms,6476783ns,6476783s,6476783us,783.936,783.936,192.0.2.0/24,-30727,-1781555540,-5474557666971701248,-123,7049Ug==,AString76783,aa:bb:cc:dd:ee:ff,"{""months"":6783,""days"":6783,""nanoseconds"":6783}",1,AString76783,00:00:00.000,00:00:00,00:00:00.000000000,00:00:00.000000,2021-01-02 00:00:00,2021-01-02 00:00:00,2021-01-02 00:00:00,2021-01-02 00:00:00,63063,506910753,14697929703826477056,78,00000000-0000-0000-0000-000000000001,826476783.5364813824,"{""test"":[""a"",""b"",3]}","[{""test"":""test""},123,{""test_number"":456}]","[true,null,true]","[""2023-07-16"",null,""2023-07-16""]","[""2023-07-16"",null,""2023-07-16""]","[{""days"":6783,""milliseconds"":6783},null,{""days"":6783,""milliseconds"":6783}]","[""6476783ms"",null,""6476783ms""]","[""6476783ns"",null,""6476783ns""]","[""6476783s"",null,""6476783s""]","[""6476783us"",null,""6476783us""]","[783.936,null,783.936]","[783.936,null,783.936]","[""192.0.2.0/24"",null,""192.0.2.0/24""]","[-30727,null,-30727]","[-1781555540,null,-1781555540]","[-5474557666971701248,null,-5474557666971701248]","[-123,null,-123]","[""AString76783"",null,""AString76783""]","[""aa:bb:cc:dd:ee:ff"",null,""aa:bb:cc:dd:ee:ff""]","[{""months"":6783,""days"":6783,""nanoseconds"":6783},null,{""months"":6783,""days"":6783,""nanoseconds"":6783}]","[{""months"":1},null,{""months"":1}]","[""AString76783"",null,""AString76783""]","[""00:00:00"",null,""00:00:00""]","[""00:00:00"",null,""00:00:00""]","[""00:00:00"",null,""00:00:00""]","[""00:00:00"",null,""00:00:00""]","[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]","[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]","[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]","[""2021-01-02 00:00:00"",null,""2021-01-02 00:00:00""]","[63063,null,63063]","[506910753,null,506910753]","[14697929703826477056,null,14697929703826477056]","[78,null,78]","[""00000000-0000-0000-0000-000000000001"",null,""00000000-0000-0000-0000-000000000001""]","[""826476783.5364813824"",null,""826476783.5364813824""]","[{""key"":""AString76783"",""value"":true},{""key"":""AString89351"",""value"":true}]","[{""key"":-5474557666971701248,""value"":true},{""key"":-3812638179211989504,""value"":true}]","[{""key"":""AString76783"",""value"":""2023-07-16""},{""key"":""AString89351"",""value"":""2023-06-14""}]","[{""key"":-5474557666971701248,""value"":""2023-07-16""},{""key"":-3812638179211989504,""value"":""2023-06-14""}]","[{""key"":""AString76783"",""value"":""2023-07-16""},{""key"":""AString89351"",""value"":""2023-06-14""}]","[{""key"":-5474557666971701248,""value"":""2023-07-16""},{""key"":-3812638179211989504,""value"":""2023-06-14""}]","[{""key"":""AString76783"",""value"":{""days"":6783,""milliseconds"":6783}},{""key"":""AString89351"",""value"":{""days"":9351,""milliseconds"":9351}}]","[{""key"":-5474557666971701248,""value"":{""days"":6783,""milliseconds"":6783}},{""key"":-3812638179211989504,""value"":{""days"":9351,""milliseconds"":9351}}]","[{""key"":""AString76783"",""value"":""6476783ms""},{""key"":""AString89351"",""value"":""1989351ms""}]","[{""key"":-5474557666971701248,""value"":""6476783ms""},{""key"":-3812638179211989504,""value"":""1989351ms""}]","[{""key"":""AString76783"",""value"":""6476783ns""},{""key"":""AString89351"",""value"":""1989351ns""}]","[{""key"":-5474557666971701248,""value"":""6476783ns""},{""key"":-3812638179211989504,""value"":""1989351ns""}]","[{""key"":""AString76783"",""value"":""6476783s""},{""key"":""AString89351"",""value"":""1989351s""}]","[{""key"":-5474557666971701248,""value"":""6476783s""},{""key"":-3812638179211989504,""value"":""1989351s""}]","[{""key"":""AString76783"",""value"":""6476783us""},{""key"":""AString89351"",""value"":""1989351us""}]","[{""key"":-5474557666971701248,""value"":""6476783us""},{""key"":-3812638179211989504,""value"":""1989351us""}]","[{""key"":""AString76783"",""value"":783.936},{""key"":""AString89351"",""value"":351.121}]","[{""key"":-5474557666971701248,""value"":783.936},{""key"":-3812638179211989504,""value"":351.121}]","[{""key"":""AString76783"",""value"":783.936},{""key"":""AString89351"",""value"":351.121}]","[{""key"":-5474557666971701248,""value"":783.936},{""key"":-3812638179211989504,""value"":351.121}]","[{""key"":""AString76783"",""value"":""192.0.2.0/24""},{""key"":""AString89351"",""value"":""192.0.2.0/24""}]","[{""key"":-5474557666971701248,""value"":""192.0.2.0/24""},{""key"":-3812638179211989504,""value"":""192.0.2.0/24""}]","[{""key"":""AString76783"",""value"":-30727},{""key"":""AString89351"",""value"":-25508}]","[{""key"":-5474557666971701248,""value"":-30727},{""key"":-3812638179211989504,""value"":-25508}]","[{""key"":""AString76783"",""value"":-1781555540},{""key"":""AString89351"",""value"":-853032410}]","[{""key"":-5474557666971701248,""value"":-1781555540},{""key"":-3812638179211989504,""value"":-853032410}]","[{""key"":""AString76783"",""value"":-5474557666971701248},{""key"":""AString89351"",""value"":-3812638179211989504}]","[{""key"":-5474557666971701248,""value"":-5474557666971701248},{""key"":-3812638179211989504,""value"":-3812638179211989504}]","[{""key"":""AString76783"",""value"":-123},{""key"":""AString89351"",""value"":-60}]","[{""key"":-5474557666971701248,""value"":-123},{""key"":-3812638179211989504,""value"":-60}]","[{""key"":""AString76783"",""value"":""AString76783""},{""key"":""AString89351"",""value"":""AString89351""}]","[{""key"":-5474557666971701248,""value"":""AString76783""},{""key"":-3812638179211989504,""value"":""AString89351""}]","[{""key"":""AString76783"",""value"":""aa:bb:cc:dd:ee:ff""},{""key"":""AString89351"",""value"":""aa:bb:cc:dd:ee:ff""}]","[{""key"":-5474557666971701248,""value"":""aa:bb:cc:dd:ee:ff""},{""key"":-3812638179211989504,""value"":""aa:bb:cc:dd:ee:ff""}]","[{""key"":""AString76783"",""value"":{""months"":6783,""days"":6783,""nanoseconds"":6783}},{""key"":""AString89351"",""value"":{""months"":9351,""days"":9351,""nanoseconds"":9351}}]","[{""key"":-5474557666971701248,""value"":{""months"":6783,""days"":6783,""nanoseconds"":6783}},{""key"":-3812638179211989504,""value"":{""months"":9351,""days"":9351,""nanoseconds"":9351}}]","[{""key"":""AString76783"",""value"":{""months"":1}},{""key"":""AString89351"",""value"":{""months"":1}}]","[{""key"":-5474557666971701248,""value"":{""months"":1}},{""key"":-3812638179211989504,""value"":{""months"":1}}]","[{""key"":""AString76783"",""value"":""AString76783""},{""key"":""AString89351"",""value"":""AString89351""}]","[{""key"":-5474557666971701248,""value"":""AString76783""},{""key"":-3812638179211989504,""value"":""AString89351""}]","[{""key"":""AString76783"",""value"":""00:00:00""},{""key"":""AString89351"",""value"":""00:00:00""}]","[{""key"":-5474557666971701248,""value"":""00:00:00""},{""key"":-3812638179211989504,""value"":""00:00:00""}]","[{""key"":""AString76783"",""value"":""00:00:00""},{""key"":""AString89351"",""value"":""00:00:00""}]","[{""key"":-5474557666971701248,""value"":""00:00:00""},{""key"":-3812638179211989504,""value"":""00:00:00""}]","[{""key"":""AString76783"",""value"":""00:00:00""},{""key"":""AString89351"",""value"":""00:00:00""}]","[{""key"":-5474557666971701248,""value"":""00:00:00""},{""key"":-3812638179211989504,""value"":""00:00:00""}]","[{""key"":""AString76783"",""value"":""00:00:00""},{""key"":""AString89351"",""value"":""00:00:00""}]","[{""key"":-5474557666971701248,""value"":""00:00:00""},{""key"":-3812638179211989504,""value"":""00:00:00""}]","[{""key"":""AString76783"",""value"":""2021-01-02 00:00:00""},{""key"":""AString89351"",""value"":""2021-01-02 00:00:00""}]","[{""key"":-5474557666971701248,""value"":""2021-01-02 00:00:00""},{""key"":-3812638179211989504,""value"":""2021-01-02 00:00:00""}]","[{""key"":""AString76783"",""value"":""2021-01-02 00:00:00""},{""key"":""AString89351"",""value"":""2021-01-02 00:00:00""}]","[{""key"":-5474557666971701248,""value"":""2021-01-02 00:00:00""},{""key"":-3812638179211989504,""value"":""2021-01-02 00:00:00""}]","[{""key"":""AString76783"",""value"":""2021-01-02 00:00:00""},{""key"":""AString89351"",""value"":""2021-01-02 00:00:00""}]","[{""key"":-5474557666971701248,""value"":""2021-01-02 00:00:00""},{""key"":-3812638179211989504,""value"":""2021-01-02 00:00:00""}]","[{""key"":""AString76783"",""value"":""2021-01-02 00:00:00""},{""key"":""AString89351"",""value"":""2021-01-02 00:00:00""}]","[{""key"":-5474557666971701248,""value"":""2021-01-02 00:00:00""},{""key"":-3812638179211989504,""value"":""2021-01-02 00:00:00""}]","[{""key"":""AString76783"",""value"":63063},{""key"":""AString89351"",""value"":34191}]","[{""key"":-5474557666971701248,""value"":63063},{""key"":-3812638179211989504,""value"":34191}]","[{""key"":""AString76783"",""value"":506910753},{""key"":""AString89351"",""value"":2112817056}]","[{""key"":-5474557666971701248,""value"":506910753},{""key"":-3812638179211989504,""value"":2112817056}]","[{""key"":""AString76783"",""value"":14697929703826477056},{""key"":""AString89351"",""value"":3812638179211989504}]","[{""key"":-5474557666971701248,""value"":14697929703826477056},{""key"":-3812638179211989504,""value"":3812638179211989504}]","[{""key"":""AString76783"",""value"":78},{""key"":""AString89351"",""value"":21}]","[{""key"":-5474557666971701248,""value"":78},{""key"":-3812638179211989504,""value"":21}]","[{""key"":""AString76783"",""value"":""00000000-0000-0000-0000-000000000001""},{""key"":""AString89351"",""value"":""00000000-0000-0000-0000-000000000001""}]","[{""key"":-5474557666971701248,""value"":""00000000-0000-0000-0000-000000000001""},{""key"":-3812638179211989504,""value"":""00000000-0000-0000-0000-000000000001""}]","[{""key"":""AString76783"",""value"":""826476783.5364813824""},{""key"":""AString89351"",""value"":""211989351.8533091328""}]","[{""key"":-5474557666971701248,""value"":""826476783.5364813824""},{""key"":-3812638179211989504,""value"":""211989351.8533091328""}]","{""binary"":""7049Ug=="",""boolean"":true,""date32"":""2023-07-16"",""date64"":""2023-07-16"",""daytimeinterval"":{""days"":6783,""milliseconds"":6783},""decimal"":""826476783.5364813824"",""duration_ms"":""6476783ms"",""duration_ns"":""6476783ns"",""duration_s"":""6476783s"",""duration_us"":""6476783us"",""float32"":783.936,""float64"":783.936,""inet"":""192.0.2.0/24"",""int16"":-30727,""int32"":-1781555540,""int64"":-5474557666971701248,""int8"":-123,""json"":{""test"":[""a"",""b"",3]},""json_array"":[{""test"":""test""},123,{""test_number"":456}],""largebinary"":""7049Ug=="",""largestring"":""AString76783"",""mac"":""aa:bb:cc:dd:ee:ff"",""monthdaynanointerval"":{""months"":6783,""days"":6783,""nanoseconds"":6783},""monthinterval"":{""months"":1},""string"":""AString76783"",""time32ms"":""00:00:00"",""time32s"":""00:00:00"",""time64ns"":""00:00:00"",""time64us"":""00:00:00"",""timestamp_ms"":""2021-01-02 00:00:00"",""timestamp_ns"":""2021-01-02 00:00:00"",""timestamp_s"":""2021-01-02 00:00:00"",""timestamp_us"":""2021-01-02 00:00:00"",""uint16"":63063,""uint32"":506910753,""uint64"":14697929703826477056,""uint8"":78,""uuid"":""00000000-0000-0000-0000-000000000001""}","{""inner"":{""binary"":""7049Ug=="",""boolean"":true,""date32"":""2023-07-16"",""date64"":""2023-07-16"",""daytimeinterval"":{""days"":6783,""milliseconds"":6783},""decimal"":""826476783.5364813824"",""duration_ms"":""6476783ms"",""duration_ns"":""6476783ns"",""duration_s"":""6476783s"",""duration_us"":""6476783us"",""float32"":783.936,""float64"":783.936,""inet"":""192.0.2.0/24"",""int16"":-30727,""int32"":-1781555540,""int64"":-5474557666971701248,""int8"":-123,""json"":{""test"":[""a"",""b"",3]},""json_array"":[{""test"":""test""},123,{""test_number"":456}],""largebinary"":""7049Ug=="",""largestring"":""AString76783"",""mac"":""aa:bb:cc:dd:ee:ff"",""monthdaynanointerval"":{""months"":6783,""days"":6783,""nanoseconds"":6783},""monthinterval"":{""months"":1},""string"":""AString76783"",""time32ms"":""00:00:00"",""time32s"":""00:00:00"",""time64ns"":""00:00:00"",""time64us"":""00:00:00"",""timestamp_ms"":""2021-01-02 00:00:00"",""timestamp_ns"":""2021-01-02 00:00:00"",""timestamp_s"":""2021-01-02 00:00:00"",""timestamp_us"":""2021-01-02 00:00:00"",""uint16"":63063,""uint32"":506910753,""uint64"":14697929703826477056,""uint8"":78,""uuid"":""00000000-0000-0000-0000-000000000001""}}" +test-source,2021-01-01 00:00:00,00000000-0000-0000-0000-000000000001,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, diff --git a/json/testdata/TestWriteRead.jsonl b/json/testdata/TestWriteRead.jsonl index aaf04aaf..566c6f41 100644 --- a/json/testdata/TestWriteRead.jsonl +++ b/json/testdata/TestWriteRead.jsonl @@ -1,3 +1,3 @@ -{"_cq_id":"00000000-0000-0000-0000-000000000001","_cq_parent_id":"00000000-0000-0000-0000-000000000001","_cq_source_name":"test-source","_cq_sync_time":"2021-01-01 00:00:00","binary":"AQIDBA==","boolean":true,"boolean_list":[true,null,true],"date32":"2023-04-24","date32_list":["2023-04-24",null,"2023-04-24"],"date64":"2023-04-24","date64_list":["2023-04-24",null,"2023-04-24"],"daytimeinterval":{"days":1,"milliseconds":1},"daytimeinterval_list":[{"days":1,"milliseconds":1},null,{"days":1,"milliseconds":1}],"duration_ms":"123456789ms","duration_ms_list":["123456789ms",null,"123456789ms"],"duration_ns":"123456789ns","duration_ns_list":["123456789ns",null,"123456789ns"],"duration_s":"123456789s","duration_s_list":["123456789s",null,"123456789s"],"duration_us":"123456789us","duration_us_list":["123456789us",null,"123456789us"],"float32":1.1,"float32_list":[1.1,null,1.1],"float64":1.1,"float64_list":[1.1,null,1.1],"inet":"192.0.2.0/24","inet_list":["192.0.2.0/24",null,"192.0.2.0/24"],"int16":-1,"int16_list":[-1,null,-1],"int32":-1,"int32_list":[-1,null,-1],"int64":-1,"int64_list":[-1,null,-1],"int8":-1,"int8_list":[-1,null,-1],"json":{"test":["a","b",3]},"json_array":[{"test":"test"},123,{"test_number":456}],"largebinary":"AQIDBA==","largestring":"AString","largestring_list":["AString",null,"AString"],"mac":"aa:bb:cc:dd:ee:ff","mac_list":["aa:bb:cc:dd:ee:ff",null,"aa:bb:cc:dd:ee:ff"],"monthdaynanointerval":{"months":1,"days":1,"nanoseconds":1},"monthdaynanointerval_list":[{"months":1,"days":1,"nanoseconds":1},null,{"months":1,"days":1,"nanoseconds":1}],"monthinterval":{"months":1},"monthinterval_list":[{"months":1},null,{"months":1}],"nested_struct":{"inner":{"binary":"AQIDBA==","boolean":true,"date32":"2023-04-24","date64":"2023-04-24","daytimeinterval":{"days":1,"milliseconds":1},"duration_ms":"123456789ms","duration_ns":"123456789ns","duration_s":"123456789s","duration_us":"123456789us","float32":1.1,"float64":1.1,"inet":"192.0.2.0/24","int16":-1,"int32":-1,"int64":-1,"int8":-1,"json":{"test":["a","b",3]},"json_array":[{"test":"test"},123,{"test_number":456}],"largebinary":"AQIDBA==","largestring":"AString","mac":"aa:bb:cc:dd:ee:ff","monthdaynanointerval":{"months":1,"days":1,"nanoseconds":1},"monthinterval":{"months":1},"string":"AString","time32ms":"00:00:00","time32s":"00:00:00","time64ns":"00:00:00","time64us":"00:00:00","timestamp_ms":"2021-01-02 00:00:00","timestamp_ns":"2021-01-02 00:00:00","timestamp_s":"2021-01-02 00:00:00","timestamp_us":"2021-01-02 00:00:00","uint16":1,"uint32":1,"uint64":1,"uint8":1,"uuid":"00000000-0000-0000-0000-000000000001"}},"string":"AString","string_list":["AString",null,"AString"],"struct":{"binary":"AQIDBA==","boolean":true,"date32":"2023-04-24","date64":"2023-04-24","daytimeinterval":{"days":1,"milliseconds":1},"duration_ms":"123456789ms","duration_ns":"123456789ns","duration_s":"123456789s","duration_us":"123456789us","float32":1.1,"float64":1.1,"inet":"192.0.2.0/24","int16":-1,"int32":-1,"int64":-1,"int8":-1,"json":{"test":["a","b",3]},"json_array":[{"test":"test"},123,{"test_number":456}],"largebinary":"AQIDBA==","largestring":"AString","mac":"aa:bb:cc:dd:ee:ff","monthdaynanointerval":{"months":1,"days":1,"nanoseconds":1},"monthinterval":{"months":1},"string":"AString","time32ms":"00:00:00","time32s":"00:00:00","time64ns":"00:00:00","time64us":"00:00:00","timestamp_ms":"2021-01-02 00:00:00","timestamp_ns":"2021-01-02 00:00:00","timestamp_s":"2021-01-02 00:00:00","timestamp_us":"2021-01-02 00:00:00","uint16":1,"uint32":1,"uint64":1,"uint8":1,"uuid":"00000000-0000-0000-0000-000000000001"},"time32ms":"00:00:00","time32ms_list":["00:00:00",null,"00:00:00"],"time32s":"00:00:00","time32s_list":["00:00:00",null,"00:00:00"],"time64ns":"00:00:00","time64ns_list":["00:00:00",null,"00:00:00"],"time64us":"00:00:00","time64us_list":["00:00:00",null,"00:00:00"],"timestamp_ms":"2021-01-02 00:00:00","timestamp_ms_list":["2021-01-02 00:00:00",null,"2021-01-02 00:00:00"],"timestamp_ns":"2021-01-02 00:00:00","timestamp_ns_list":["2021-01-02 00:00:00",null,"2021-01-02 00:00:00"],"timestamp_s":"2021-01-02 00:00:00","timestamp_s_list":["2021-01-02 00:00:00",null,"2021-01-02 00:00:00"],"timestamp_us":"2021-01-02 00:00:00","timestamp_us_list":["2021-01-02 00:00:00",null,"2021-01-02 00:00:00"],"uint16":1,"uint16_list":[1,null,1],"uint32":1,"uint32_list":[1,null,1],"uint64":1,"uint64_list":[1,null,1],"uint8":1,"uint8_list":[1,null,1],"uuid":"00000000-0000-0000-0000-000000000001","uuid_list":["00000000-0000-0000-0000-000000000001",null,"00000000-0000-0000-0000-000000000001"]} -{"_cq_id":"00000000-0000-0000-0000-000000000001","_cq_parent_id":null,"_cq_source_name":"test-source","_cq_sync_time":"2021-01-01 00:00:00","binary":null,"boolean":null,"boolean_list":null,"date32":null,"date32_list":null,"date64":null,"date64_list":null,"daytimeinterval":null,"daytimeinterval_list":null,"duration_ms":null,"duration_ms_list":null,"duration_ns":null,"duration_ns_list":null,"duration_s":null,"duration_s_list":null,"duration_us":null,"duration_us_list":null,"float32":null,"float32_list":null,"float64":null,"float64_list":null,"inet":null,"inet_list":null,"int16":null,"int16_list":null,"int32":null,"int32_list":null,"int64":null,"int64_list":null,"int8":null,"int8_list":null,"json":null,"json_array":null,"largebinary":null,"largestring":null,"largestring_list":null,"mac":null,"mac_list":null,"monthdaynanointerval":null,"monthdaynanointerval_list":null,"monthinterval":null,"monthinterval_list":null,"nested_struct":null,"string":null,"string_list":null,"struct":null,"time32ms":null,"time32ms_list":null,"time32s":null,"time32s_list":null,"time64ns":null,"time64ns_list":null,"time64us":null,"time64us_list":null,"timestamp_ms":null,"timestamp_ms_list":null,"timestamp_ns":null,"timestamp_ns_list":null,"timestamp_s":null,"timestamp_s_list":null,"timestamp_us":null,"timestamp_us_list":null,"uint16":null,"uint16_list":null,"uint32":null,"uint32_list":null,"uint64":null,"uint64_list":null,"uint8":null,"uint8_list":null,"uuid":null,"uuid_list":null} +{"_cq_id":"00000000-0000-0000-0000-000000000001","_cq_parent_id":"00000000-0000-0000-0000-000000000001","_cq_source_name":"test-source","_cq_sync_time":"2021-01-01 00:00:00","binary":"7049Ug==","boolean":true,"boolean_list":[true,null,true],"date32":"2023-07-16","date32_list":["2023-07-16",null,"2023-07-16"],"date64":"2023-07-16","date64_list":["2023-07-16",null,"2023-07-16"],"daytimeinterval":{"days":6783,"milliseconds":6783},"daytimeinterval_list":[{"days":6783,"milliseconds":6783},null,{"days":6783,"milliseconds":6783}],"decimal":"826476783.5364813824","decimal_list":["826476783.5364813824",null,"826476783.5364813824"],"duration_ms":"6476783ms","duration_ms_list":["6476783ms",null,"6476783ms"],"duration_ns":"6476783ns","duration_ns_list":["6476783ns",null,"6476783ns"],"duration_s":"6476783s","duration_s_list":["6476783s",null,"6476783s"],"duration_us":"6476783us","duration_us_list":["6476783us",null,"6476783us"],"float32":783.936,"float32_list":[783.936,null,783.936],"float64":783.936,"float64_list":[783.936,null,783.936],"inet":"192.0.2.0/24","inet_list":["192.0.2.0/24",null,"192.0.2.0/24"],"int16":-30727,"int16_list":[-30727,null,-30727],"int32":-1781555540,"int32_list":[-1781555540,null,-1781555540],"int64":-5474557666971701248,"int64_list":[-5474557666971701248,null,-5474557666971701248],"int8":-123,"int8_list":[-123,null,-123],"int_boolean_map":[{"key":-5474557666971701248,"value":true},{"key":-3812638179211989504,"value":true}],"int_date32_map":[{"key":-5474557666971701248,"value":"2023-07-16"},{"key":-3812638179211989504,"value":"2023-06-14"}],"int_date64_map":[{"key":-5474557666971701248,"value":"2023-07-16"},{"key":-3812638179211989504,"value":"2023-06-14"}],"int_daytimeinterval_map":[{"key":-5474557666971701248,"value":{"days":6783,"milliseconds":6783}},{"key":-3812638179211989504,"value":{"days":9351,"milliseconds":9351}}],"int_decimal_map":[{"key":-5474557666971701248,"value":"826476783.5364813824"},{"key":-3812638179211989504,"value":"211989351.8533091328"}],"int_duration_ms_map":[{"key":-5474557666971701248,"value":"6476783ms"},{"key":-3812638179211989504,"value":"1989351ms"}],"int_duration_ns_map":[{"key":-5474557666971701248,"value":"6476783ns"},{"key":-3812638179211989504,"value":"1989351ns"}],"int_duration_s_map":[{"key":-5474557666971701248,"value":"6476783s"},{"key":-3812638179211989504,"value":"1989351s"}],"int_duration_us_map":[{"key":-5474557666971701248,"value":"6476783us"},{"key":-3812638179211989504,"value":"1989351us"}],"int_float32_map":[{"key":-5474557666971701248,"value":783.936},{"key":-3812638179211989504,"value":351.121}],"int_float64_map":[{"key":-5474557666971701248,"value":783.936},{"key":-3812638179211989504,"value":351.121}],"int_inet_map":[{"key":-5474557666971701248,"value":"192.0.2.0/24"},{"key":-3812638179211989504,"value":"192.0.2.0/24"}],"int_int16_map":[{"key":-5474557666971701248,"value":-30727},{"key":-3812638179211989504,"value":-25508}],"int_int32_map":[{"key":-5474557666971701248,"value":-1781555540},{"key":-3812638179211989504,"value":-853032410}],"int_int64_map":[{"key":-5474557666971701248,"value":-5474557666971701248},{"key":-3812638179211989504,"value":-3812638179211989504}],"int_int8_map":[{"key":-5474557666971701248,"value":-123},{"key":-3812638179211989504,"value":-60}],"int_largestring_map":[{"key":-5474557666971701248,"value":"AString76783"},{"key":-3812638179211989504,"value":"AString89351"}],"int_mac_map":[{"key":-5474557666971701248,"value":"aa:bb:cc:dd:ee:ff"},{"key":-3812638179211989504,"value":"aa:bb:cc:dd:ee:ff"}],"int_monthdaynanointerval_map":[{"key":-5474557666971701248,"value":{"months":6783,"days":6783,"nanoseconds":6783}},{"key":-3812638179211989504,"value":{"months":9351,"days":9351,"nanoseconds":9351}}],"int_monthinterval_map":[{"key":-5474557666971701248,"value":{"months":1}},{"key":-3812638179211989504,"value":{"months":1}}],"int_string_map":[{"key":-5474557666971701248,"value":"AString76783"},{"key":-3812638179211989504,"value":"AString89351"}],"int_time32ms_map":[{"key":-5474557666971701248,"value":"00:00:00"},{"key":-3812638179211989504,"value":"00:00:00"}],"int_time32s_map":[{"key":-5474557666971701248,"value":"00:00:00"},{"key":-3812638179211989504,"value":"00:00:00"}],"int_time64ns_map":[{"key":-5474557666971701248,"value":"00:00:00"},{"key":-3812638179211989504,"value":"00:00:00"}],"int_time64us_map":[{"key":-5474557666971701248,"value":"00:00:00"},{"key":-3812638179211989504,"value":"00:00:00"}],"int_timestamp_ms_map":[{"key":-5474557666971701248,"value":"2021-01-02 00:00:00"},{"key":-3812638179211989504,"value":"2021-01-02 00:00:00"}],"int_timestamp_ns_map":[{"key":-5474557666971701248,"value":"2021-01-02 00:00:00"},{"key":-3812638179211989504,"value":"2021-01-02 00:00:00"}],"int_timestamp_s_map":[{"key":-5474557666971701248,"value":"2021-01-02 00:00:00"},{"key":-3812638179211989504,"value":"2021-01-02 00:00:00"}],"int_timestamp_us_map":[{"key":-5474557666971701248,"value":"2021-01-02 00:00:00"},{"key":-3812638179211989504,"value":"2021-01-02 00:00:00"}],"int_uint16_map":[{"key":-5474557666971701248,"value":63063},{"key":-3812638179211989504,"value":34191}],"int_uint32_map":[{"key":-5474557666971701248,"value":506910753},{"key":-3812638179211989504,"value":2112817056}],"int_uint64_map":[{"key":-5474557666971701248,"value":14697929703826477056},{"key":-3812638179211989504,"value":3812638179211989504}],"int_uint8_map":[{"key":-5474557666971701248,"value":78},{"key":-3812638179211989504,"value":21}],"int_uuid_map":[{"key":-5474557666971701248,"value":"00000000-0000-0000-0000-000000000001"},{"key":-3812638179211989504,"value":"00000000-0000-0000-0000-000000000001"}],"json":{"test":["a","b",3]},"json_array":[{"test":"test"},123,{"test_number":456}],"largebinary":"7049Ug==","largestring":"AString76783","largestring_list":["AString76783",null,"AString76783"],"mac":"aa:bb:cc:dd:ee:ff","mac_list":["aa:bb:cc:dd:ee:ff",null,"aa:bb:cc:dd:ee:ff"],"monthdaynanointerval":{"months":6783,"days":6783,"nanoseconds":6783},"monthdaynanointerval_list":[{"months":6783,"days":6783,"nanoseconds":6783},null,{"months":6783,"days":6783,"nanoseconds":6783}],"monthinterval":{"months":1},"monthinterval_list":[{"months":1},null,{"months":1}],"nested_struct":{"inner":{"binary":"7049Ug==","boolean":true,"date32":"2023-07-16","date64":"2023-07-16","daytimeinterval":{"days":6783,"milliseconds":6783},"decimal":"826476783.5364813824","duration_ms":"6476783ms","duration_ns":"6476783ns","duration_s":"6476783s","duration_us":"6476783us","float32":783.936,"float64":783.936,"inet":"192.0.2.0/24","int16":-30727,"int32":-1781555540,"int64":-5474557666971701248,"int8":-123,"json":{"test":["a","b",3]},"json_array":[{"test":"test"},123,{"test_number":456}],"largebinary":"7049Ug==","largestring":"AString76783","mac":"aa:bb:cc:dd:ee:ff","monthdaynanointerval":{"months":6783,"days":6783,"nanoseconds":6783},"monthinterval":{"months":1},"string":"AString76783","time32ms":"00:00:00","time32s":"00:00:00","time64ns":"00:00:00","time64us":"00:00:00","timestamp_ms":"2021-01-02 00:00:00","timestamp_ns":"2021-01-02 00:00:00","timestamp_s":"2021-01-02 00:00:00","timestamp_us":"2021-01-02 00:00:00","uint16":63063,"uint32":506910753,"uint64":14697929703826477056,"uint8":78,"uuid":"00000000-0000-0000-0000-000000000001"}},"string":"AString76783","string_boolean_map":[{"key":"AString76783","value":true},{"key":"AString89351","value":true}],"string_date32_map":[{"key":"AString76783","value":"2023-07-16"},{"key":"AString89351","value":"2023-06-14"}],"string_date64_map":[{"key":"AString76783","value":"2023-07-16"},{"key":"AString89351","value":"2023-06-14"}],"string_daytimeinterval_map":[{"key":"AString76783","value":{"days":6783,"milliseconds":6783}},{"key":"AString89351","value":{"days":9351,"milliseconds":9351}}],"string_decimal_map":[{"key":"AString76783","value":"826476783.5364813824"},{"key":"AString89351","value":"211989351.8533091328"}],"string_duration_ms_map":[{"key":"AString76783","value":"6476783ms"},{"key":"AString89351","value":"1989351ms"}],"string_duration_ns_map":[{"key":"AString76783","value":"6476783ns"},{"key":"AString89351","value":"1989351ns"}],"string_duration_s_map":[{"key":"AString76783","value":"6476783s"},{"key":"AString89351","value":"1989351s"}],"string_duration_us_map":[{"key":"AString76783","value":"6476783us"},{"key":"AString89351","value":"1989351us"}],"string_float32_map":[{"key":"AString76783","value":783.936},{"key":"AString89351","value":351.121}],"string_float64_map":[{"key":"AString76783","value":783.936},{"key":"AString89351","value":351.121}],"string_inet_map":[{"key":"AString76783","value":"192.0.2.0/24"},{"key":"AString89351","value":"192.0.2.0/24"}],"string_int16_map":[{"key":"AString76783","value":-30727},{"key":"AString89351","value":-25508}],"string_int32_map":[{"key":"AString76783","value":-1781555540},{"key":"AString89351","value":-853032410}],"string_int64_map":[{"key":"AString76783","value":-5474557666971701248},{"key":"AString89351","value":-3812638179211989504}],"string_int8_map":[{"key":"AString76783","value":-123},{"key":"AString89351","value":-60}],"string_largestring_map":[{"key":"AString76783","value":"AString76783"},{"key":"AString89351","value":"AString89351"}],"string_list":["AString76783",null,"AString76783"],"string_mac_map":[{"key":"AString76783","value":"aa:bb:cc:dd:ee:ff"},{"key":"AString89351","value":"aa:bb:cc:dd:ee:ff"}],"string_monthdaynanointerval_map":[{"key":"AString76783","value":{"months":6783,"days":6783,"nanoseconds":6783}},{"key":"AString89351","value":{"months":9351,"days":9351,"nanoseconds":9351}}],"string_monthinterval_map":[{"key":"AString76783","value":{"months":1}},{"key":"AString89351","value":{"months":1}}],"string_string_map":[{"key":"AString76783","value":"AString76783"},{"key":"AString89351","value":"AString89351"}],"string_time32ms_map":[{"key":"AString76783","value":"00:00:00"},{"key":"AString89351","value":"00:00:00"}],"string_time32s_map":[{"key":"AString76783","value":"00:00:00"},{"key":"AString89351","value":"00:00:00"}],"string_time64ns_map":[{"key":"AString76783","value":"00:00:00"},{"key":"AString89351","value":"00:00:00"}],"string_time64us_map":[{"key":"AString76783","value":"00:00:00"},{"key":"AString89351","value":"00:00:00"}],"string_timestamp_ms_map":[{"key":"AString76783","value":"2021-01-02 00:00:00"},{"key":"AString89351","value":"2021-01-02 00:00:00"}],"string_timestamp_ns_map":[{"key":"AString76783","value":"2021-01-02 00:00:00"},{"key":"AString89351","value":"2021-01-02 00:00:00"}],"string_timestamp_s_map":[{"key":"AString76783","value":"2021-01-02 00:00:00"},{"key":"AString89351","value":"2021-01-02 00:00:00"}],"string_timestamp_us_map":[{"key":"AString76783","value":"2021-01-02 00:00:00"},{"key":"AString89351","value":"2021-01-02 00:00:00"}],"string_uint16_map":[{"key":"AString76783","value":63063},{"key":"AString89351","value":34191}],"string_uint32_map":[{"key":"AString76783","value":506910753},{"key":"AString89351","value":2112817056}],"string_uint64_map":[{"key":"AString76783","value":14697929703826477056},{"key":"AString89351","value":3812638179211989504}],"string_uint8_map":[{"key":"AString76783","value":78},{"key":"AString89351","value":21}],"string_uuid_map":[{"key":"AString76783","value":"00000000-0000-0000-0000-000000000001"},{"key":"AString89351","value":"00000000-0000-0000-0000-000000000001"}],"struct":{"binary":"7049Ug==","boolean":true,"date32":"2023-07-16","date64":"2023-07-16","daytimeinterval":{"days":6783,"milliseconds":6783},"decimal":"826476783.5364813824","duration_ms":"6476783ms","duration_ns":"6476783ns","duration_s":"6476783s","duration_us":"6476783us","float32":783.936,"float64":783.936,"inet":"192.0.2.0/24","int16":-30727,"int32":-1781555540,"int64":-5474557666971701248,"int8":-123,"json":{"test":["a","b",3]},"json_array":[{"test":"test"},123,{"test_number":456}],"largebinary":"7049Ug==","largestring":"AString76783","mac":"aa:bb:cc:dd:ee:ff","monthdaynanointerval":{"months":6783,"days":6783,"nanoseconds":6783},"monthinterval":{"months":1},"string":"AString76783","time32ms":"00:00:00","time32s":"00:00:00","time64ns":"00:00:00","time64us":"00:00:00","timestamp_ms":"2021-01-02 00:00:00","timestamp_ns":"2021-01-02 00:00:00","timestamp_s":"2021-01-02 00:00:00","timestamp_us":"2021-01-02 00:00:00","uint16":63063,"uint32":506910753,"uint64":14697929703826477056,"uint8":78,"uuid":"00000000-0000-0000-0000-000000000001"},"time32ms":"00:00:00","time32ms_list":["00:00:00",null,"00:00:00"],"time32s":"00:00:00","time32s_list":["00:00:00",null,"00:00:00"],"time64ns":"00:00:00","time64ns_list":["00:00:00",null,"00:00:00"],"time64us":"00:00:00","time64us_list":["00:00:00",null,"00:00:00"],"timestamp_ms":"2021-01-02 00:00:00","timestamp_ms_list":["2021-01-02 00:00:00",null,"2021-01-02 00:00:00"],"timestamp_ns":"2021-01-02 00:00:00","timestamp_ns_list":["2021-01-02 00:00:00",null,"2021-01-02 00:00:00"],"timestamp_s":"2021-01-02 00:00:00","timestamp_s_list":["2021-01-02 00:00:00",null,"2021-01-02 00:00:00"],"timestamp_us":"2021-01-02 00:00:00","timestamp_us_list":["2021-01-02 00:00:00",null,"2021-01-02 00:00:00"],"uint16":63063,"uint16_list":[63063,null,63063],"uint32":506910753,"uint32_list":[506910753,null,506910753],"uint64":14697929703826477056,"uint64_list":[14697929703826477056,null,14697929703826477056],"uint8":78,"uint8_list":[78,null,78],"uuid":"00000000-0000-0000-0000-000000000001","uuid_list":["00000000-0000-0000-0000-000000000001",null,"00000000-0000-0000-0000-000000000001"]} +{"_cq_id":"00000000-0000-0000-0000-000000000001","_cq_parent_id":null,"_cq_source_name":"test-source","_cq_sync_time":"2021-01-01 00:00:00","binary":null,"boolean":null,"boolean_list":null,"date32":null,"date32_list":null,"date64":null,"date64_list":null,"daytimeinterval":null,"daytimeinterval_list":null,"decimal":null,"decimal_list":null,"duration_ms":null,"duration_ms_list":null,"duration_ns":null,"duration_ns_list":null,"duration_s":null,"duration_s_list":null,"duration_us":null,"duration_us_list":null,"float32":null,"float32_list":null,"float64":null,"float64_list":null,"inet":null,"inet_list":null,"int16":null,"int16_list":null,"int32":null,"int32_list":null,"int64":null,"int64_list":null,"int8":null,"int8_list":null,"int_boolean_map":null,"int_date32_map":null,"int_date64_map":null,"int_daytimeinterval_map":null,"int_decimal_map":null,"int_duration_ms_map":null,"int_duration_ns_map":null,"int_duration_s_map":null,"int_duration_us_map":null,"int_float32_map":null,"int_float64_map":null,"int_inet_map":null,"int_int16_map":null,"int_int32_map":null,"int_int64_map":null,"int_int8_map":null,"int_largestring_map":null,"int_mac_map":null,"int_monthdaynanointerval_map":null,"int_monthinterval_map":null,"int_string_map":null,"int_time32ms_map":null,"int_time32s_map":null,"int_time64ns_map":null,"int_time64us_map":null,"int_timestamp_ms_map":null,"int_timestamp_ns_map":null,"int_timestamp_s_map":null,"int_timestamp_us_map":null,"int_uint16_map":null,"int_uint32_map":null,"int_uint64_map":null,"int_uint8_map":null,"int_uuid_map":null,"json":null,"json_array":null,"largebinary":null,"largestring":null,"largestring_list":null,"mac":null,"mac_list":null,"monthdaynanointerval":null,"monthdaynanointerval_list":null,"monthinterval":null,"monthinterval_list":null,"nested_struct":null,"string":null,"string_boolean_map":null,"string_date32_map":null,"string_date64_map":null,"string_daytimeinterval_map":null,"string_decimal_map":null,"string_duration_ms_map":null,"string_duration_ns_map":null,"string_duration_s_map":null,"string_duration_us_map":null,"string_float32_map":null,"string_float64_map":null,"string_inet_map":null,"string_int16_map":null,"string_int32_map":null,"string_int64_map":null,"string_int8_map":null,"string_largestring_map":null,"string_list":null,"string_mac_map":null,"string_monthdaynanointerval_map":null,"string_monthinterval_map":null,"string_string_map":null,"string_time32ms_map":null,"string_time32s_map":null,"string_time64ns_map":null,"string_time64us_map":null,"string_timestamp_ms_map":null,"string_timestamp_ns_map":null,"string_timestamp_s_map":null,"string_timestamp_us_map":null,"string_uint16_map":null,"string_uint32_map":null,"string_uint64_map":null,"string_uint8_map":null,"string_uuid_map":null,"struct":null,"time32ms":null,"time32ms_list":null,"time32s":null,"time32s_list":null,"time64ns":null,"time64ns_list":null,"time64us":null,"time64us_list":null,"timestamp_ms":null,"timestamp_ms_list":null,"timestamp_ns":null,"timestamp_ns_list":null,"timestamp_s":null,"timestamp_s_list":null,"timestamp_us":null,"timestamp_us_list":null,"uint16":null,"uint16_list":null,"uint32":null,"uint32_list":null,"uint64":null,"uint64_list":null,"uint8":null,"uint8_list":null,"uuid":null,"uuid_list":null} From 3585cd802359a5ff22fa3cd98bdf6590c8a1ebab Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Tue, 27 Jun 2023 08:56:13 +0100 Subject: [PATCH 14/16] replace `if !assert.NoError(t, ...) return` with `require.NoError(t, ...)` --- stream_test.go | 46 +++++++++++++--------------------------------- 1 file changed, 13 insertions(+), 33 deletions(-) diff --git a/stream_test.go b/stream_test.go index b906a15f..896e7bb2 100644 --- a/stream_test.go +++ b/stream_test.go @@ -12,6 +12,7 @@ import ( "github.com/cloudquery/filetypes/v4" "github.com/cloudquery/plugin-sdk/v4/schema" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) type uploadHelper struct { @@ -44,9 +45,7 @@ func TestHappyPath(t *testing.T) { cl, err := filetypes.NewClient(&filetypes.FileSpec{ Format: filetypes.FormatTypeJSON, }) - if !assert.NoError(t, err) { - return - } + require.NoError(t, err) table := &schema.Table{ Name: "test", @@ -61,29 +60,22 @@ func TestHappyPath(t *testing.T) { expectAt: 1, } s, err := cl.StartStream(table, u.Upload) - if !assert.NoError(t, err) { - return - } + require.NoError(t, err) bldr := array.NewRecordBuilder(memory.DefaultAllocator, table.ToArrowSchema()) bldr.Field(0).(*array.StringBuilder).Append("foo") bldr.Field(0).(*array.StringBuilder).Append("bar") record := bldr.NewRecord() - if !assert.NoError(t, s.Write([]arrow.Record{record})) { - return - } - - assert.NoError(t, s.Finish()) + require.NoError(t, s.Write([]arrow.Record{record})) + require.NoError(t, s.Finish()) } func TestWriteError(t *testing.T) { cl, err := filetypes.NewClient(&filetypes.FileSpec{ Format: filetypes.FormatTypeJSON, }) - if !assert.NoError(t, err) { - return - } + require.NoError(t, err) table := &schema.Table{ Name: "test", @@ -97,29 +89,22 @@ func TestWriteError(t *testing.T) { failAfter: 1, } s, err := cl.StartStream(table, u.Upload) - if !assert.NoError(t, err) { - return - } + require.NoError(t, err) bldr := array.NewRecordBuilder(memory.DefaultAllocator, table.ToArrowSchema()) bldr.Field(0).(*array.StringBuilder).Append("foo") bldr.Field(0).(*array.StringBuilder).Append("bar") record := bldr.NewRecord() - if !assert.NoError(t, s.Write([]arrow.Record{record})) { - return - } - - assert.ErrorIs(t, s.Finish(), errTest) + require.NoError(t, s.Write([]arrow.Record{record})) + require.ErrorIs(t, s.Finish(), errTest) } func TestCloseError(t *testing.T) { cl, err := filetypes.NewClient(&filetypes.FileSpec{ Format: filetypes.FormatTypeJSON, }) - if !assert.NoError(t, err) { - return - } + require.NoError(t, err) table := &schema.Table{ Name: "test", @@ -132,18 +117,13 @@ func TestCloseError(t *testing.T) { t: t, } s, err := cl.StartStream(table, u.Upload) - if !assert.NoError(t, err) { - return - } + require.NoError(t, err) bldr := array.NewRecordBuilder(memory.DefaultAllocator, table.ToArrowSchema()) bldr.Field(0).(*array.StringBuilder).Append("foo") bldr.Field(0).(*array.StringBuilder).Append("bar") record := bldr.NewRecord() - if !assert.NoError(t, s.Write([]arrow.Record{record})) { - return - } - - assert.ErrorIs(t, s.FinishWithError(errTest), errTest) + require.NoError(t, s.Write([]arrow.Record{record})) + require.ErrorIs(t, s.FinishWithError(errTest), errTest) } From ab63f052c54bf922c14d37789d2a0b3fc1e458ef Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Wed, 28 Jun 2023 19:48:03 +0100 Subject: [PATCH 15/16] Use 4.2.0-rc1 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c5b60c85..e8409d78 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.19 require ( github.com/apache/arrow/go/v13 v13.0.0-20230622042343-ec413b7763fe github.com/bradleyjkemp/cupaloy/v2 v2.8.0 - github.com/cloudquery/plugin-sdk/v4 v4.0.0-rc1 + github.com/cloudquery/plugin-sdk/v4 v4.2.0-rc1 github.com/goccy/go-json v0.10.2 github.com/google/uuid v1.3.0 github.com/stretchr/testify v1.8.4 diff --git a/go.sum b/go.sum index 67ae3d5a..8a538eb8 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oM github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= github.com/cloudquery/arrow/go/v13 v13.0.0-20230626001500-065602842c3a h1:O/FNq1+8YlWzHYNj2tokFQyja6GXsQBdkuvLMdpuaSw= github.com/cloudquery/arrow/go/v13 v13.0.0-20230626001500-065602842c3a/go.mod h1:W69eByFNO0ZR30q1/7Sr9d83zcVZmF2MiP3fFYAWJOc= -github.com/cloudquery/plugin-sdk/v4 v4.0.0-rc1 h1:YX8QPlzm5kiwbYsM8zKVXk1yu7ONeooJD77ZoA4hI5o= -github.com/cloudquery/plugin-sdk/v4 v4.0.0-rc1/go.mod h1:QfZgX4QC3Z1SdjnDmtzGAETaPDOYVTSWTW6XZQWHjeQ= +github.com/cloudquery/plugin-sdk/v4 v4.2.0-rc1 h1:sRjZ/Lb/yjLw92HzvgPiyVynbocbtaa13fEgS9MN/DQ= +github.com/cloudquery/plugin-sdk/v4 v4.2.0-rc1/go.mod h1:gn2ANihFC5SUMPCcYnVD+Gt3Cgn8OeXJW2/0lRUoB68= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= From d62fca069956d088600c717673db613095424c44 Mon Sep 17 00:00:00 2001 From: Kemal Hadimli Date: Wed, 28 Jun 2023 19:49:06 +0100 Subject: [PATCH 16/16] tidy --- go.mod | 1 - go.sum | 4 ---- 2 files changed, 5 deletions(-) diff --git a/go.mod b/go.mod index 673770a8..e8409d78 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,6 @@ require ( github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect github.com/andybalholm/brotli v1.0.5 // indirect github.com/apache/thrift v0.18.1 // indirect - github.com/cloudquery/plugin-pb-go v1.4.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect diff --git a/go.sum b/go.sum index a7984fd4..8a538eb8 100644 --- a/go.sum +++ b/go.sum @@ -10,10 +10,6 @@ github.com/cloudquery/arrow/go/v13 v13.0.0-20230626001500-065602842c3a h1:O/FNq1 github.com/cloudquery/arrow/go/v13 v13.0.0-20230626001500-065602842c3a/go.mod h1:W69eByFNO0ZR30q1/7Sr9d83zcVZmF2MiP3fFYAWJOc= github.com/cloudquery/plugin-sdk/v4 v4.2.0-rc1 h1:sRjZ/Lb/yjLw92HzvgPiyVynbocbtaa13fEgS9MN/DQ= github.com/cloudquery/plugin-sdk/v4 v4.2.0-rc1/go.mod h1:gn2ANihFC5SUMPCcYnVD+Gt3Cgn8OeXJW2/0lRUoB68= -github.com/cloudquery/plugin-pb-go v1.4.0 h1:sfy0oWSFac2JCJQJuKoR+8flZGKkEoUVORwZDNM3aiI= -github.com/cloudquery/plugin-pb-go v1.4.0/go.mod h1:NbWAtT2BzJQ9+XUWwh3IKBg3MOeV9ZEpHoHNAQ/YDV8= -github.com/cloudquery/plugin-sdk/v3 v3.10.6 h1:KqTsLZ6OA1h8BUMeMcU6BAD6TBW6ojgQaC4zDZMgvu0= -github.com/cloudquery/plugin-sdk/v3 v3.10.6/go.mod h1:QhBaVgiNyQ3P6uAzJWOYpYykHXL+WDZffwg1riTwv60= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=