From aaf00e4fd036b0746d203c6a27a241db6c372f45 Mon Sep 17 00:00:00 2001 From: Kilian Schulte Date: Fri, 19 Feb 2021 21:45:02 +0100 Subject: [PATCH 01/16] add types point, intArray, textArray, doubleArray --- lib/src/binary_codec.dart | 89 +++++++++++++++++++++++++++++++++++++++ lib/src/types.dart | 14 +++++- pubspec.yaml | 1 + 3 files changed, 103 insertions(+), 1 deletion(-) diff --git a/lib/src/binary_codec.dart b/lib/src/binary_codec.dart index 89c75cef..613a1a67 100644 --- a/lib/src/binary_codec.dart +++ b/lib/src/binary_codec.dart @@ -2,6 +2,7 @@ import 'dart:convert'; import 'dart:typed_data'; import 'package:buffer/buffer.dart'; +import 'package:latlong/latlong.dart'; import '../postgres.dart'; import 'types.dart'; @@ -199,12 +200,68 @@ class PostgresBinaryEncoder extends Converter { } return outBuffer; } + + case PostgreSQLDataType.point: + { + if (value is LatLng) { + final bd = ByteData(16); + bd.setFloat64(0, value.latitude); + bd.setFloat64(8, value.longitude); + return bd.buffer.asUint8List(); + } + throw FormatException('Invalid type for parameter value. Expected: LatLng Got: ${value.runtimeType}'); + } + + case PostgreSQLDataType.integerArray: + { + if (value is List) { + return writeListBytes(value, 23, (_) => 4, (bd, offset, item) => bd.setInt32(offset, item)); + } + throw FormatException('Invalid type for parameter value. Expected: List Got: ${value.runtimeType}'); + } + + case PostgreSQLDataType.textArray: + { + if (value is List) { + final bytesArray = value.map((v) => castBytes(utf8.encode(v))); + return writeListBytes(bytesArray, 25, (item) => item.length, (bd, offset, item) { + item.forEach((i) => bd.setUint8(offset++, i)); + }); + } + throw FormatException('Invalid type for parameter value. Expected: List Got: ${value.runtimeType}'); + } + + case PostgreSQLDataType.doubleArray: + { + if (value is List) { + return writeListBytes(value, 701, (_) => 8, (bd, offset, item) => bd.setFloat64(offset, item)); + } + throw FormatException('Invalid type for parameter value. Expected: List Got: ${value.runtimeType}'); + } } throw PostgreSQLException('Unsupported datatype'); } } +Uint8List writeListBytes(Iterable value, int type, int Function(T item) lengthEncoder, + void Function(ByteData bd, int offset, T item) valueEncoder) { + final bd = ByteData(20 + value.fold(0, (sum, item) => 4 + lengthEncoder(item))); + bd.setInt32(0, 1); // dimension + bd.setInt32(4, 0); // ign + bd.setInt32(8, type); // type + bd.setInt32(12, value.length); // size + bd.setInt32(16, 1); // index + var offset = 20; + for (var i in value) { + final len = lengthEncoder(i); + bd.setInt32(offset, len); // value length + valueEncoder(bd, offset + 4, i); // value + offset += 4 + len; + } + return bd.buffer.asUint8List(); +} + class PostgresBinaryDecoder extends Converter { const PostgresBinaryDecoder(this.typeCode); @@ -277,6 +334,18 @@ class PostgresBinaryDecoder extends Converter { return buf.toString(); } + + case PostgreSQLDataType.point: + return LatLng(buffer.getFloat64(0), buffer.getFloat64(8)); + + case PostgreSQLDataType.integerArray: + return readListBytes(buffer, (offset, _) => buffer.getInt32(offset)); + + case PostgreSQLDataType.textArray: + return readListBytes(buffer, (offset, length) => utf8.decode(value.sublist(offset, offset + length))); + + case PostgreSQLDataType.doubleArray: + return readListBytes(buffer, (offset, _) => buffer.getFloat64(offset)); } // We'll try and decode this as a utf8 string and return that @@ -290,6 +359,21 @@ class PostgresBinaryDecoder extends Converter { } } + List readListBytes(ByteData buffer, T Function(int offset, int length) valueDecoder) { + final decoded = [].cast(); + + final size = buffer.getInt32(12); + + var offset = 20; + for (var i = 0; i <= size; i++) { + final len = buffer.getInt32(offset); + decoded.add(valueDecoder(offset + 4, len)); + offset += 4 + len; + } + + return decoded; + } + static final Map typeMap = { 16: PostgreSQLDataType.boolean, 17: PostgreSQLDataType.byteArray, @@ -298,8 +382,13 @@ class PostgresBinaryDecoder extends Converter { 21: PostgreSQLDataType.smallInteger, 23: PostgreSQLDataType.integer, 25: PostgreSQLDataType.text, + 114: PostgreSQLDataType.json, + 600: PostgreSQLDataType.point, 700: PostgreSQLDataType.real, 701: PostgreSQLDataType.double, + 1007: PostgreSQLDataType.integerArray, + 1009: PostgreSQLDataType.textArray, + 1022: PostgreSQLDataType.doubleArray, 1082: PostgreSQLDataType.date, 1114: PostgreSQLDataType.timestampWithoutTimezone, 1184: PostgreSQLDataType.timestampWithTimezone, diff --git a/lib/src/types.dart b/lib/src/types.dart index ee76281f..8619fdf6 100644 --- a/lib/src/types.dart +++ b/lib/src/types.dart @@ -64,5 +64,17 @@ enum PostgreSQLDataType { /// /// Must contain 32 hexadecimal characters. May contain any number of '-' characters. /// When returned from database, format will be xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx. - uuid + uuid, + + /// Must be a [LatLng] + point, + + /// Must be a [List] + integerArray, + + /// Must be a [List] + textArray, + + /// Must be a [List] + doubleArray, } diff --git a/pubspec.yaml b/pubspec.yaml index 48ef18e4..53d6447a 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -9,6 +9,7 @@ environment: dependencies: buffer: ^1.0.6 crypto: ^2.0.0 + latlong: ^0.6.1 dev_dependencies: pedantic: ^1.0.0 From cd31200243b98f8916f773070aad3b5facb3de2b Mon Sep 17 00:00:00 2001 From: Kilian Schulte Date: Fri, 19 Feb 2021 22:09:57 +0100 Subject: [PATCH 02/16] fix json type --- lib/src/binary_codec.dart | 70 +++++++++++++++++++++++++-------------- lib/src/query.dart | 9 +++-- lib/src/substituter.dart | 12 ++++++- lib/src/types.dart | 5 +++ test/encoding_test.dart | 8 ++--- test/query_test.dart | 2 +- 6 files changed, 73 insertions(+), 33 deletions(-) diff --git a/lib/src/binary_codec.dart b/lib/src/binary_codec.dart index 613a1a67..2ae626a7 100644 --- a/lib/src/binary_codec.dart +++ b/lib/src/binary_codec.dart @@ -145,7 +145,7 @@ class PostgresBinaryEncoder extends Converter { 'Invalid type for parameter value. Expected: DateTime Got: ${value.runtimeType}'); } - case PostgreSQLDataType.json: + case PostgreSQLDataType.jsonb: { final jsonBytes = utf8.encode(json.encode(value)); final writer = ByteDataWriter(bufferLength: jsonBytes.length + 1); @@ -154,6 +154,9 @@ class PostgresBinaryEncoder extends Converter { return writer.toBytes(); } + case PostgreSQLDataType.json: + return castBytes(utf8.encode(json.encode(value))); + case PostgreSQLDataType.byteArray: { if (value is List) { @@ -209,34 +212,41 @@ class PostgresBinaryEncoder extends Converter { bd.setFloat64(8, value.longitude); return bd.buffer.asUint8List(); } - throw FormatException('Invalid type for parameter value. Expected: LatLng Got: ${value.runtimeType}'); + throw FormatException( + 'Invalid type for parameter value. Expected: LatLng Got: ${value.runtimeType}'); } case PostgreSQLDataType.integerArray: { if (value is List) { - return writeListBytes(value, 23, (_) => 4, (bd, offset, item) => bd.setInt32(offset, item)); + return writeListBytes(value, 23, (_) => 4, + (bd, offset, item) => bd.setInt32(offset, item)); } - throw FormatException('Invalid type for parameter value. Expected: List Got: ${value.runtimeType}'); + throw FormatException( + 'Invalid type for parameter value. Expected: List Got: ${value.runtimeType}'); } case PostgreSQLDataType.textArray: { if (value is List) { final bytesArray = value.map((v) => castBytes(utf8.encode(v))); - return writeListBytes(bytesArray, 25, (item) => item.length, (bd, offset, item) { + return writeListBytes( + bytesArray, 25, (item) => item.length, (bd, offset, item) { item.forEach((i) => bd.setUint8(offset++, i)); }); } - throw FormatException('Invalid type for parameter value. Expected: List Got: ${value.runtimeType}'); + throw FormatException( + 'Invalid type for parameter value. Expected: List Got: ${value.runtimeType}'); } case PostgreSQLDataType.doubleArray: { if (value is List) { - return writeListBytes(value, 701, (_) => 8, (bd, offset, item) => bd.setFloat64(offset, item)); + return writeListBytes(value, 701, (_) => 8, + (bd, offset, item) => bd.setFloat64(offset, item)); } - throw FormatException('Invalid type for parameter value. Expected: List Got: ${value.runtimeType}'); + throw FormatException( + 'Invalid type for parameter value. Expected: List Got: ${value.runtimeType}'); } } @@ -244,9 +254,13 @@ class PostgresBinaryEncoder extends Converter { } } -Uint8List writeListBytes(Iterable value, int type, int Function(T item) lengthEncoder, +Uint8List writeListBytes( + Iterable value, + int type, + int Function(T item) lengthEncoder, void Function(ByteData bd, int offset, T item) valueEncoder) { - final bd = ByteData(20 + value.fold(0, (sum, item) => 4 + lengthEncoder(item))); + final bd = + ByteData(20 + value.fold(0, (sum, item) => 4 + lengthEncoder(item))); bd.setInt32(0, 1); // dimension bd.setInt32(4, 0); // ign bd.setInt32(8, type); // type @@ -304,7 +318,7 @@ class PostgresBinaryDecoder extends Converter { case PostgreSQLDataType.date: return DateTime.utc(2000).add(Duration(days: buffer.getInt32(0))); - case PostgreSQLDataType.json: + case PostgreSQLDataType.jsonb: { // Removes version which is first character and currently always '1' final bytes = value.buffer @@ -312,6 +326,9 @@ class PostgresBinaryDecoder extends Converter { return json.decode(utf8.decode(bytes)); } + case PostgreSQLDataType.json: + return json.decode(utf8.decode(value)); + case PostgreSQLDataType.byteArray: return value; @@ -339,27 +356,30 @@ class PostgresBinaryDecoder extends Converter { return LatLng(buffer.getFloat64(0), buffer.getFloat64(8)); case PostgreSQLDataType.integerArray: - return readListBytes(buffer, (offset, _) => buffer.getInt32(offset)); + return readListBytes( + buffer, (offset, _) => buffer.getInt32(offset)); case PostgreSQLDataType.textArray: - return readListBytes(buffer, (offset, length) => utf8.decode(value.sublist(offset, offset + length))); + return readListBytes( + buffer, + (offset, length) => + utf8.decode(value.sublist(offset, offset + length))); case PostgreSQLDataType.doubleArray: - return readListBytes(buffer, (offset, _) => buffer.getFloat64(offset)); + return readListBytes( + buffer, (offset, _) => buffer.getFloat64(offset)); } - // We'll try and decode this as a utf8 string and return that - // for many internal types, this is valid. If it fails, - // we just return the bytes and let the caller figure out what to - // do with it. - try { - return utf8.decode(value); - } catch (_) { - return value; - } + // prepend type-code and return raw bytes + + final writer = ByteDataWriter(bufferLength: value.length + 4); + writer.writeUint32(typeCode); + writer.write(value); + return writer.toBytes(); } - List readListBytes(ByteData buffer, T Function(int offset, int length) valueDecoder) { + List readListBytes( + ByteData buffer, T Function(int offset, int length) valueDecoder) { final decoded = [].cast(); final size = buffer.getInt32(12); @@ -393,6 +413,6 @@ class PostgresBinaryDecoder extends Converter { 1114: PostgreSQLDataType.timestampWithoutTimezone, 1184: PostgreSQLDataType.timestampWithTimezone, 2950: PostgreSQLDataType.uuid, - 3802: PostgreSQLDataType.json, + 3802: PostgreSQLDataType.jsonb, }; } diff --git a/lib/src/query.dart b/lib/src/query.dart index 26bd2d59..7973b1cc 100644 --- a/lib/src/query.dart +++ b/lib/src/query.dart @@ -320,10 +320,15 @@ class PostgreSQLFormatIdentifier { 'date': PostgreSQLDataType.date, 'timestamp': PostgreSQLDataType.timestampWithoutTimezone, 'timestamptz': PostgreSQLDataType.timestampWithTimezone, - 'jsonb': PostgreSQLDataType.json, + 'jsonb': PostgreSQLDataType.jsonb, 'bytea': PostgreSQLDataType.byteArray, 'name': PostgreSQLDataType.name, - 'uuid': PostgreSQLDataType.uuid + 'uuid': PostgreSQLDataType.uuid, + 'json': PostgreSQLDataType.json, + 'point': PostgreSQLDataType.point, + '_int4': PostgreSQLDataType.integerArray, + '_text': PostgreSQLDataType.textArray, + '_float8': PostgreSQLDataType.doubleArray, }; factory PostgreSQLFormatIdentifier(String t) { diff --git a/lib/src/substituter.dart b/lib/src/substituter.dart index 0d7f6474..0f45e48f 100644 --- a/lib/src/substituter.dart +++ b/lib/src/substituter.dart @@ -39,7 +39,7 @@ class PostgreSQLFormat { return 'timestamptz'; case PostgreSQLDataType.date: return 'date'; - case PostgreSQLDataType.json: + case PostgreSQLDataType.jsonb: return 'jsonb'; case PostgreSQLDataType.byteArray: return 'bytea'; @@ -47,6 +47,16 @@ class PostgreSQLFormat { return 'name'; case PostgreSQLDataType.uuid: return 'uuid'; + case PostgreSQLDataType.point: + return 'point'; + case PostgreSQLDataType.json: + return 'json'; + case PostgreSQLDataType.integerArray: + return '_int4'; + case PostgreSQLDataType.textArray: + return '_text'; + case PostgreSQLDataType.doubleArray: + return '_float8'; } return null; diff --git a/lib/src/types.dart b/lib/src/types.dart index 8619fdf6..d010e321 100644 --- a/lib/src/types.dart +++ b/lib/src/types.dart @@ -45,6 +45,11 @@ enum PostgreSQLDataType { /// Must be a [DateTime] (contains year, month and day only) date, + /// Must be encodable via [json.encode]. + /// + /// Values will be encoded via [json.encode] before being sent to the database. + jsonb, + /// Must be encodable via [json.encode]. /// /// Values will be encoded via [json.encode] before being sent to the database. diff --git a/test/encoding_test.dart b/test/encoding_test.dart index edfc81ed..076c07fa 100644 --- a/test/encoding_test.dart +++ b/test/encoding_test.dart @@ -185,14 +185,14 @@ void main() { }); test('jsonb', () async { - await expectInverse('string', PostgreSQLDataType.json); - await expectInverse(2, PostgreSQLDataType.json); - await expectInverse(['foo'], PostgreSQLDataType.json); + await expectInverse('string', PostgreSQLDataType.jsonb); + await expectInverse(2, PostgreSQLDataType.jsonb); + await expectInverse(['foo'], PostgreSQLDataType.jsonb); await expectInverse({ 'key': 'val', 'key1': 1, 'array': ['foo'] - }, PostgreSQLDataType.json); + }, PostgreSQLDataType.jsonb); try { await conn.query('INSERT INTO t (v) VALUES (@v:jsonb)', diff --git a/test/query_test.dart b/test/query_test.dart index 96276321..cf40dd6c 100644 --- a/test/query_test.dart +++ b/test/query_test.dart @@ -179,7 +179,7 @@ void main() { '${PostgreSQLFormat.id('dt', type: PostgreSQLDataType.date)},' '${PostgreSQLFormat.id('ts', type: PostgreSQLDataType.timestampWithoutTimezone)},' '${PostgreSQLFormat.id('tsz', type: PostgreSQLDataType.timestampWithTimezone)},' - '${PostgreSQLFormat.id('j', type: PostgreSQLDataType.json)},' + '${PostgreSQLFormat.id('j', type: PostgreSQLDataType.jsonb)},' '${PostgreSQLFormat.id('u', type: PostgreSQLDataType.uuid)})' ' returning i,s, bi, bs, bl, si, t, f, d, dt, ts, tsz, j, u', substitutionValues: { From cfcd70021872fad1d1687e3be3fed097774660ff Mon Sep 17 00:00:00 2001 From: Kilian Schulte Date: Fri, 19 Feb 2021 22:24:04 +0100 Subject: [PATCH 03/16] add varchar type --- lib/src/binary_codec.dart | 3 +++ lib/src/query.dart | 1 + lib/src/substituter.dart | 2 ++ lib/src/types.dart | 3 +++ 4 files changed, 9 insertions(+) diff --git a/lib/src/binary_codec.dart b/lib/src/binary_codec.dart index 2ae626a7..74fb06b6 100644 --- a/lib/src/binary_codec.dart +++ b/lib/src/binary_codec.dart @@ -83,6 +83,7 @@ class PostgresBinaryEncoder extends Converter { } case PostgreSQLDataType.name: case PostgreSQLDataType.text: + case PostgreSQLDataType.varChar: { if (value is String) { return castBytes(utf8.encode(value)); @@ -295,6 +296,7 @@ class PostgresBinaryDecoder extends Converter { switch (dataType) { case PostgreSQLDataType.name: case PostgreSQLDataType.text: + case PostgreSQLDataType.varChar: return utf8.decode(value); case PostgreSQLDataType.boolean: return buffer.getInt8(0) != 0; @@ -408,6 +410,7 @@ class PostgresBinaryDecoder extends Converter { 701: PostgreSQLDataType.double, 1007: PostgreSQLDataType.integerArray, 1009: PostgreSQLDataType.textArray, + 1043: PostgreSQLDataType.varChar, 1022: PostgreSQLDataType.doubleArray, 1082: PostgreSQLDataType.date, 1114: PostgreSQLDataType.timestampWithoutTimezone, diff --git a/lib/src/query.dart b/lib/src/query.dart index 7973b1cc..69ab9a58 100644 --- a/lib/src/query.dart +++ b/lib/src/query.dart @@ -329,6 +329,7 @@ class PostgreSQLFormatIdentifier { '_int4': PostgreSQLDataType.integerArray, '_text': PostgreSQLDataType.textArray, '_float8': PostgreSQLDataType.doubleArray, + 'varchar': PostgreSQLDataType.varChar, }; factory PostgreSQLFormatIdentifier(String t) { diff --git a/lib/src/substituter.dart b/lib/src/substituter.dart index 0f45e48f..d4ab49cb 100644 --- a/lib/src/substituter.dart +++ b/lib/src/substituter.dart @@ -57,6 +57,8 @@ class PostgreSQLFormat { return '_text'; case PostgreSQLDataType.doubleArray: return '_float8'; + case PostgreSQLDataType.varChar: + return 'varchar'; } return null; diff --git a/lib/src/types.dart b/lib/src/types.dart index d010e321..a1569c17 100644 --- a/lib/src/types.dart +++ b/lib/src/types.dart @@ -82,4 +82,7 @@ enum PostgreSQLDataType { /// Must be a [List] doubleArray, + + /// Must be a [String] + varChar, } From 66c5de8252d76cb9a6fa067bd12e80cec4de97bf Mon Sep 17 00:00:00 2001 From: Kilian Schulte Date: Sat, 20 Feb 2021 00:15:14 +0100 Subject: [PATCH 04/16] fix array decoder --- lib/src/binary_codec.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/binary_codec.dart b/lib/src/binary_codec.dart index 74fb06b6..8639637a 100644 --- a/lib/src/binary_codec.dart +++ b/lib/src/binary_codec.dart @@ -387,7 +387,7 @@ class PostgresBinaryDecoder extends Converter { final size = buffer.getInt32(12); var offset = 20; - for (var i = 0; i <= size; i++) { + for (var i = 0; i < size; i++) { final len = buffer.getInt32(offset); decoded.add(valueDecoder(offset + 4, len)); offset += 4 + len; From af466d37687519bb6f8677cef1345ca6bb6b1939 Mon Sep 17 00:00:00 2001 From: Kilian Schulte Date: Sat, 20 Feb 2021 06:39:23 +0100 Subject: [PATCH 05/16] fix empty array --- lib/src/binary_codec.dart | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/src/binary_codec.dart b/lib/src/binary_codec.dart index 8639637a..33b7fbad 100644 --- a/lib/src/binary_codec.dart +++ b/lib/src/binary_codec.dart @@ -384,6 +384,10 @@ class PostgresBinaryDecoder extends Converter { ByteData buffer, T Function(int offset, int length) valueDecoder) { final decoded = [].cast(); + if (buffer.lengthInBytes < 20) { + return decoded; + } + final size = buffer.getInt32(12); var offset = 20; From 66d12ea57cf825f427067b2c111f7889a7fd2bd7 Mon Sep 17 00:00:00 2001 From: Kilian Schulte Date: Sat, 20 Feb 2021 06:48:59 +0100 Subject: [PATCH 06/16] check offset overflow --- lib/src/binary_codec.dart | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/src/binary_codec.dart b/lib/src/binary_codec.dart index 33b7fbad..a0ed125c 100644 --- a/lib/src/binary_codec.dart +++ b/lib/src/binary_codec.dart @@ -392,7 +392,13 @@ class PostgresBinaryDecoder extends Converter { var offset = 20; for (var i = 0; i < size; i++) { + if (offset >= buffer.lengthInBytes - 4) { + break; + } final len = buffer.getInt32(offset); + if (offset + 4 + len > buffer.lengthInBytes) { + break; + } decoded.add(valueDecoder(offset + 4, len)); offset += 4 + len; } From a552371d5144bcf6b6abc65ca8517d54dca00226 Mon Sep 17 00:00:00 2001 From: Kilian Schulte Date: Sat, 20 Feb 2021 18:37:28 +0100 Subject: [PATCH 07/16] add trycatch --- lib/src/binary_codec.dart | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/lib/src/binary_codec.dart b/lib/src/binary_codec.dart index a0ed125c..fa535d6f 100644 --- a/lib/src/binary_codec.dart +++ b/lib/src/binary_codec.dart @@ -384,26 +384,20 @@ class PostgresBinaryDecoder extends Converter { ByteData buffer, T Function(int offset, int length) valueDecoder) { final decoded = [].cast(); - if (buffer.lengthInBytes < 20) { - return decoded; - } - - final size = buffer.getInt32(12); - - var offset = 20; - for (var i = 0; i < size; i++) { - if (offset >= buffer.lengthInBytes - 4) { - break; - } - final len = buffer.getInt32(offset); - if (offset + 4 + len > buffer.lengthInBytes) { - break; + try { + final size = buffer.getInt32(12); + + var offset = 20; + for (var i = 0; i < size; i++) { + final len = buffer.getInt32(offset); + decoded.add(valueDecoder(offset + 4, len)); + offset += 4 + len; } - decoded.add(valueDecoder(offset + 4, len)); - offset += 4 + len; - } - return decoded; + return decoded; + } on RangeError catch (_) { + return decoded; + } } static final Map typeMap = { @@ -429,3 +423,11 @@ class PostgresBinaryDecoder extends Converter { 3802: PostgreSQLDataType.jsonb, }; } + +/* + [0,0,0,1,0,0,0,0,0,0,2,189,0,0,0,2,0,0,0,1,0,0,0,8,64,0,0,0,0,0,0,0,0,0,0,8,64,16,0,0,0,0,0,0] + dim ign type size index len1 int1 len2 int2 + + [0,0,0,1,0,0,0,0,0,0,0,25,0,0,0,1,0,0,0,1,0,0,0,13,84,101,115,108,97,32,77,111,100,101,108,32,83] + dim ign type size index len1 str1 + */ From a6e9dd62bdad08129d2e54f488b8c981a4eb023b Mon Sep 17 00:00:00 2001 From: Kilian Schulte Date: Tue, 16 Mar 2021 13:09:21 +0100 Subject: [PATCH 08/16] add jsonb array type --- lib/src/binary_codec.dart | 33 +++++++++++++++++++++++++++++---- lib/src/query.dart | 1 + lib/src/substituter.dart | 2 ++ lib/src/types.dart | 3 +++ 4 files changed, 35 insertions(+), 4 deletions(-) diff --git a/lib/src/binary_codec.dart b/lib/src/binary_codec.dart index fa535d6f..11753fdf 100644 --- a/lib/src/binary_codec.dart +++ b/lib/src/binary_codec.dart @@ -249,6 +249,25 @@ class PostgresBinaryEncoder extends Converter { throw FormatException( 'Invalid type for parameter value. Expected: List Got: ${value.runtimeType}'); } + + case PostgreSQLDataType.jsonbArray: + { + if (value is List) { + final bytesArray = value.map((v) { + final jsonBytes = utf8.encode(json.encode(v)); + final writer = ByteDataWriter(bufferLength: jsonBytes.length + 1); + writer.writeUint8(1); + writer.write(jsonBytes); + return writer.toBytes(); + }); + return writeListBytes( + bytesArray, 3802, (item) => item.length, (bd, offset, item) { + item.forEach((i) => bd.setUint8(offset++, i)); + }); + } + throw FormatException( + 'Invalid type for parameter value. Expected: List Got: ${value.runtimeType}'); + } } throw PostgreSQLException('Unsupported datatype'); @@ -362,14 +381,19 @@ class PostgresBinaryDecoder extends Converter { buffer, (offset, _) => buffer.getInt32(offset)); case PostgreSQLDataType.textArray: - return readListBytes( - buffer, - (offset, length) => - utf8.decode(value.sublist(offset, offset + length))); + return readListBytes(buffer, (offset, length) { + return utf8.decode(value.sublist(offset, offset + length)); + }); case PostgreSQLDataType.doubleArray: return readListBytes( buffer, (offset, _) => buffer.getFloat64(offset)); + + case PostgreSQLDataType.jsonbArray: + return readListBytes(buffer, (offset, length) { + final bytes = value.sublist(offset, offset + length); + return json.decode(utf8.decode(bytes)); + }); } // prepend type-code and return raw bytes @@ -421,6 +445,7 @@ class PostgresBinaryDecoder extends Converter { 1184: PostgreSQLDataType.timestampWithTimezone, 2950: PostgreSQLDataType.uuid, 3802: PostgreSQLDataType.jsonb, + 3807: PostgreSQLDataType.jsonbArray, }; } diff --git a/lib/src/query.dart b/lib/src/query.dart index 69ab9a58..b351997b 100644 --- a/lib/src/query.dart +++ b/lib/src/query.dart @@ -330,6 +330,7 @@ class PostgreSQLFormatIdentifier { '_text': PostgreSQLDataType.textArray, '_float8': PostgreSQLDataType.doubleArray, 'varchar': PostgreSQLDataType.varChar, + '_jsonb': PostgreSQLDataType.jsonbArray, }; factory PostgreSQLFormatIdentifier(String t) { diff --git a/lib/src/substituter.dart b/lib/src/substituter.dart index d4ab49cb..46f710ec 100644 --- a/lib/src/substituter.dart +++ b/lib/src/substituter.dart @@ -59,6 +59,8 @@ class PostgreSQLFormat { return '_float8'; case PostgreSQLDataType.varChar: return 'varchar'; + case PostgreSQLDataType.jsonbArray: + return '_jsonb'; } return null; diff --git a/lib/src/types.dart b/lib/src/types.dart index a1569c17..55f08209 100644 --- a/lib/src/types.dart +++ b/lib/src/types.dart @@ -85,4 +85,7 @@ enum PostgreSQLDataType { /// Must be a [String] varChar, + + /// Must be a [List] of encodable objects + jsonbArray, } From 34b4cd64d2f12ce1eb2465d1bde6374ba8dc879d Mon Sep 17 00:00:00 2001 From: Kilian Schulte Date: Tue, 16 Mar 2021 14:11:29 +0100 Subject: [PATCH 09/16] fix jsonb array decoding --- lib/src/binary_codec.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/binary_codec.dart b/lib/src/binary_codec.dart index 11753fdf..c36b6e6e 100644 --- a/lib/src/binary_codec.dart +++ b/lib/src/binary_codec.dart @@ -391,7 +391,7 @@ class PostgresBinaryDecoder extends Converter { case PostgreSQLDataType.jsonbArray: return readListBytes(buffer, (offset, length) { - final bytes = value.sublist(offset, offset + length); + final bytes = value.sublist(offset + 1, offset + length - 1); return json.decode(utf8.decode(bytes)); }); } From 7840b21657eb45923fb1deda661fb3b3f1888e83 Mon Sep 17 00:00:00 2001 From: Kilian Schulte Date: Tue, 16 Mar 2021 14:12:36 +0100 Subject: [PATCH 10/16] fix length index --- lib/src/binary_codec.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/binary_codec.dart b/lib/src/binary_codec.dart index c36b6e6e..3aa55ea6 100644 --- a/lib/src/binary_codec.dart +++ b/lib/src/binary_codec.dart @@ -391,7 +391,7 @@ class PostgresBinaryDecoder extends Converter { case PostgreSQLDataType.jsonbArray: return readListBytes(buffer, (offset, length) { - final bytes = value.sublist(offset + 1, offset + length - 1); + final bytes = value.sublist(offset + 1, offset + length); return json.decode(utf8.decode(bytes)); }); } From b01be549945190312b08a24937c5786f9e9de5c9 Mon Sep 17 00:00:00 2001 From: Kilian Schulte Date: Wed, 24 Mar 2021 14:50:54 +0100 Subject: [PATCH 11/16] switch to latlng package and cleanup code --- lib/src/binary_codec.dart | 24 ++++++++++-------------- pubspec.yaml | 2 +- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/lib/src/binary_codec.dart b/lib/src/binary_codec.dart index 3aa55ea6..2058c737 100644 --- a/lib/src/binary_codec.dart +++ b/lib/src/binary_codec.dart @@ -2,7 +2,7 @@ import 'dart:convert'; import 'dart:typed_data'; import 'package:buffer/buffer.dart'; -import 'package:latlong/latlong.dart'; +import 'package:latlng/latlng.dart'; import '../postgres.dart'; import 'types.dart'; @@ -396,12 +396,16 @@ class PostgresBinaryDecoder extends Converter { }); } - // prepend type-code and return raw bytes + // We'll try and decode this as a utf8 string and return that + // for many internal types, this is valid. If it fails, + // we just return the bytes and let the caller figure out what to + // do with it. - final writer = ByteDataWriter(bufferLength: value.length + 4); - writer.writeUint32(typeCode); - writer.write(value); - return writer.toBytes(); + try { + return utf8.decode(value); + } catch (_) { + return value; + } } List readListBytes( @@ -448,11 +452,3 @@ class PostgresBinaryDecoder extends Converter { 3807: PostgreSQLDataType.jsonbArray, }; } - -/* - [0,0,0,1,0,0,0,0,0,0,2,189,0,0,0,2,0,0,0,1,0,0,0,8,64,0,0,0,0,0,0,0,0,0,0,8,64,16,0,0,0,0,0,0] - dim ign type size index len1 int1 len2 int2 - - [0,0,0,1,0,0,0,0,0,0,0,25,0,0,0,1,0,0,0,1,0,0,0,13,84,101,115,108,97,32,77,111,100,101,108,32,83] - dim ign type size index len1 str1 - */ diff --git a/pubspec.yaml b/pubspec.yaml index 53d6447a..79fbf505 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -9,7 +9,7 @@ environment: dependencies: buffer: ^1.0.6 crypto: ^2.0.0 - latlong: ^0.6.1 + latlng: ^0.1.0 dev_dependencies: pedantic: ^1.0.0 From dc96b190f5643e876684ec218a854117b63fecea Mon Sep 17 00:00:00 2001 From: Kilian Schulte Date: Wed, 24 Mar 2021 15:15:51 +0100 Subject: [PATCH 12/16] fix dependencies --- pubspec.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pubspec.yaml b/pubspec.yaml index c2e9b5f6..d2d2210e 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -7,8 +7,8 @@ environment: sdk: '>=2.12.0 <3.0.0' dependencies: - buffer: ^1.0.6 - crypto: ^2.0.0 + buffer: ^1.1.0 + crypto: ^3.0.0 latlng: ^0.1.0 collection: ^1.15.0 From 8bb0878f6e6757b290237c7df26b09ee80ad9d0c Mon Sep 17 00:00:00 2001 From: Kilian Schulte Date: Wed, 24 Mar 2021 21:50:35 +0100 Subject: [PATCH 13/16] implement code suggestions and tests --- lib/postgres.dart | 1 + lib/src/binary_codec.dart | 120 ++++++++++++++++++-------------------- lib/src/models.dart | 16 +++++ lib/src/text_codec.dart | 50 ++++++++++++++++ pubspec.yaml | 1 - test/decode_test.dart | 45 +++++++++++--- test/encoding_test.dart | 120 +++++++++++++++++++++++++++++++++++++- test/query_test.dart | 91 ++++++++++++++++++++++++----- 8 files changed, 355 insertions(+), 89 deletions(-) create mode 100644 lib/src/models.dart diff --git a/lib/postgres.dart b/lib/postgres.dart index 01c9235f..fdde21da 100644 --- a/lib/postgres.dart +++ b/lib/postgres.dart @@ -2,5 +2,6 @@ library postgres; export 'src/connection.dart'; export 'src/execution_context.dart'; +export 'src/models.dart'; export 'src/substituter.dart'; export 'src/types.dart'; diff --git a/lib/src/binary_codec.dart b/lib/src/binary_codec.dart index 2058c737..31201562 100644 --- a/lib/src/binary_codec.dart +++ b/lib/src/binary_codec.dart @@ -2,7 +2,6 @@ import 'dart:convert'; import 'dart:typed_data'; import 'package:buffer/buffer.dart'; -import 'package:latlng/latlng.dart'; import '../postgres.dart'; import 'types.dart'; @@ -207,21 +206,21 @@ class PostgresBinaryEncoder extends Converter { case PostgreSQLDataType.point: { - if (value is LatLng) { + if (value is PgPoint) { final bd = ByteData(16); bd.setFloat64(0, value.latitude); bd.setFloat64(8, value.longitude); return bd.buffer.asUint8List(); } throw FormatException( - 'Invalid type for parameter value. Expected: LatLng Got: ${value.runtimeType}'); + 'Invalid type for parameter value. Expected: PgPoint Got: ${value.runtimeType}'); } case PostgreSQLDataType.integerArray: { if (value is List) { - return writeListBytes(value, 23, (_) => 4, - (bd, offset, item) => bd.setInt32(offset, item)); + return writeListBytes( + value, 23, (_) => 4, (writer, item) => writer.writeInt32(item)); } throw FormatException( 'Invalid type for parameter value. Expected: List Got: ${value.runtimeType}'); @@ -230,11 +229,9 @@ class PostgresBinaryEncoder extends Converter { case PostgreSQLDataType.textArray: { if (value is List) { - final bytesArray = value.map((v) => castBytes(utf8.encode(v))); - return writeListBytes( - bytesArray, 25, (item) => item.length, (bd, offset, item) { - item.forEach((i) => bd.setUint8(offset++, i)); - }); + final bytesArray = value.map((v) => utf8.encode(v)); + return writeListBytes>(bytesArray, 25, + (item) => item.length, (writer, item) => writer.write(item)); } throw FormatException( 'Invalid type for parameter value. Expected: List Got: ${value.runtimeType}'); @@ -244,7 +241,7 @@ class PostgresBinaryEncoder extends Converter { { if (value is List) { return writeListBytes(value, 701, (_) => 8, - (bd, offset, item) => bd.setFloat64(offset, item)); + (writer, item) => writer.writeFloat64(item)); } throw FormatException( 'Invalid type for parameter value. Expected: List Got: ${value.runtimeType}'); @@ -253,16 +250,11 @@ class PostgresBinaryEncoder extends Converter { case PostgreSQLDataType.jsonbArray: { if (value is List) { - final bytesArray = value.map((v) { - final jsonBytes = utf8.encode(json.encode(v)); - final writer = ByteDataWriter(bufferLength: jsonBytes.length + 1); + final objectsArray = value.map((v) => utf8.encode(json.encode(v))); + return writeListBytes>( + objectsArray, 3802, (item) => item.length + 1, (writer, item) { writer.writeUint8(1); - writer.write(jsonBytes); - return writer.toBytes(); - }); - return writeListBytes( - bytesArray, 3802, (item) => item.length, (bd, offset, item) { - item.forEach((i) => bd.setUint8(offset++, i)); + writer.write(item); }); } throw FormatException( @@ -272,28 +264,28 @@ class PostgresBinaryEncoder extends Converter { throw PostgreSQLException('Unsupported datatype'); } -} -Uint8List writeListBytes( - Iterable value, - int type, - int Function(T item) lengthEncoder, - void Function(ByteData bd, int offset, T item) valueEncoder) { - final bd = - ByteData(20 + value.fold(0, (sum, item) => 4 + lengthEncoder(item))); - bd.setInt32(0, 1); // dimension - bd.setInt32(4, 0); // ign - bd.setInt32(8, type); // type - bd.setInt32(12, value.length); // size - bd.setInt32(16, 1); // index - var offset = 20; - for (var i in value) { - final len = lengthEncoder(i); - bd.setInt32(offset, len); // value length - valueEncoder(bd, offset + 4, i); // value - offset += 4 + len; + Uint8List writeListBytes( + Iterable value, + int type, + int Function(T item) lengthEncoder, + void Function(ByteDataWriter writer, T item) valueEncoder) { + final writer = ByteDataWriter(); + + writer.writeInt32(1); // dimension + writer.writeInt32(0); // ign + writer.writeInt32(type); // type + writer.writeInt32(value.length); // size + writer.writeInt32(1); // index + + for (var i in value) { + final len = lengthEncoder(i); + writer.writeInt32(len); + valueEncoder(writer, i); + } + + return writer.toBytes(); } - return bd.buffer.asUint8List(); } class PostgresBinaryDecoder extends Converter { @@ -374,24 +366,24 @@ class PostgresBinaryDecoder extends Converter { } case PostgreSQLDataType.point: - return LatLng(buffer.getFloat64(0), buffer.getFloat64(8)); + return PgPoint(buffer.getFloat64(0), buffer.getFloat64(8)); case PostgreSQLDataType.integerArray: - return readListBytes( - buffer, (offset, _) => buffer.getInt32(offset)); + return readListBytes(value, (reader, _) => reader.readInt32()); case PostgreSQLDataType.textArray: - return readListBytes(buffer, (offset, length) { - return utf8.decode(value.sublist(offset, offset + length)); + return readListBytes(value, (reader, length) { + return utf8.decode(length > 0 ? reader.read(length) : []); }); case PostgreSQLDataType.doubleArray: return readListBytes( - buffer, (offset, _) => buffer.getFloat64(offset)); + value, (reader, _) => reader.readFloat64()); case PostgreSQLDataType.jsonbArray: - return readListBytes(buffer, (offset, length) { - final bytes = value.sublist(offset + 1, offset + length); + return readListBytes(value, (reader, length) { + reader.read(1); + final bytes = reader.read(length - 1); return json.decode(utf8.decode(bytes)); }); } @@ -408,24 +400,26 @@ class PostgresBinaryDecoder extends Converter { } } - List readListBytes( - ByteData buffer, T Function(int offset, int length) valueDecoder) { + List readListBytes(Uint8List data, + T Function(ByteDataReader reader, int length) valueDecoder) { + if (data.length < 16) { + return []; + } + + final reader = ByteDataReader()..add(data); + reader.read(12); // header + final decoded = [].cast(); + final size = reader.readInt32(); - try { - final size = buffer.getInt32(12); - - var offset = 20; - for (var i = 0; i < size; i++) { - final len = buffer.getInt32(offset); - decoded.add(valueDecoder(offset + 4, len)); - offset += 4 + len; - } - - return decoded; - } on RangeError catch (_) { - return decoded; + reader.read(4); // index + + for (var i = 0; i < size; i++) { + final len = reader.readInt32(); + decoded.add(valueDecoder(reader, len)); } + + return decoded; } static final Map typeMap = { diff --git a/lib/src/models.dart b/lib/src/models.dart new file mode 100644 index 00000000..55c6285f --- /dev/null +++ b/lib/src/models.dart @@ -0,0 +1,16 @@ +class PgPoint { + final double latitude; + final double longitude; + const PgPoint(this.latitude, this.longitude); + + @override + bool operator ==(Object other) => + identical(this, other) || + other is PgPoint && + runtimeType == other.runtimeType && + latitude == other.latitude && + longitude == other.longitude; + + @override + int get hashCode => latitude.hashCode ^ longitude.hashCode; +} diff --git a/lib/src/text_codec.dart b/lib/src/text_codec.dart index 852f986a..b07bde50 100644 --- a/lib/src/text_codec.dart +++ b/lib/src/text_codec.dart @@ -32,6 +32,14 @@ class PostgresTextEncoder { return _encodeJSON(value); } + if (value is PgPoint) { + return _encodePoint(value); + } + + if (value is List) { + return _encodeList(value); + } + // TODO: use custom type encoders throw PostgreSQLException("Could not infer type of value '$value'."); @@ -155,4 +163,46 @@ class PostgresTextEncoder { return json.encode(value); } + + String _encodePoint(PgPoint value) { + return '(${_encodeDouble(value.latitude)}, ${_encodeDouble(value.longitude)})'; + } + + String _encodeList(List value) { + if (value.isEmpty) { + return '{}'; + } + + final type = value.fold(value.first.runtimeType, (type, item) { + if (type == item.runtimeType) { + return type; + } else if ((type == int || type == double) && item is num) { + return double; + } else { + return Map; + } + }); + + if (type == int || type == double) { + return '{${value.cast().map((s) => s is double ? _encodeDouble(s) : _encodeNumber(s)).join(',')}}'; + } + + if (type == String) { + return '{${value.cast().map((s) { + final escaped = s.replaceAll(r'\', r'\\').replaceAll('"', r'\"'); + return '"$escaped"'; + }).join(',')}}'; + } + + if (type == Map) { + return '{${value.map((s) { + final escaped = + json.encode(s).replaceAll(r'\', r'\\').replaceAll('"', r'\"'); + + return '"$escaped"'; + }).join(',')}}'; + } + + throw PostgreSQLException("Could not infer array type of value '$value'."); + } } diff --git a/pubspec.yaml b/pubspec.yaml index 79fbf505..48ef18e4 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -9,7 +9,6 @@ environment: dependencies: buffer: ^1.0.6 crypto: ^2.0.0 - latlng: ^0.1.0 dev_dependencies: pedantic: ^1.0.0 diff --git a/test/decode_test.dart b/test/decode_test.dart index 27096fb8..50ca327a 100644 --- a/test/decode_test.dart +++ b/test/decode_test.dart @@ -12,26 +12,30 @@ void main() { CREATE TEMPORARY TABLE t ( i int, s serial, bi bigint, bs bigserial, bl boolean, si smallint, t text, f real, d double precision, dt date, ts timestamp, tsz timestamptz, j jsonb, ba bytea, - u uuid) + u uuid, v varchar, p point, jj json, ia _int4, ta _text, da _float8, ja _jsonb) '''); await connection.execute( - 'INSERT INTO t (i, bi, bl, si, t, f, d, dt, ts, tsz, j, ba, u) ' + 'INSERT INTO t (i, bi, bl, si, t, f, d, dt, ts, tsz, j, ba, u, v, p, jj, ia, ta, da, ja) ' 'VALUES (-2147483648, -9223372036854775808, TRUE, -32768, ' "'string', 10.0, 10.0, '1983-11-06', " "'1983-11-06 06:00:00.000000', '1983-11-06 06:00:00.000000', " - "'{\"key\":\"value\"}', E'\\\\000', '00000000-0000-0000-0000-000000000000')"); + "'{\"key\":\"value\"}', E'\\\\000', '00000000-0000-0000-0000-000000000000', " + "'abcdef', '(0.01, 12.34)', '{\"key\": \"value\"}', '{}', '{}', '{}', '{}')"); await connection.execute( - 'INSERT INTO t (i, bi, bl, si, t, f, d, dt, ts, tsz, j, ba, u) ' + 'INSERT INTO t (i, bi, bl, si, t, f, d, dt, ts, tsz, j, ba, u, v, p, jj, ia, ta, da, ja) ' 'VALUES (2147483647, 9223372036854775807, FALSE, 32767, ' "'a significantly longer string to the point where i doubt this actually matters', " "10.25, 10.125, '2183-11-06', '2183-11-06 00:00:00.111111', " "'2183-11-06 00:00:00.999999', " - "'[{\"key\":1}]', E'\\\\377', 'FFFFFFFF-ffff-ffff-ffff-ffffffffffff')"); + "'[{\"key\":1}]', E'\\\\377', 'FFFFFFFF-ffff-ffff-ffff-ffffffffffff', " + "'01234', '(0.2, 100)', '{}', '{-123, 999}', '{\"a\", \"lorem ipsum\", \"\"}', " + "'{1, 2, 4.5, 1234.5}', '{1, \"\\\"test\\\"\", \"{\\\"a\\\": \\\"b\\\"}\"}')"); await connection.execute( - 'INSERT INTO t (i, bi, bl, si, t, f, d, dt, ts, tsz, j, ba, u) ' - 'VALUES (null, null, null, null, null, null, null, null, null, null, null, null, null)'); + 'INSERT INTO t (i, bi, bl, si, t, f, d, dt, ts, tsz, j, ba, u, v, p, jj, ia, ta, da, ja) ' + 'VALUES (null, null, null, null, null, null, null, null, null, null, null, null, null, ' + 'null, null, null, null, null, null, null )'); }); tearDown(() async { await connection?.close(); @@ -62,6 +66,13 @@ void main() { expect(row1[12], equals({'key': 'value'})); expect(row1[13], equals([0])); expect(row1[14], equals('00000000-0000-0000-0000-000000000000')); + expect(row1[15], equals('abcdef')); + expect(row1[16], equals(PgPoint(0.01, 12.34))); + expect(row1[17], equals({'key': 'value'})); + expect(row1[18], equals([])); + expect(row1[19], equals([])); + expect(row1[20], equals([])); + expect(row1[21], equals([])); // upper bound row expect(row2[0], equals(2147483647)); @@ -88,6 +99,19 @@ void main() { ])); expect(row2[13], equals([255])); expect(row2[14], equals('ffffffff-ffff-ffff-ffff-ffffffffffff')); + expect(row2[15], equals('01234')); + expect(row2[16], equals(PgPoint(0.2, 100))); + expect(row2[17], equals({})); + expect(row2[18], equals([-123, 999])); + expect(row2[19], equals(['a', 'lorem ipsum', ''])); + expect(row2[20], equals([1, 2, 4.5, 1234.5])); + expect( + row2[21], + equals([ + 1, + 'test', + {'a': 'b'} + ])); // all null row expect(row3[0], isNull); @@ -105,6 +129,13 @@ void main() { expect(row3[12], isNull); expect(row3[13], isNull); expect(row3[14], isNull); + expect(row3[15], isNull); + expect(row3[16], isNull); + expect(row3[17], isNull); + expect(row3[18], isNull); + expect(row3[19], isNull); + expect(row3[20], isNull); + expect(row3[21], isNull); }); test('Fetch/insert empty string', () async { diff --git a/test/encoding_test.dart b/test/encoding_test.dart index 076c07fa..bb26403a 100644 --- a/test/encoding_test.dart +++ b/test/encoding_test.dart @@ -1,13 +1,12 @@ import 'dart:async'; import 'dart:convert'; -import 'package:test/test.dart'; - import 'package:postgres/postgres.dart'; import 'package:postgres/src/binary_codec.dart'; import 'package:postgres/src/text_codec.dart'; import 'package:postgres/src/types.dart'; import 'package:postgres/src/utf8_backed_string.dart'; +import 'package:test/test.dart'; PostgreSQLConnection conn; @@ -229,6 +228,123 @@ void main() { expect(e.toString(), contains('Expected: String')); } }); + + test('varchar', () async { + await expectInverse('', PostgreSQLDataType.varChar); + await expectInverse('foo', PostgreSQLDataType.varChar); + await expectInverse('foo\n', PostgreSQLDataType.varChar); + await expectInverse('foo\nbar;s', PostgreSQLDataType.varChar); + try { + await conn.query('INSERT INTO t (v) VALUES (@v:varchar)', + substitutionValues: {'v': 0}); + fail('unreachable'); + } on FormatException catch (e) { + expect(e.toString(), contains('Expected: String')); + } + }); + + test('json', () async { + await expectInverse('string', PostgreSQLDataType.json); + await expectInverse(2, PostgreSQLDataType.json); + await expectInverse(['foo'], PostgreSQLDataType.json); + await expectInverse({ + 'key': 'val', + 'key1': 1, + 'array': ['foo'] + }, PostgreSQLDataType.json); + + try { + await conn.query('INSERT INTO t (v) VALUES (@v:json)', + substitutionValues: {'v': DateTime.now()}); + fail('unreachable'); + } on JsonUnsupportedObjectError catch (_) {} + }); + + test('point', () async { + await expectInverse(PgPoint(0, 0), PostgreSQLDataType.point); + await expectInverse(PgPoint(100, 123.456), PostgreSQLDataType.point); + await expectInverse(PgPoint(0.001, -999), PostgreSQLDataType.point); + + try { + await conn.query('INSERT INTO t (v) VALUES (@v:point)', + substitutionValues: {'v': 'text'}); + fail('unreachable'); + } on FormatException catch (e) { + expect(e.toString(), contains('Expected: PgPoint')); + } + }); + + test('integerArray', () async { + await expectInverse([], PostgreSQLDataType.integerArray); + await expectInverse([-1, 0, 200], PostgreSQLDataType.integerArray); + await expectInverse([-123], PostgreSQLDataType.integerArray); + try { + await conn.query('INSERT INTO t (v) VALUES (@v:_int4)', + substitutionValues: {'v': 'not-list-int'}); + fail('unreachable'); + } on FormatException catch (e) { + expect(e.toString(), contains('Expected: List')); + } + }); + + test('doubleArray', () async { + await expectInverse([], PostgreSQLDataType.doubleArray); + await expectInverse([-123.0, 0.0, 1.0], PostgreSQLDataType.doubleArray); + await expectInverse([0.001, 45.678], PostgreSQLDataType.doubleArray); + try { + await conn.query('INSERT INTO t (v) VALUES (@v:_float8)', + substitutionValues: {'v': 'not-list-double'}); + fail('unreachable'); + } on FormatException catch (e) { + expect(e.toString(), contains('Expected: List')); + } + }); + + test('textArray', () async { + await expectInverse([], PostgreSQLDataType.textArray); + await expectInverse(['', 'foo', 'foo\n'], PostgreSQLDataType.textArray); + await expectInverse(['foo\nbar;s', '"\'"'], PostgreSQLDataType.textArray); + try { + await conn.query('INSERT INTO t (v) VALUES (@v:_text)', + substitutionValues: {'v': 0}); + fail('unreachable'); + } on FormatException catch (e) { + expect(e.toString(), contains('Expected: List')); + } + }); + + test('jsonbArray', () async { + await expectInverse(['string', 2, 0.1], PostgreSQLDataType.jsonbArray); + await expectInverse([ + 1, + {}, + {'a': 'b'} + ], PostgreSQLDataType.jsonbArray); + await expectInverse([ + ['foo'], + [ + 1, + { + 'a': ['b'] + } + ] + ], PostgreSQLDataType.jsonbArray); + await expectInverse([ + { + 'key': 'val', + 'key1': 1, + 'array': ['foo'] + } + ], PostgreSQLDataType.jsonbArray); + + try { + await conn + .query('INSERT INTO t (v) VALUES (@v:_jsonb)', substitutionValues: { + 'v': [DateTime.now()] + }); + fail('unreachable'); + } on JsonUnsupportedObjectError catch (_) {} + }); }); group('Text encoders', () { diff --git a/test/query_test.dart b/test/query_test.dart index cf40dd6c..280d723a 100644 --- a/test/query_test.dart +++ b/test/query_test.dart @@ -1,6 +1,6 @@ import 'package:postgres/postgres.dart'; -import 'package:test/test.dart'; import 'package:postgres/src/types.dart'; +import 'package:test/test.dart'; void main() { group('Successful queries', () { @@ -15,7 +15,8 @@ void main() { '(i int, s serial, bi bigint, ' 'bs bigserial, bl boolean, si smallint, ' 't text, f real, d double precision, ' - 'dt date, ts timestamp, tsz timestamptz, j jsonb, u uuid)'); + 'dt date, ts timestamp, tsz timestamptz, j jsonb, u uuid, ' + 'v varchar, p point, jj json, ia _int4, ta _text, da _float8, ja _jsonb)'); await connection.execute( 'CREATE TEMPORARY TABLE u (i1 int not null, i2 int not null);'); await connection @@ -110,7 +111,7 @@ void main() { test('Query without specifying types', () async { var result = await connection.query( - 'INSERT INTO t (i, bi, bl, si, t, f, d, dt, ts, tsz, j, u) values ' + 'INSERT INTO t (i, bi, bl, si, t, f, d, dt, ts, tsz, j, u, v, p, jj, ia, ta, da, ja) values ' '(${PostgreSQLFormat.id('i')},' '${PostgreSQLFormat.id('bi')},' '${PostgreSQLFormat.id('bl')},' @@ -122,8 +123,15 @@ void main() { '${PostgreSQLFormat.id('ts')},' '${PostgreSQLFormat.id('tsz')},' '${PostgreSQLFormat.id('j')},' - '${PostgreSQLFormat.id('u')}' - ') returning i,s, bi, bs, bl, si, t, f, d, dt, ts, tsz, j, u', + '${PostgreSQLFormat.id('u')},' + '${PostgreSQLFormat.id('v')},' + '${PostgreSQLFormat.id('p')},' + '${PostgreSQLFormat.id('jj')},' + '${PostgreSQLFormat.id('ia')},' + '${PostgreSQLFormat.id('ta')},' + '${PostgreSQLFormat.id('da')},' + '${PostgreSQLFormat.id('ja')}' + ') returning i,s, bi, bs, bl, si, t, f, d, dt, ts, tsz, j, u, v, p, jj, ia, ta, da, ja', substitutionValues: { 'i': 1, 'bi': 2, @@ -136,7 +144,18 @@ void main() { 'ts': DateTime.utc(2000, 2), 'tsz': DateTime.utc(2000, 3), 'j': {'a': 'b'}, - 'u': '01234567-89ab-cdef-0123-0123456789ab' + 'u': '01234567-89ab-cdef-0123-0123456789ab', + 'v': 'abcdef', + 'p': PgPoint(1.0, 0.1), + 'jj': {'k': 'v'}, + 'ia': [1, 2, 3], + 'ta': ['a', 'b"\'\\"'], + 'da': [0.1, 2.3, 1], + 'ja': [ + 1, + 'a"\'\\"', + {'k': 'v"\'\\"'} + ], }); final expectedRow = [ @@ -153,22 +172,33 @@ void main() { DateTime.utc(2000, 2), DateTime.utc(2000, 3), {'a': 'b'}, - '01234567-89ab-cdef-0123-0123456789ab' + '01234567-89ab-cdef-0123-0123456789ab', + 'abcdef', + PgPoint(1.0, 0.1), + {'k': 'v'}, + [1, 2, 3], + ['a', 'b"\'\\"'], + [0.1, 2.3, 1], + [ + 1, + 'a"\'\\"', + {'k': 'v"\'\\"'} + ] ]; - expect(result.columnDescriptions, hasLength(14)); + expect(result.columnDescriptions, hasLength(21)); expect(result.columnDescriptions.first.tableName, 't'); expect(result.columnDescriptions.first.columnName, 'i'); expect(result.columnDescriptions.last.tableName, 't'); - expect(result.columnDescriptions.last.columnName, 'u'); + expect(result.columnDescriptions.last.columnName, 'ja'); expect(result, [expectedRow]); result = await connection.query( - 'select i,s, bi, bs, bl, si, t, f, d, dt, ts, tsz, j, u from t'); + 'select i,s, bi, bs, bl, si, t, f, d, dt, ts, tsz, j, u, v, p, jj, ia, ta, da, ja from t'); expect(result, [expectedRow]); }); test('Query by specifying all types', () async { var result = await connection.query( - 'INSERT INTO t (i, bi, bl, si, t, f, d, dt, ts, tsz, j, u) values ' + 'INSERT INTO t (i, bi, bl, si, t, f, d, dt, ts, tsz, j, u, v, p, jj, ia, ta, da, ja) values ' '(${PostgreSQLFormat.id('i', type: PostgreSQLDataType.integer)},' '${PostgreSQLFormat.id('bi', type: PostgreSQLDataType.bigInteger)},' '${PostgreSQLFormat.id('bl', type: PostgreSQLDataType.boolean)},' @@ -180,8 +210,15 @@ void main() { '${PostgreSQLFormat.id('ts', type: PostgreSQLDataType.timestampWithoutTimezone)},' '${PostgreSQLFormat.id('tsz', type: PostgreSQLDataType.timestampWithTimezone)},' '${PostgreSQLFormat.id('j', type: PostgreSQLDataType.jsonb)},' - '${PostgreSQLFormat.id('u', type: PostgreSQLDataType.uuid)})' - ' returning i,s, bi, bs, bl, si, t, f, d, dt, ts, tsz, j, u', + '${PostgreSQLFormat.id('u', type: PostgreSQLDataType.uuid)},' + '${PostgreSQLFormat.id('v', type: PostgreSQLDataType.varChar)},' + '${PostgreSQLFormat.id('p', type: PostgreSQLDataType.point)},' + '${PostgreSQLFormat.id('jj', type: PostgreSQLDataType.json)},' + '${PostgreSQLFormat.id('ia', type: PostgreSQLDataType.integerArray)},' + '${PostgreSQLFormat.id('ta', type: PostgreSQLDataType.textArray)},' + '${PostgreSQLFormat.id('da', type: PostgreSQLDataType.doubleArray)},' + '${PostgreSQLFormat.id('ja', type: PostgreSQLDataType.jsonbArray)}' + ') returning i,s, bi, bs, bl, si, t, f, d, dt, ts, tsz, j, u, v, p, jj, ia, ta, da, ja', substitutionValues: { 'i': 1, 'bi': 2, @@ -194,7 +231,18 @@ void main() { 'ts': DateTime.utc(2000, 2), 'tsz': DateTime.utc(2000, 3), 'j': {'key': 'value'}, - 'u': '01234567-89ab-cdef-0123-0123456789ab' + 'u': '01234567-89ab-cdef-0123-0123456789ab', + 'v': 'abcdef', + 'p': PgPoint(1.0, 0.1), + 'jj': {'k': 'v'}, + 'ia': [1, 2, 3], + 'ta': ['a', 'b'], + 'da': [0.1, 2.3, 1.0], + 'ja': [ + 1, + 'a', + {'k': 'v'} + ], }); final expectedRow = [ @@ -211,12 +259,23 @@ void main() { DateTime.utc(2000, 2), DateTime.utc(2000, 3), {'key': 'value'}, - '01234567-89ab-cdef-0123-0123456789ab' + '01234567-89ab-cdef-0123-0123456789ab', + 'abcdef', + PgPoint(1.0, 0.1), + {'k': 'v'}, + [1, 2, 3], + ['a', 'b'], + [0.1, 2.3, 1], + [ + 1, + 'a', + {'k': 'v'} + ], ]; expect(result, [expectedRow]); result = await connection.query( - 'select i,s, bi, bs, bl, si, t, f, d, dt, ts, tsz, j, u from t'); + 'select i,s, bi, bs, bl, si, t, f, d, dt, ts, tsz, j, u, v, p, jj, ia, ta, da, ja from t'); expect(result, [expectedRow]); }); From aab4c8d07d67db45291d452e471526e2fe6d3bd4 Mon Sep 17 00:00:00 2001 From: Kilian Schulte Date: Wed, 24 Mar 2021 21:56:13 +0100 Subject: [PATCH 14/16] updated changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc1717fd..9b36ce3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 2.3.1 + +- Added support for types varchar, point, integerArray, doubleArray, textArray and jsonArray. + ## 2.3.0 - Finalized null-safe release. From 052d5e8548578df8ac02a908ede0c274c79744d0 Mon Sep 17 00:00:00 2001 From: Kilian Schulte Date: Wed, 24 Mar 2021 21:59:12 +0100 Subject: [PATCH 15/16] fix docu comment --- lib/src/types.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/types.dart b/lib/src/types.dart index 55f08209..a53ab6bd 100644 --- a/lib/src/types.dart +++ b/lib/src/types.dart @@ -71,7 +71,7 @@ enum PostgreSQLDataType { /// When returned from database, format will be xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx. uuid, - /// Must be a [LatLng] + /// Must be a [PgPoint] point, /// Must be a [List] From 99b5bbe11b72113f516f76664f649579517ea38e Mon Sep 17 00:00:00 2001 From: Kilian Schulte Date: Wed, 24 Mar 2021 22:04:52 +0100 Subject: [PATCH 16/16] fix tests --- test/encoding_test.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/encoding_test.dart b/test/encoding_test.dart index 439765d8..32ff3381 100644 --- a/test/encoding_test.dart +++ b/test/encoding_test.dart @@ -450,7 +450,7 @@ void main() { test('Attempt to infer unknown type throws exception', () { try { - encoder.convert([]); + encoder.convert(Object()); fail('unreachable'); } on PostgreSQLException catch (e) { expect(e.toString(), contains('Could not infer type'));