Request througth gRPC , output in JSON

[andrelec1]

I try to connect to a clickhouse server througth gRPC in Dart ...

For my testing i use a simple request like SELECT * FROM computed_datum LIMIT xxxx; , i increase the xxxx value ....

So i use the executeQueryWithStreamOutput from the gRPC proto ...

i make this function :

  FutureOr<dynamic> _getResultsStream(String query) async {
    QueryInfo queryInfo = this._getQueryInfo(query);
    queryInfo.outputFormat = 'JSON';
    print(query);
    ResponseStream<Result> streamResult = this.client.executeQueryWithStreamOutput(queryInfo);
    int length = 0;
    await for (Result value in streamResult) {
      print(value.output);
      print('----');
      length++;
    }

    return length;
  }

Output for limit 1 look like :

SELECT * FROM computed_datum LIMIT 1;
{
	"meta":
	[
		{
			"name": "EventDateTime",
			"type": "DateTime('UTC')"
		},
		{
			"name": "insert",
			"type": "String"
		},
		{
			"name": "uuid",
			"type": "String"
		},
		{
			"name": "datetime_value",
			"type": "DateTime('UTC')"
		},
		{
			"name": "computed_id",
			"type": "Int32"
		},
		{
			"name": "computed_kind",
			"type": "String"
		},
		{
			"name": "computed_type",
			"type": "String"
		},
		{
			"name": "value",
			"type": "Float64"
		},
		{
			"name": "flags",
			"type": "String"
		}
	],

	"data":
	[
		{
			"EventDateTime": "2020-05-26 07:18:46",
			"insert": "2020-05-26T09:18:46.287Z",
			"uuid": "9970bf0d-4afb-47ea-99b9-f8abbc226ffd",
			"datetime_value": "2000-01-01 00:00:00",
			"computed_id": 11,
			"computed_kind": "testData",
			"computed_type": "extensive.some",
			"value": 145,
			"flags": ""
		}
	],

	"rows": 1,

	"statistics":
	{
		"elapsed": 0.001071028,
		"rows_read": 0,
		"bytes_read": 0
	}
}

----
1

look good ...
if i use LIMIT 250 in my request i have this :
( i strip to top ... )

			"flags": ""
		},
		{
			"EventDateTime": "2020-05-26 07:18:49",
			"insert": "2020-05-26T09:18:48.971Z",
			"uuid": "9ffe1595-984f-4045-a870-7ee628d4aa47",
			"datetime_value": "2000-01-02 17:10:00",
			"computed_id": 11,
			"computed_kind": "testDa
----
ta",
			"computed_type": "extensive.some",
			"value": 680,
			"flags": ""
		},
		{
			"EventDateTime": "2020-05-26 07:18:49",
			"insert": "2020-05-26T09:18:48.971Z",
			"uuid": "058ca1ed-26cb-42ac-9bda-5f2ef103fdc7",
			"datetime_value": "2000-01-02 17:20:00",
			"computed_id": 11,
			"computed_kind": "testData",
			"computed_type": "extensive.some",
			"value": 681,
			"flags": ""
		},
		{
			"EventDateTime": "2020-05-26 07:18:49",
			"insert": "2020-05-26T09:18:48.971Z",
			"uuid": "34c766ec-4ab3-4c47-994d-70810a791c02",
			"datetime_value": "2000-01-02 17:30:00",
			"computed_id": 11,
			"computed_kind": "testData",
			"computed_type": "extensive.some",
			"value": 682,
			"flags": ""
		}
	],

	"rows": 250,

	"statistics":
	{
		"elapsed": 1.353628734,
		"rows_read": 0,
		"bytes_read": 0
	}
}

----
2

so data seem split in middle of nower but that work ...
But if my request return more than 270 elements ... i only get the first element of the stream :/

I change the outputFormat to CSV and i can see all the data ...

[andrelec1]

the executequery function return :

exception: {
  code: 241
  name: DB::Exception
  displayText: DB::Exception: Memory limit (for query) exceeded: would use 18.00 GiB (attempt to allocate chunk of 12884901888 bytes), maximum: 9.31 GiB
  stackTrace: 0. DB::Exception::Exception(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, int, bool) @ 0x88b64ca in /usr/bin/clickhouse
1. DB::Exception::Exception<char const*, char const*, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >(int, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, char const*&&, char const*&&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >&&, long&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >&&) @ 0x88caef9 in /usr/bin/clickhouse
2. MemoryTracker::alloc(long) @ 0x88ca86d in /usr/bin/clickhouse
3. MemoryTracker::alloc(long) @ 0x88ca5d4 in /usr/bin/clickhouse
4. operator new(unsigned long) @ 0x88b1be8 in /usr/bin/clickhouse
5. std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >::append(unsigned long, char) @ 0x1426afb6 in ?
6. DB::WriteBufferFromVector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >::nextImpl() @ 0x88bc083 in /usr/bin/clickhouse
7. DB::WriteBufferValidUTF8::nextImpl() @ 0xd6709c3 in /usr/bin/clickhouse
8. DB::WriteBuffer::write(char const*, unsigned long) @ 0x88bc336 in /usr/bin/clickhouse
9. DB::JSONRowOutputFormat::writeField(DB::IColumn const&, DB::ISerialization const&, unsigned long) @ 0x1015ea76 in /usr/bin/clickhouse
10. DB::IRowOutputFormat::write(std::__1::vector<COW<DB::IColumn>::immutable_ptr<DB::IColumn>, std::__1::allocator<COW<DB::IColumn>::immutable_ptr<DB::IColumn> > > const&, unsigned long) @ 0x101298b1 in /usr/bin/clickhouse
11. DB::IRowOutputFormat::consume(DB::Chunk) @ 0x10129352 in /usr/bin/clickhouse
12. DB::IOutputFormat::write(DB::Block const&) @ 0x100f3023 in /usr/bin/clickhouse
13. DB::MaterializingBlockOutputStream::write(DB::Block const&) @ 0x100a0e22 in /usr/bin/clickhouse
14. ? @ 0xffd8289 in /usr/bin/clickhouse
15. ? @ 0xffd2f0f in /usr/bin/clickhouse
16. ThreadPoolImpl<std::__1::thread>::worker(std::__1::__list_iterator<std::__1::thread, void*>) @ 0x88f65af in /usr/bin/clickhouse
17. ? @ 0x88fa043 in /usr/bin/clickhouse
18. start_thread @ 0x9609 in /lib/x86_64-linux-gnu/libpthread-2.31.so
19. /build/glibc-YbNSs7/glibc-2.31/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:97: clone @ 0x122293 in /usr/lib/debug/lib/x86_64-linux-gnu/libc-2.31.so

}