From d3e1e03d5bc1cbc143521368b201de6af7d73f0a Mon Sep 17 00:00:00 2001 From: Jeffrey Lovitz Date: Wed, 12 May 2021 17:30:49 -0400 Subject: [PATCH] Emit commands when max buffer size is exceeded by a single token --- redisgraph_bulk_loader/label.py | 3 ++- redisgraph_bulk_loader/relation_type.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/redisgraph_bulk_loader/label.py b/redisgraph_bulk_loader/label.py index 80013b6..1dfc0f4 100644 --- a/redisgraph_bulk_loader/label.py +++ b/redisgraph_bulk_loader/label.py @@ -72,7 +72,8 @@ def process_entities(self): # If the addition of this entity will make the binary token grow too large, # send the buffer now. # TODO how much of this can be made uniform w/ relations and moved to Querybuffer? - if self.binary_size + row_binary_len > self.config.max_token_size: + added_size = self.binary_size + row_binary_len + if added_size >= self.config.max_token_size or self.query_buffer.buffer_size + added_size >= self.config.max_buffer_size: self.query_buffer.labels.append(self.to_binary()) self.query_buffer.send_buffer() self.reset_partial_binary() diff --git a/redisgraph_bulk_loader/relation_type.py b/redisgraph_bulk_loader/relation_type.py index cdb0205..83f9718 100644 --- a/redisgraph_bulk_loader/relation_type.py +++ b/redisgraph_bulk_loader/relation_type.py @@ -74,7 +74,8 @@ def process_entities(self): row_binary_len = len(row_binary) # If the addition of this entity will make the binary token grow too large, # send the buffer now. - if self.binary_size + row_binary_len > self.config.max_token_size: + added_size = self.binary_size + row_binary_len + if added_size >= self.config.max_token_size or self.query_buffer.buffer_size + added_size >= self.config.max_buffer_size: self.query_buffer.reltypes.append(self.to_binary()) self.query_buffer.send_buffer() self.reset_partial_binary()