From b7e6d6f964a030695ef9cd16c4156e3d3cba25c6 Mon Sep 17 00:00:00 2001 From: Evgeniy Andreev Date: Thu, 25 Jul 2013 01:19:50 +0400 Subject: [PATCH] LineReader and CircularBuffer are modified --- .../undocumented/libshogun/io_linereader.cpp | 19 +- .../libshogun/library_circularbuffer.cpp | 3 +- src/shogun/io/LineReader.cpp | 107 +++++++---- src/shogun/io/LineReader.h | 65 +++---- src/shogun/lib/CircularBuffer.cpp | 169 +++++++++++++----- src/shogun/lib/CircularBuffer.h | 50 ++++-- tests/unit/io/LineReader_unittest.cc | 27 ++- 7 files changed, 289 insertions(+), 151 deletions(-) diff --git a/examples/undocumented/libshogun/io_linereader.cpp b/examples/undocumented/libshogun/io_linereader.cpp index 07e7c189127..c3ab05ecd1b 100644 --- a/examples/undocumented/libshogun/io_linereader.cpp +++ b/examples/undocumented/libshogun/io_linereader.cpp @@ -1,6 +1,6 @@ #include - #include +#include #include #include @@ -12,13 +12,17 @@ int main(int argc, char** argv) init_shogun_with_defaults(); FILE* fin=fopen("io_linereader.cpp", "r"); - CLineReader* reader=new CLineReader(fin); + + CDelimiterTokenizer* tokenizer=new CDelimiterTokenizer(); + tokenizer->delimiters['\n']=1; + + CLineReader* reader=new CLineReader(fin, tokenizer); int lines_count=0; - SGVector tmp_string; - while (reader->has_next_line()) + SGVector tmp_string(true); + while (reader->has_next()) { - tmp_string=reader->get_next_line(); + tmp_string=reader->read_line(); SG_SPRINT("%d %d ", lines_count, tmp_string.vlen); for (int i=0; i(); + SG_UNREF(tokenizer); + SG_UNREF(reader); + fclose(fin); exit_shogun(); diff --git a/examples/undocumented/libshogun/library_circularbuffer.cpp b/examples/undocumented/libshogun/library_circularbuffer.cpp index ac1c7043193..583b43039a9 100644 --- a/examples/undocumented/libshogun/library_circularbuffer.cpp +++ b/examples/undocumented/libshogun/library_circularbuffer.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include @@ -13,7 +14,7 @@ int main(int argc, char** argv) { init_shogun_with_defaults(); - SGVector test_string("all your bayes are belong to us! ", 33, false); + SGVector test_string(const_cast("all your bayes are belong to us! "), 33, false); CCircularBuffer* buffer=new CCircularBuffer(max_line_length); CDelimiterTokenizer* tokenizer=new CDelimiterTokenizer(); diff --git a/src/shogun/io/LineReader.cpp b/src/shogun/io/LineReader.cpp index 06396180b0d..4ef66260309 100644 --- a/src/shogun/io/LineReader.cpp +++ b/src/shogun/io/LineReader.cpp @@ -7,94 +7,125 @@ * Written (W) 2013 Evgeniy Andreev (gsomix) */ -#include #include using namespace shogun; CLineReader::CLineReader() - : m_stream(NULL), m_max_line_length(0), m_next_line_length(-1) { - m_buffer=new CCircularBuffer(0); - m_tokenizer=NULL; + init(); + + m_buffer=new CCircularBuffer(); } -CLineReader::CLineReader(FILE* stream, char delimiter) - : m_stream(stream), m_max_line_length(10*1024*1024), m_next_line_length(-1) +CLineReader::CLineReader(FILE* stream, CTokenizer* tokenizer) { - m_buffer=new CCircularBuffer(m_max_line_length); - m_tokenizer=new CDelimiterTokenizer(); - m_tokenizer->delimiters[delimiter]=1; - m_buffer->set_tokenizer(m_tokenizer); + init(); + + m_stream=stream; + m_max_token_length=10*1024*1024; + + m_buffer=new CCircularBuffer(m_max_token_length); + m_buffer->set_tokenizer(tokenizer); + + m_tokenizer=tokenizer; } -CLineReader::CLineReader(int32_t max_line_length, FILE* stream, char delimiter) - : m_stream(stream), m_max_line_length(max_line_length), m_next_line_length(-1) +CLineReader::CLineReader(int32_t max_token_length, FILE* stream, CTokenizer* tokenizer) { - m_buffer=new CCircularBuffer(m_max_line_length); - m_tokenizer=new CDelimiterTokenizer(); - m_tokenizer->delimiters[delimiter]=1; - m_buffer->set_tokenizer(m_tokenizer); + init(); + + m_stream=stream; + m_max_token_length=max_token_length; + + m_buffer=new CCircularBuffer(m_max_token_length); + m_buffer->set_tokenizer(tokenizer); + + m_tokenizer=tokenizer; } CLineReader::~CLineReader() { - SG_UNREF(m_tokenizer); SG_UNREF(m_buffer); } -bool CLineReader::has_next_line() +bool CLineReader::has_next() { - if (m_stream==NULL || m_max_line_length==0) + if (m_stream==NULL || m_max_token_length==0 || m_tokenizer==NULL) { - SG_ERROR("Class is not initialized"); + SG_ERROR("CLineReader::has_next():: Class is not initialized\n"); return false; } if (ferror(m_stream)) { - SG_ERROR("Error reading file"); + SG_ERROR("CLineReader::has_next():: Error reading file\n"); return false; } - if (feof(m_stream) && m_buffer->num_bytes_contained()<=0) + if (feof(m_stream) && (m_buffer->num_bytes_contained()<=0 || !m_buffer->has_next())) + { return false; // nothing to read + } return true; } -SGVector CLineReader::get_next_line() +void CLineReader::skip_line() +{ + int32_t bytes_to_skip=0; + m_next_token_length=read(bytes_to_skip); + if (m_next_token_length==-1) + return; + else + m_buffer->skip_characters(bytes_to_skip); +} + +SGVector CLineReader::read_line() { SGVector line; - m_next_line_length=read_line(); - if (m_next_line_length==-1) + int32_t bytes_to_skip=0; + m_next_token_length=read(bytes_to_skip); + if (m_next_token_length==-1) line=SGVector(); else - line=copy_line(m_next_line_length); + { + m_buffer->skip_characters(bytes_to_skip); + line=read_token(m_next_token_length-bytes_to_skip); + } return line; } -void CLineReader::set_delimiter(char delimiter) +void CLineReader::set_tokenizer(CTokenizer* tokenizer) { - m_tokenizer->delimiters[delimiter]=1; + m_buffer->set_tokenizer(tokenizer); + m_tokenizer=tokenizer; } -void CLineReader::clear_delimiters() +void CLineReader::init() { - m_tokenizer->clear_delimiters(); + m_buffer=NULL; + m_tokenizer=NULL; + m_stream=NULL; + + m_max_token_length=0; + m_next_token_length=-1; } -int32_t CLineReader::read_line() +int32_t CLineReader::read(int32_t& bytes_to_skip) { int32_t line_end=0; - int32_t bytes_to_skip=0; int32_t bytes_to_read=0; + int32_t temp_bytes_to_skip=0; while (1) { - line_end+=m_buffer->next_token_idx(bytes_to_skip)-bytes_to_skip; + if (bytes_to_skip==line_end) + line_end=m_buffer->next_token_idx(bytes_to_skip); + else + line_end=m_buffer->next_token_idx(temp_bytes_to_skip); if (m_buffer->num_bytes_contained()!=0 && line_endnum_bytes_contained()) return line_end; @@ -104,10 +135,10 @@ int32_t CLineReader::read_line() // if there is no delimiter in buffer // try get more data from stream // and write it into buffer - if (m_buffer->available() < m_max_line_length) + if (m_buffer->available() < m_max_token_length) bytes_to_read=m_buffer->available(); else - bytes_to_read=m_max_line_length; + bytes_to_read=m_max_token_length; if (feof(m_stream)) return line_end; @@ -116,13 +147,13 @@ int32_t CLineReader::read_line() if (ferror(m_stream)) { - SG_ERROR("Error reading file"); + SG_ERROR("CLineReader::read(int32_t&):: Error reading file\n"); return -1; } } } -SGVector CLineReader::copy_line(int32_t line_len) +SGVector CLineReader::read_token(int32_t line_len) { SGVector line; @@ -131,7 +162,5 @@ SGVector CLineReader::copy_line(int32_t line_len) else line=m_buffer->pop(line_len); - m_buffer->skip_characters(1); - return line; } diff --git a/src/shogun/io/LineReader.h b/src/shogun/io/LineReader.h index 7e872ae3c23..117c487a902 100644 --- a/src/shogun/io/LineReader.h +++ b/src/shogun/io/LineReader.h @@ -10,14 +10,13 @@ #ifndef __LINE_READER_H__ #define __LINE_READER_H__ -#include #include +#include #include namespace shogun { -/** @brief Class for buffered reading lines from a ascii file - */ +/** @brief Class for buffered reading from a ascii file */ class CLineReader : public CSGObject { public: @@ -28,7 +27,7 @@ class CLineReader : public CSGObject * * @param stream readable stream */ - CLineReader(FILE* stream, char delimiter='\n'); + CLineReader(FILE* stream, CTokenizer* tokenizer); /** create object associated with the stream to read * and specify maximum length of a string that can be read @@ -36,69 +35,59 @@ class CLineReader : public CSGObject * @param stream readable stream * @param buffer_size size of internal buffer */ - CLineReader(int32_t max_string_length, FILE* stream, char delimiter='\n'); + CLineReader(int32_t max_string_length, FILE* stream, CTokenizer* tokenizer); /** deconstructor */ - ~CLineReader(); + virtual ~CLineReader(); /** check for next line in the stream * * @return true if there is next line, false - otherwise */ - bool has_next_line(); + virtual bool has_next(); - /** get line from the buffer into SGVector - * there is no warranty that after reading the caret will - * set at the beginning of a new line - * - * @return SGVector that contains line - */ - SGVector get_next_line(); + /** skip next line */ + virtual void skip_line(); - /** set delimiter active for tokenizing + /** read string */ + virtual SGVector read_line(); + + /** set tokenizer * - * @param delimiter delimiter + * @param tokenizer tokenizer */ - void set_delimiter(char delimiter); - - /** clear all delimiters */ - void clear_delimiters(); + void set_tokenizer(CTokenizer* tokenizer); /** @return object name */ virtual const char* get_name() const { return "LineReader"; } private: - /** read one line into buffer - * - * @return length of line - */ - int32_t read_line(); + /** class initialization */ + void init(); - /** copy chars into SGVector from source - * - * @param line destination string - * @param line_len length of line in source - * @param source source array of chars - */ - SGVector copy_line(int32_t line_len); + /** read file into memory */ + int32_t read(int32_t& bytes_to_skip); + + /** read token from internal buffer */ + SGVector read_token(int32_t line_len); private: /** internal buffer for searching */ CCircularBuffer* m_buffer; - /** tokenizer */ - CDelimiterTokenizer* m_tokenizer; + /** */ + CTokenizer* m_tokenizer; /** readable stream */ - FILE* m_stream; + FILE* m_stream; /** maximum length of a line that can be read */ - int32_t m_max_line_length; + int32_t m_max_token_length; /** length of next line in the buffer */ - int32_t m_next_line_length; + int32_t m_next_token_length; }; } -#endif /* __LINE_READER_H__ */ +#endif /* __FILE_READER_H__ */ diff --git a/src/shogun/lib/CircularBuffer.cpp b/src/shogun/lib/CircularBuffer.cpp index 37406fe6e30..9bab63b94b0 100644 --- a/src/shogun/lib/CircularBuffer.cpp +++ b/src/shogun/lib/CircularBuffer.cpp @@ -15,23 +15,29 @@ using namespace shogun; CCircularBuffer::CCircularBuffer() - : m_buffer(), m_tokenizer(NULL), - m_begin_pos(NULL), m_end_pos(NULL), m_finder_pos(NULL), - m_bytes_available(0), m_bytes_count(0) { - + init(); } -CCircularBuffer::CCircularBuffer(int32_t buffer_size) - : m_buffer(buffer_size), m_tokenizer(NULL), - m_begin_pos(m_buffer.vector), m_end_pos(m_begin_pos), m_finder_pos(m_begin_pos), - m_bytes_available(m_buffer.vlen), m_bytes_count(0) - +CCircularBuffer::CCircularBuffer(int32_t buffer_size) { + init(); + + m_buffer=SGVector(buffer_size); + m_buffer_end=m_buffer.vector+m_buffer.vlen; + + m_begin_pos=m_buffer.vector; + m_end_pos=m_begin_pos; + + m_bytes_available=m_buffer.vlen; +} +CCircularBuffer::~CCircularBuffer() +{ + m_buffer=SGVector(); } -void CCircularBuffer::set_tokenizer(CDelimiterTokenizer* tokenizer) +void CCircularBuffer::set_tokenizer(CTokenizer* tokenizer) { m_tokenizer=tokenizer; } @@ -40,7 +46,7 @@ int32_t CCircularBuffer::push(SGVector source) { if (source.vector==NULL || source.vlen==0) { - SG_ERROR("Invalid parameters! Source shouldn't be NULL or zero sized"); + SG_ERROR("CCircularBuffer::push(SGVector):: Invalid parameters! Source shouldn't be NULL or zero sized\n"); return -1; } @@ -84,7 +90,7 @@ int32_t CCircularBuffer::push(FILE* source, int32_t source_size) { if (source==NULL || source_size==0) { - SG_ERROR("Invalid parameters! Source shouldn't be NULL or zero sized"); + SG_ERROR("CCircularBuffer::push(FILE*, int32_t):: Invalid parameters! Source shouldn't be NULL or zero sized\n"); return -1; } @@ -163,26 +169,69 @@ SGVector CCircularBuffer::pop(int32_t num_bytes) return result; } +bool CCircularBuffer::has_next() +{ + if (m_tokenizer==NULL) + { + SG_ERROR("CCircularBuffer::has_next():: Tokenizer is not initialized\n"); + return false; + } + + if (m_bytes_count==0) + return false; + + int32_t tail_length=m_end_pos-m_buffer.vector; + int32_t head_length=m_buffer_end-m_begin_pos; + + // determine position of finder pointer in memory block + if (m_last_idx=m_begin_pos && m_bytes_available!=0) + { + return has_next_locally(m_begin_pos+m_last_idx, m_end_pos); + } + else + { + bool temp=false; + temp=has_next_locally(m_begin_pos+m_last_idx, m_buffer_end); + + if (temp) + return temp; + + return has_next_locally(m_buffer.vector+m_last_idx-head_length, m_end_pos); + } + } + else + { + return has_next_locally(m_buffer.vector+m_last_idx-head_length, m_end_pos); + } + + return false; +} + index_t CCircularBuffer::next_token_idx(index_t &start) { index_t end; if (m_tokenizer==NULL) { - SG_ERROR("Tokenizer is not initialized"); + SG_ERROR("CCircularBuffer::next_token_idx(index_t&):: Tokenizer is not initialized\n"); return 0; } if (m_bytes_count==0) return m_bytes_count; + int32_t tail_length=m_end_pos-m_buffer.vector; + int32_t head_length=m_buffer_end-m_begin_pos; + // determine position of finder pointer in memory block - if (m_finder_pos>=m_begin_pos) + if (m_last_idx=m_begin_pos && m_bytes_available!=0) { - end=next_token_idx_locally(start, m_finder_pos, m_end_pos); - if (m_finder_pos<=m_end_pos) + end=next_token_idx_locally(start, m_begin_pos+m_last_idx, m_end_pos); + if (end<=m_bytes_count) return end; } else @@ -190,27 +239,25 @@ index_t CCircularBuffer::next_token_idx(index_t &start) index_t temp_start; // in this case we should find first at end of memory block - int32_t bytes_to_memory_end=m_buffer.vlen-(m_begin_pos-m_buffer.vector); - end=next_token_idx_locally(start, m_finder_pos, m_begin_pos+bytes_to_memory_end); + end=next_token_idx_locally(start, m_begin_pos+m_last_idx, m_buffer_end); - if (m_finder_pos>=m_begin_pos) + if (end=head_length) + start=temp_start; + return end; } } else - { - end=next_token_idx_locally(start, m_finder_pos, m_end_pos); - if (m_finder_pos<=m_end_pos) - { - int32_t bytes_to_memory_end=m_buffer.vlen-(m_begin_pos-m_buffer.vector); - start+=bytes_to_memory_end; - return end+bytes_to_memory_end; - } + { + end=next_token_idx_locally(start, m_buffer.vector+m_last_idx-head_length, m_end_pos); + if (end-head_length<=tail_length) + return end; } start=0; @@ -220,7 +267,10 @@ index_t CCircularBuffer::next_token_idx(index_t &start) void CCircularBuffer::skip_characters(int32_t num_chars) { move_pointer(&m_begin_pos, m_begin_pos+num_chars); - m_finder_pos=m_begin_pos; + + m_last_idx-=num_chars; + if (m_last_idx<0) + m_last_idx=0; m_bytes_available+=num_chars; m_bytes_count-=num_chars; @@ -230,15 +280,23 @@ void CCircularBuffer::clear() { m_begin_pos=m_buffer.vector; m_end_pos=m_begin_pos; - m_finder_pos=m_begin_pos; + m_last_idx=0; m_bytes_available=m_buffer.vlen; m_bytes_count=0; } -void CCircularBuffer::debug_print() +void CCircularBuffer::init() { - SGVector::display_vector(m_buffer); + m_buffer_end=m_buffer.vector+m_buffer.vlen; + m_tokenizer=NULL; + + m_begin_pos=NULL; + m_end_pos=NULL; + + m_last_idx=0; + m_bytes_available=0; + m_bytes_count=0; } int32_t CCircularBuffer::append_chunk(const char* source, int32_t source_size, @@ -246,7 +304,8 @@ int32_t CCircularBuffer::append_chunk(const char* source, int32_t source_size, { if (source==NULL || source_size==0) { - SG_ERROR("Invalid parameters! Source shouldn't be NULL or zero sized"); + SG_ERROR("CCircularBuffer::append_chunk(const char*, int32_t, bool):: Invalid parameters!\ + Source shouldn't be NULL or zero sized\n"); return -1; } @@ -282,7 +341,7 @@ void CCircularBuffer::detach_chunk(char** dest, int32_t* dest_size, int32_t dest { if (dest==NULL || dest_size==NULL) { - SG_ERROR("Invalid parameters! Pointers are NULL"); + SG_ERROR("CCircularBuffer::detach_chunk(...):: Invalid parameters! Pointers are NULL\n"); return; } @@ -303,31 +362,47 @@ void CCircularBuffer::detach_chunk(char** dest, int32_t* dest_size, int32_t dest memcpy(*dest+dest_offset, m_begin_pos, num_bytes); move_pointer(&m_begin_pos, m_begin_pos+num_bytes); - m_finder_pos=m_begin_pos; + + m_last_idx-=num_bytes; + if (m_last_idx<0) + m_last_idx=0; m_bytes_available+=num_bytes; m_bytes_count-=num_bytes; } -index_t CCircularBuffer::next_token_idx_locally(index_t &start, char* part_begin, char* part_end) +bool CCircularBuffer::has_next_locally(char* part_begin, char* part_end) { - index_t end=0; int32_t num_bytes_to_search=part_end-part_begin; SGVector buffer_part(part_begin, num_bytes_to_search, false); m_tokenizer->set_text(buffer_part); - if (m_tokenizer->has_next()) + + return m_tokenizer->has_next(); +} + +index_t CCircularBuffer::next_token_idx_locally(index_t &start, char* part_begin, char* part_end) +{ + index_t end=0; + int32_t num_bytes_to_search=part_end-part_begin; + if (num_bytes_to_search<=0) { - end=m_tokenizer->next_token_idx(start); - move_pointer(&m_finder_pos, m_finder_pos+end); - return end; + start=0; + return m_last_idx; } + + SGVector buffer_part(part_begin, num_bytes_to_search, false); + m_tokenizer->set_text(buffer_part); + + end=m_tokenizer->next_token_idx(start); + + start+=m_last_idx; + m_last_idx+=end; + + if (end==num_bytes_to_search) + return m_last_idx; else - { - start=num_bytes_to_search; - move_pointer(&m_finder_pos, part_end); - return start; - } + return m_last_idx++; } void CCircularBuffer::move_pointer(char** pointer, char* new_position) diff --git a/src/shogun/lib/CircularBuffer.h b/src/shogun/lib/CircularBuffer.h index eb631205aad..4abffe49944 100644 --- a/src/shogun/lib/CircularBuffer.h +++ b/src/shogun/lib/CircularBuffer.h @@ -12,7 +12,7 @@ #include #include -#include +#include namespace shogun { @@ -38,8 +38,14 @@ class CCircularBuffer : public CSGObject */ CCircularBuffer(int32_t buffer_size); - /** */ - void set_tokenizer(CDelimiterTokenizer* tokenizer); + /** destructor */ + ~CCircularBuffer(); + + /** set tokenizer + * + * @param tokenizer tokenizer + */ + void set_tokenizer(CTokenizer* tokenizer); /** push data into buffer from memory block * @@ -64,11 +70,19 @@ class CCircularBuffer : public CSGObject */ SGVector pop(int32_t num_chars); - /** search next character in buffer from last found + /** returns true or false based on whether + * there exists another token in the text * - * @param value character to find - * @return number of bytes from buffer's begin to character - */ + * @return if another token exists + */ + bool has_next(); + + /** method that returns the indices, start and end, of + * the next token in buffer + * + * @param start token's starting index + * @return token's ending index (inclusive) + */ index_t next_token_idx(index_t &start); /** remove characters from buffer @@ -93,13 +107,13 @@ class CCircularBuffer : public CSGObject /** clear buffer */ void clear(); - /** */ - void debug_print(); - /** @return object name */ virtual const char* get_name() const { return "CircularBuffer"; } private: + /** class initialization */ + void init(); + /** append memory block to buffer */ int32_t append_chunk(const char* source, int32_t source_size, bool from_buffer_begin); @@ -112,7 +126,14 @@ class CCircularBuffer : public CSGObject void detach_chunk(char** dest, int32_t* dest_size, int32_t dest_offset, int32_t num_bytes, bool from_buffer_begin); - /** find character in buffer, like memchr */ + /** returns true or false based on whether + * there exists another token in the text + */ + bool has_next_locally(char* begin, char* end); + + /** method that returns the indices, start and end, of + * the next token in part of buffer + */ index_t next_token_idx_locally(index_t &start, char* begin, char* end); /** move pointer to another position */ @@ -122,8 +143,11 @@ class CCircularBuffer : public CSGObject /** internal memory */ SGVector m_buffer; + /** pointer to end of buffer's memory */ + char* m_buffer_end; + /** tokenizer */ - CDelimiterTokenizer* m_tokenizer; + CTokenizer* m_tokenizer; /** begin of buffer */ char* m_begin_pos; @@ -132,7 +156,7 @@ class CCircularBuffer : public CSGObject char* m_end_pos; /** position at which the search starts */ - char* m_finder_pos; + index_t m_last_idx; /** number of free bytes */ int32_t m_bytes_available; diff --git a/tests/unit/io/LineReader_unittest.cc b/tests/unit/io/LineReader_unittest.cc index 52cd411f853..2cc36d94d4f 100644 --- a/tests/unit/io/LineReader_unittest.cc +++ b/tests/unit/io/LineReader_unittest.cc @@ -9,10 +9,11 @@ #include #include +#include #include -#include #include +#include using namespace shogun; @@ -22,11 +23,17 @@ const int max_num_lines = 100; TEST(LineReaderTest, constructor) { CLineReader* reader; + CDelimiterTokenizer* tokenizer; FILE* fin=fopen("io/LineReader_unittest.cc", "r"); - reader=new CLineReader(fin); - EXPECT_TRUE(reader->has_next_line()); + + tokenizer=new CDelimiterTokenizer(); + tokenizer->delimiters['\n']=1; + + reader=new CLineReader(fin, tokenizer); + EXPECT_TRUE(reader->has_next()); SG_UNREF(reader); + SG_UNREF(tokenizer); fclose(fin); } @@ -39,16 +46,21 @@ TEST(LineReaderTest, read_yourself) int lines_count; CLineReader* reader; + CDelimiterTokenizer* tokenizer; FILE* fin=fopen("io/LineReader_unittest.cc", "r"); - reader=new CLineReader(max_line_length, fin); - EXPECT_TRUE(reader->has_next_line()); + + tokenizer=new CDelimiterTokenizer(); + tokenizer->delimiters['\n']=1; + + reader=new CLineReader(max_line_length, fin, tokenizer); + EXPECT_TRUE(reader->has_next()); // read all strings from source code using LineReader lines_count=0; - while (reader->has_next_line()) + while (reader->has_next()) { - strings[lines_count]=reader->get_next_line(); + strings[lines_count]=reader->read_line(); lines_count++; } @@ -66,6 +78,7 @@ TEST(LineReaderTest, read_yourself) lines_count++; } SG_UNREF(reader); + SG_UNREF(tokenizer); fclose(fin); }