coverage wip

samples/quickstart.cpp

@@ -3902,12 +3902,9 @@ struct GlobalAllocatorExample
             ptr = (void*)(((char*)ptr) + corr);
             corr_size += corr;
         }
-        // check
-        if(C4_UNLIKELY(alloc_size + corr_size > memory_pool.size()))
-        {
-            std::cerr << "out of memory! requested=" << alloc_size << "+" << corr_size << " vs " << memory_pool.size() << " available" << std::endl;
-            std::abort();
-        }
+        C4_CHECK_MSG(alloc_size + corr_size <= memory_pool.size(),
+                     "out of memory! requested=%zu+%zu available=%zu\n",
+                     alloc_size, corr_size, memory_pool.size());
         return ptr;
     }
 

src/c4/yml/common.hpp

@@ -329,6 +329,7 @@ C4_NO_INLINE void _parse_dump(DumpFn dumpfn, csubstr fmt, Args&& ...args)
         if(C4_UNLIKELY(results.bufsize > sizeof(writebuf)))
         {
             results = format_dump_resume(dumpfn, results, writebuf, fmt, std::forward<Args>(args)...);
+            C4_CHECK(results.bufsize <= sizeof(writebuf));
         }
     }
 }

src/c4/yml/filter_processor.hpp

@@ -112,7 +112,7 @@ struct FilterProcessorSrcDst
 // filter in place
 
 // debugging scaffold
-#if 0
+#if 1
 #define _c4dbgip(...) _c4dbgpf(__VA_ARGS__);
 #else
 #define _c4dbgip(...)
@@ -124,14 +124,14 @@ struct FilterProcessorInplace
     size_t wcap; ///< write capacity - the capacity of the subject string's buffer
     size_t rpos; ///< read position
     size_t wpos; ///< write position
-    bool unfiltered_chars; ///< whether there are characters left unfiltered (eg because the string was too small)
+    size_t unwritten_excess; ///< number of characters that were not added to wpos from lack of capacity
 
     C4_ALWAYS_INLINE FilterProcessorInplace(substr src_, size_t wcap_) noexcept
         : src(src_)
         , wcap(wcap_)
         , rpos(0)
         , wpos(0)
-        , unfiltered_chars(false)
+        , unwritten_excess(0)
     {
         RYML_ASSERT(wcap >= src.len);
     }
@@ -145,10 +145,10 @@ struct FilterProcessorInplace
 
     C4_ALWAYS_INLINE FilterResult result() const noexcept
     {
-        _c4dbgip("inplace: wpos={} wcap={} unfiltered={}", this->wpos, this->wcap, this->unfiltered_chars);
+        _c4dbgip("inplace: wpos={} wcap={} unfiltered={}", this->wpos, this->wcap, this->unwritten_excess);
         FilterResult ret;
-        ret.str.str = (wpos <= wcap && !unfiltered_chars) ? src.str : nullptr;
-        ret.str.len = wpos;
+        ret.str.str = (wpos <= wcap && !unwritten_excess) ? src.str : nullptr;
+        ret.str.len = wpos + unwritten_excess;
         return ret;
     }
     C4_ALWAYS_INLINE csubstr sofar() const noexcept { return csubstr(src.str, wpos <= wcap ? wpos : wcap); }
@@ -177,8 +177,8 @@ struct FilterProcessorInplace
         }
         else
         {
-            _c4dbgip("inplace: set unfiltered {}->1!", unfiltered_chars);
-            unfiltered_chars = true;
+            _c4dbgip("inplace: add unwritten {}->{}!", unwritten_excess, unwritten_excess+1u);
+            ++unwritten_excess;
         }
         ++wpos;
     }
@@ -192,8 +192,8 @@ struct FilterProcessorInplace
         }
         else
         {
-            _c4dbgip("inplace: set unfiltered {}->1!", unfiltered_chars);
-            unfiltered_chars = true;
+            _c4dbgip("inplace: add unwritten {}->{}!", unwritten_excess, unwritten_excess+1u);
+            unwritten_excess += num;
         }
         wpos += num;
     }
@@ -208,8 +208,8 @@ struct FilterProcessorInplace
         }
         else
         {
-            _c4dbgip("inplace: set unfiltered {}->1 (wpos={}!=rpos={})={}  (wpos={}<wcap={})!", unfiltered_chars, wpos, rpos, wpos!=rpos, wpos, wcap, wpos<wcap);
-            unfiltered_chars = true;
+            _c4dbgip("inplace: add unwritten {}->{} (wpos={}!=rpos={})={}  (wpos={}<wcap={})!", unwritten_excess, unwritten_excess+1, wpos, rpos, wpos!=rpos, wpos, wcap, wpos<wcap);
+            ++unwritten_excess;
         }
         ++wpos;
         ++rpos;
@@ -230,8 +230,8 @@ struct FilterProcessorInplace
         }
         else
         {
-            _c4dbgip("inplace: set unfiltered {}->1 (wpos={}<rpos={})={}  (wpos={}<wcap={})!", unfiltered_chars, wpos, rpos, wpos<rpos, wpos, wcap, wpos<wcap);
-            unfiltered_chars = true;
+            _c4dbgip("inplace: add unwritten {}->{} (wpos={}!=rpos={})={}  (wpos={}<wcap={})!", unwritten_excess, unwritten_excess+num, wpos, rpos, wpos!=rpos, wpos, wcap, wpos<wcap);
+            unwritten_excess += num;
         }
         wpos += num;
         rpos += num;
@@ -246,8 +246,8 @@ struct FilterProcessorInplace
         }
         else
         {
-            _c4dbgip("inplace: set unfiltered {}->1!", unfiltered_chars);
-            unfiltered_chars = true;
+            _c4dbgip("inplace: add unfiltered {}->{}!", unwritten_excess, unwritten_excess+1u);
+            ++unwritten_excess;
         }
         ++wpos;
         rpos += 2;
@@ -259,7 +259,7 @@ struct FilterProcessorInplace
         RYML_ASSERT(nr > 0);
         RYML_ASSERT(rpos+nr <= src.len);
         const size_t wpos_next = wpos + nw;
-        const size_t rpos_next = rpos + 1 + nr;
+        const size_t rpos_next = rpos + nr + 1u; // add 1u to account for the escape character
         if(wpos_next <= rpos_next) // read and write do not overlap. just do a vanilla copy.
         {
             if(wpos_next <= wcap)
@@ -282,17 +282,19 @@ struct FilterProcessorInplace
                 else
                 {
                     rpos = rpos_next;
-                    _c4dbgip("inplace: set unfiltered {}->1!", unfiltered_chars);
-                    unfiltered_chars = true;
+                    const size_t unw = nw > (nr + 1u) ? nw - (nr + 1u) : 0;
+                    _c4dbgip("inplace: add unfiltered {}->{}!", unwritten_excess, unwritten_excess+unw);
+                    unwritten_excess += unw;
                 }
                 // extend the string up to capacity
                 src.len += excess;
             }
             else
             {
-                _c4dbgip("inplace: set unfiltered {}->1!", unfiltered_chars);
                 rpos = rpos_next;
-                unfiltered_chars = true;
+                const size_t unw = nw > (nr + 1u) ? nw - (nr + 1u) : 0;
+                _c4dbgip("inplace: add unfiltered {}->{}!", unwritten_excess, unwritten_excess+unw);
+                unwritten_excess += unw;
             }
             wpos = wpos_next;
         }

src/c4/yml/parse.cpp

@@ -4593,11 +4593,7 @@ void Parser::_filter_ws_copy_trailing(FilterProcessor &proc) noexcept
 template<class FilterProcessor>
 void Parser::_filter_ws_skip_trailing(FilterProcessor &proc) noexcept
 {
-    if(!_filter_ws_handle_to_first_non_space(proc))
-    {
-        _c4dbgfws("... everything else is trailing whitespace - skip {} chars", proc.src.len - proc.rpos);
-        proc.skip(proc.src.len - proc.rpos);
-    }
+    _RYML_CB_ASSERT(this->callbacks(), _filter_ws_handle_to_first_non_space(proc));
 }
 
 #undef _c4dbgfws
@@ -5128,6 +5124,7 @@ void Parser::_filter_chomp(FilterProcessor &C4_RESTRICT proc, BlockChomp_e chomp
             {
                 const char curr = proc.curr();
                 _c4dbgchomp("curr='{}'", _c4prc(curr));
+                _RYML_CB_ASSERT(this->callbacks(), curr == '\n' || curr == '\r');
                 switch(curr)
                 {
                 case '\n':
@@ -5166,9 +5163,6 @@ void Parser::_filter_chomp(FilterProcessor &C4_RESTRICT proc, BlockChomp_e chomp
                 case '\r':
                     proc.skip();
                     break;
-                default:
-                    _RYML_CB_ERR(this->callbacks(), "never reach this");
-                    break;
                 }
             }
         }
@@ -5272,6 +5266,8 @@ void Parser::_filter_block_indentation(FilterProcessor &C4_RESTRICT proc, size_t
         }
         else
         {
+            C4_ERROR("crl");
+            // UNCOVERED
             _c4dbgfb("all spaces to the end: {} spaces", first);
             first = rem.len;
             if(first)
@@ -5689,9 +5685,16 @@ csubstr Parser::_filter_scalar_dquot(substr s)
     }
     else
     {
-        _c4dbgpf("filtering dquo scalar: not enough space: needs {}, have {}", r.required_len(), s.len);
-        substr dst = m_tree->alloc_arena(r.required_len());
+        const size_t len = r.required_len();
+        _RYML_CB_ASSERT(this->callbacks(), s.len < len);
+        _c4dbgpf("filtering dquo scalar: not enough space: needs {}, have {}", len, s.len);
+        _RYML_CB_ASSERT(this->callbacks(), m_tree);
+        substr dst = m_tree->alloc_arena(len);
+        _c4dbgpf("filtering dquo scalar: dst.len={}", dst.len);
+        _RYML_CB_ASSERT(this->callbacks(), dst.len == len);
         r = this->filter_scalar_dquoted(s, dst);
+        _c4dbgpf("filtering dquo scalar: ... result now needs {} was {}", r.required_len(), len);
+        _RYML_CB_ASSERT(this->callbacks(), r.required_len() == len);
         _RYML_CB_CHECK(m_stack.m_callbacks, r.valid());
         _c4dbgpf("filtering dquo scalar: success! s=[{}]~~~{}~~~", r.get().len, r.get());
         return r.get();
@@ -5712,6 +5715,7 @@ csubstr Parser::_filter_scalar_block_literal(substr s, BlockChomp_e chomp, size_
     else
     {
         _c4dbgpf("filtering block literal scalar: not enough space: needs {}, have {}", r.required_len(), s.len);
+        _RYML_CB_ASSERT(this->callbacks(), m_tree);
         substr dst = m_tree->alloc_arena(r.required_len());
         r = this->filter_scalar_block_literal(s, dst, indentation, chomp);
         _RYML_CB_CHECK(m_stack.m_callbacks, r.valid());
@@ -5734,6 +5738,7 @@ csubstr Parser::_filter_scalar_block_folded(substr s, BlockChomp_e chomp, size_t
     else
     {
         _c4dbgpf("filtering block folded scalar: not enough space: needs {}, have {}", r.required_len(), s.len);
+        _RYML_CB_ASSERT(this->callbacks(), m_tree);
         substr dst = m_tree->alloc_arena(r.required_len());
         r = this->filter_scalar_block_folded(s, dst, indentation, chomp);
         _RYML_CB_CHECK(m_stack.m_callbacks, r.valid());

src/c4/yml/parse.hpp

@@ -356,12 +356,16 @@ class RYML_EXPORT Parser
     csubstr _scan_to_next_nonempty_line(size_t indentation);
     csubstr _extend_scanned_scalar(csubstr currscalar);
 
+public: // exposed for testing. to be cleared.
+
     csubstr _filter_scalar_squot(substr s);
     csubstr _filter_scalar_dquot(substr s);
     csubstr _filter_scalar_plain(substr s, size_t indentation);
     csubstr _filter_scalar_block_literal(substr s, BlockChomp_e chomp, size_t indentation);
     csubstr _filter_scalar_block_folded(substr s, BlockChomp_e chomp, size_t indentation);
 
+private:
+
     void  _handle_finished_file();
     void  _handle_line();
 

test/test_double_quoted.cpp

@@ -66,21 +66,33 @@ void test_filter_inplace(csubstr input, csubstr expected, csubstr leading_input,
     auto run = [&](size_t cap){
         // create the string
         std::string subject_(leading_input.str, leading_input.len);
-        subject_ += std::string(input.str, input.len);
+        subject_.append(input.str, input.len);
+        std::string subject_2 = subject_;
         subject_.resize(full_sz);
         // fill the canary region
         const char refchar = '`';
         const substr full = to_substr(subject_);
         full.sub(max_sz).fill(refchar);
         substr dst = full.first(input_sz);
         // filter now
-        Parser proc = {};
-        FilterResult result = proc.filter_scalar_dquoted_in_place(dst, cap);
+        Parser parser1 = {};
+std::cout << "WTF0: input_sz=" << input_sz << "  -->  expected_sz=" << expected_sz << " " << subject_.size() << "\n";
+        FilterResult result = parser1.filter_scalar_dquoted_in_place(dst, cap);
+std::cout << "WTF1: input_sz=" << input_sz << "  -->  expected_sz=" << expected_sz << " " << subject_2.size() << "\n";
+        Parser parser2 = {};
+        Tree tree = parser2.parse_in_arena("file", "# set the tree in the parser");
+std::cout << "WTF2: input_sz=" << input_sz << "  -->  expected_sz=" << expected_sz << " " << subject_2.size() << "\n";
+        csubstr sresult = parser2._filter_scalar_dquot(to_substr(subject_2));
+std::cout << "WTF3: input_sz=" << input_sz << "  -->  expected_sz=" << expected_sz << " " << subject_2.size() << "\n";
         EXPECT_EQ(result.required_len(), expected_sz);
+        EXPECT_EQ(sresult.len, expected_sz);
         if(result.valid())
         {
             const csubstr out = result.get();
             EXPECT_EQ(out, expected_);
+            EXPECT_EQ(sresult, expected_);
+            EXPECT_EQ(sresult.str, out.str);
+            EXPECT_EQ(sresult.len, out.len);
             // check the fill character in the canary region.
             EXPECT_GT(full.sub(max_sz).len, 0u);
             EXPECT_EQ(full.first_not_of(refchar, max_sz), csubstr::npos);
@@ -319,14 +331,10 @@ dquoted_case test_cases_filter[] = {
 C4_SUPPRESS_WARNING_MSVC_POP
 
 
+//-----------------------------------------------------------------------------
+
 TEST(double_quoted_filter, leading_tab)
 {
-    // this case cannot have a prefix
-    csubstr expected = "\t\ndetected\n";
-    test_filter_src_dst("	\n\ndetected\n\n", expected, /*sz*/expected.len + 2u);
-    test_filter_src_dst("\t\n\ndetected\n\n", expected, /*sz*/expected.len + 2u);
-    test_filter_inplace("	\n\ndetected\n\n", "\t\ndetected\n", "", "");
-    test_filter_inplace("\t\n\ndetected\n\n", "\t\ndetected\n", "", "");
 }