token_functions.hpp

// Boost token_functions.hpp  ------------------------------------------------//

// Copyright John R. Bandela 2001. 

// Distributed under the Boost Software License, Version 1.0. (See
// accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)

// See http://www.boost.org/libs/tokenizer/ for documentation.

// Revision History:
// 01 Oct 2004   Joaquín M López Muñoz
//      Workaround for a problem with string::assign in msvc-stlport
// 06 Apr 2004   John Bandela
//      Fixed a bug involving using char_delimiter with a true input iterator
// 28 Nov 2003   Robert Zeh and John Bandela
//      Converted into "fast" functions that avoid using += when
//      the supplied iterator isn't an input_iterator; based on
//      some work done at Archelon and a version that was checked into
//      the boost CVS for a short period of time.
// 20 Feb 2002   John Maddock
//      Removed using namespace std declarations and added
//      workaround for BOOST_NO_STDC_NAMESPACE (the library
//      can be safely mixed with regex).
// 06 Feb 2002   Jeremy Siek
//      Added char_separator.
// 02 Feb 2002   Jeremy Siek
//      Removed tabs and a little cleanup.


#ifndef BOOST_TOKEN_FUNCTIONS_JRB120303_HPP_
#define BOOST_TOKEN_FUNCTIONS_JRB120303_HPP_

#include <vector>
#include <stdexcept>
#include <string>
#include <cctype>
#include <algorithm> // for find_if
#include <boost/config.hpp>
#include <boost/assert.hpp>
#include <boost/detail/workaround.hpp>
#include <boost/mpl/if.hpp>

//
// the following must not be macros if we are to prefix them
// with std:: (they shouldn't be macros anyway...)
//
#ifdef ispunct
#  undef ispunct
#endif
#ifdef isspace
#  undef isspace
#endif
//
// fix namespace problems:
//
#ifdef BOOST_NO_STDC_NAMESPACE
namespace std{
 using ::ispunct;
 using ::isspace;
}
#endif

namespace boost{

  //===========================================================================
  // The escaped_list_separator class. Which is a model of TokenizerFunction
  // An escaped list is a super-set of what is commonly known as a comma 
  // separated value (csv) list.It is separated into fields by a comma or 
  // other character. If the delimiting character is inside quotes, then it is
  // counted as a regular character.To allow for embedded quotes in a field,
  // there can be escape sequences using the \ much like C. 
  // The role of the comma, the quotation mark, and the escape 
  // character (backslash \), can be assigned to other characters.

  struct escaped_list_error : public std::runtime_error{
    escaped_list_error(const std::string& what_arg):std::runtime_error(what_arg) { }
  };
  

// The out of the box GCC 2.95 on cygwin does not have a char_traits class.
// MSVC does not like the following typename
#if !defined(BOOST_MSVC) || BOOST_MSVC > 1300
  template <class Char, 
    class Traits = typename std::basic_string<Char>::traits_type >
#else
  template <class Char, 
    class Traits = std::basic_string<Char>::traits_type >
#endif
  class escaped_list_separator {

  private:
    typedef std::basic_string<Char,Traits> string_type;
    struct char_eq {
      Char e_;
      char_eq(Char e):e_(e) { }
      bool operator()(Char c) {
        return Traits::eq(e_,c);
      }
    };
    string_type  escape_;
    string_type  c_;
    string_type  quote_;      
    bool last_;

    bool is_escape(Char e) {
      char_eq f(e);
      return std::find_if(escape_.begin(),escape_.end(),f)!=escape_.end();
    }
    bool is_c(Char e) {
      char_eq f(e);
      return std::find_if(c_.begin(),c_.end(),f)!=c_.end();
    }
    bool is_quote(Char e) {
      char_eq f(e);
      return std::find_if(quote_.begin(),quote_.end(),f)!=quote_.end();
    }
    template <typename iterator, typename Token>
    void do_escape(iterator& next,iterator end,Token& tok) {
      if (++next == end)
        throw escaped_list_error(std::string("cannot end with escape"));
      if (Traits::eq(*next,'n')) {
        tok+='\n';
        return;
      }
      else if (is_quote(*next)) {
        tok+=*next;
        return;
      }
      else if (is_c(*next)) {
        tok+=*next;
        return;
      }
      else if (is_escape(*next)) {
        tok+=*next;
        return;
      }
      else
        throw escaped_list_error(std::string("unknown escape sequence"));
    }

    public:
    
    explicit escaped_list_separator(Char  e = '\\',
                                    Char c = ',',Char  q = '\"')
      : escape_(1,e), c_(1,c), quote_(1,q), last_(false) { }
    
    escaped_list_separator(string_type e, string_type c, string_type q)
      : escape_(e), c_(c), quote_(q), last_(false) { }
    
    void reset() {last_=false;}

    template <typename InputIterator, typename Token>
    bool operator()(InputIterator& next,InputIterator end,Token& tok) {
      bool bInQuote = false;
      tok = Token();
      
      if (next == end) {
        if (last_) {
          last_ = false;
          return true;
        }
        else
          return false;
      }
      last_ = false;
      for (;next != end;++next) {
        if (is_escape(*next)) {
          do_escape(next,end,tok);
        }
        else if (is_c(*next)) {
          if (!bInQuote) {
            // If we are not in quote, then we are done
            ++next;
            // The last character was a c, that means there is
            // 1 more blank field
            last_ = true; 
            return true;
          }
          else tok+=*next;
        }
        else if (is_quote(*next)) {
          bInQuote=!bInQuote;
        }
        else {
          tok += *next;
        }
      }
      return true;
    }
  };

  //===========================================================================
  // The classes here are used by offset_separator and char_separator to implement
  // faster assigning of tokens using assign instead of +=
  
  namespace tokenizer_detail {

  // The assign_or_plus_equal struct contains functions that implement
  // assign, +=, and clearing based on the iterator type.  The
  // generic case does nothing for plus_equal and clearing, while
  // passing through the call for assign.
  //
  // When an input iterator is being used, the situation is reversed.
  // The assign method does nothing, plus_equal invokes operator +=,
  // and the clearing method sets the supplied token to the default
  // token constructor's result.
  //

  template<class IteratorTag>
  struct assign_or_plus_equal {
    template<class Iterator, class Token>
    static void assign(Iterator b, Iterator e, Token &t) {

#if BOOST_WORKAROUND(BOOST_MSVC, < 1300) &&\
    BOOST_WORKAROUND(__SGI_STL_PORT, < 0x500) &&\
    defined(_STLP_DEBUG) &&\
    (defined(_STLP_USE_DYNAMIC_LIB) || defined(_DLL))
    // Problem with string::assign for msvc-stlport in debug mode: the
    // linker tries to import the templatized version of this memfun,
    // which is obviously not exported.
    // See http://www.stlport.com/dcforum/DCForumID6/1763.html for details.

      t = Token();
      while(b != e) t += *b++;
#else
      t.assign(b, e);
#endif

    }

    template<class Token, class Value> 
    static void plus_equal(Token &, const Value &) {
    
  }

    // If we are doing an assign, there is no need for the
    // the clear. 
    //
    template<class Token>
    static void clear(Token &) {

    }
  };

  template <>
  struct assign_or_plus_equal<std::input_iterator_tag> {
    template<class Iterator, class Token>
    static void assign(Iterator b, Iterator e, Token &t) {

    }
    template<class Token, class Value> 
    static void plus_equal(Token &t, const Value &v) {
      t += v;
    }
    template<class Token>
    static void clear(Token &t) {
      t = Token();
    }
  };


  template<class Iterator>
  struct pointer_iterator_category{
    typedef std::random_access_iterator_tag type;
  };


  template<class Iterator>
  struct class_iterator_category{
    typedef typename Iterator::iterator_category type;
  };



  // This portably gets the iterator_tag without partial template specialization
  template<class Iterator>
    struct get_iterator_category{
    typedef typename mpl::if_<is_pointer<Iterator>,
      pointer_iterator_category<Iterator>,
      class_iterator_category<Iterator>
    >::type cat;

    typedef typename cat::type iterator_category;
  };

  
}

   
  //===========================================================================
  // The offset_separator class, which is a model of TokenizerFunction.
  // Offset breaks a string into tokens based on a range of offsets

  class offset_separator {
  private:

    std::vector<int> offsets_;
    unsigned int current_offset_;
    bool wrap_offsets_;
    bool return_partial_last_;
    
  public:
    template <typename Iter>
    offset_separator(Iter begin, Iter end, bool wrap_offsets = true,
                     bool return_partial_last = true)
      : offsets_(begin,end), current_offset_(0),
        wrap_offsets_(wrap_offsets),
        return_partial_last_(return_partial_last) { }
   
    offset_separator()
      : offsets_(1,1), current_offset_(),
        wrap_offsets_(true), return_partial_last_(true) { }

    void reset() {
      current_offset_ = 0;
    }

    template <typename InputIterator, typename Token>
    bool operator()(InputIterator& next, InputIterator end, Token& tok)
    {
      typedef tokenizer_detail::assign_or_plus_equal<
#if     !defined(BOOST_MSVC) || BOOST_MSVC > 1300
        typename
#endif
        tokenizer_detail::get_iterator_category<
        InputIterator>::iterator_category> assigner;


      BOOST_ASSERT(!offsets_.empty());
    
      assigner::clear(tok);
      InputIterator start(next);
      
      if (next == end)
        return false;

      if (current_offset_ == offsets_.size())
      {
        if (wrap_offsets_)
          current_offset_=0;
        else
          return false;
      }
      
      int c = offsets_[current_offset_];
      int i = 0;
      for (; i < c; ++i) {
        if (next == end)break;
        assigner::plus_equal(tok,*next++);
      }
      assigner::assign(start,next,tok);
    
      if (!return_partial_last_)
        if (i < (c-1) )
          return false;
      
      ++current_offset_;
      return true;
    }
  };


  //===========================================================================
  // The char_separator class breaks a sequence of characters into
  // tokens based on the character delimiters (very much like bad old
  // strtok). A delimiter character can either be kept or dropped. A
  // kept delimiter shows up as an output token, whereas a dropped
  // delimiter does not.

  // This class replaces the char_delimiters_separator class. The
  // constructor for the char_delimiters_separator class was too
  // confusing and needed to be deprecated. However, because of the
  // default arguments to the constructor, adding the new constructor
  // would cause ambiguity, so instead I deprecated the whole class.
  // The implementation of the class was also simplified considerably.

  enum empty_token_policy { drop_empty_tokens, keep_empty_tokens };

  // The out of the box GCC 2.95 on cygwin does not have a char_traits class.
#if !defined(BOOST_MSVC) || BOOST_MSVC > 1300
  template <typename Char, 
    typename Traits = typename std::basic_string<Char>::traits_type >
#else
  template <typename Char, 
    typename Traits = std::basic_string<Char>::traits_type >
#endif
  class char_separator
  {
    typedef std::basic_string<Char,Traits> string_type;
  public:
    explicit 
    char_separator(const Char* dropped_delims,
                   const Char* kept_delims = 0,
                   empty_token_policy empty_tokens = drop_empty_tokens)
      : m_dropped_delims(dropped_delims),
        m_use_ispunct(false),
        m_use_isspace(false),
        m_empty_tokens(empty_tokens),
        m_output_done(false)
    {
      // Borland workaround
      if (kept_delims)
        m_kept_delims = kept_delims;
    }

                // use ispunct() for kept delimiters and isspace for dropped.
    explicit
    char_separator()
      : m_use_ispunct(true), 
        m_use_isspace(true), 
        m_empty_tokens(drop_empty_tokens) { }

    void reset() { }

    template <typename InputIterator, typename Token>
    bool operator()(InputIterator& next, InputIterator end, Token& tok)
    {
      typedef tokenizer_detail::assign_or_plus_equal<
#if     !defined(BOOST_MSVC) || BOOST_MSVC > 1300
        typename
#endif
        tokenizer_detail::get_iterator_category<
        InputIterator>::iterator_category> assigner;

      assigner::clear(tok);

      // skip past all dropped_delims
      if (m_empty_tokens == drop_empty_tokens)
        for (; next != end  && is_dropped(*next); ++next)
          { }
      
      InputIterator start(next);

      if (m_empty_tokens == drop_empty_tokens) {

        if (next == end)
          return false;


        // if we are on a kept_delims move past it and stop
        if (is_kept(*next)) {
          assigner::plus_equal(tok,*next);
          ++next;
        } else
          // append all the non delim characters
          for (; next != end && !is_dropped(*next) && !is_kept(*next); ++next)
            assigner::plus_equal(tok,*next);
      } 
      else { // m_empty_tokens == keep_empty_tokens
        
        // Handle empty token at the end
        if (next == end)
        {
          if (m_output_done == false) 
          {
            m_output_done = true;
            assigner::assign(start,next,tok);
            return true;
          } 
          else
            return false;
        }
        
        if (is_kept(*next)) {
          if (m_output_done == false)
            m_output_done = true;
          else {
            assigner::plus_equal(tok,*next);
            ++next;
            m_output_done = false;
          }
        } 
        else if (m_output_done == false && is_dropped(*next)) {
          m_output_done = true;
        } 
        else {
          if (is_dropped(*next))
            start=++next;
          for (; next != end && !is_dropped(*next) && !is_kept(*next); ++next)
            assigner::plus_equal(tok,*next);
          m_output_done = true;
        }
      }
      assigner::assign(start,next,tok);
      return true;
    }

  private:
    string_type m_kept_delims;
    string_type m_dropped_delims;
    bool m_use_ispunct;
    bool m_use_isspace;
    empty_token_policy m_empty_tokens;
    bool m_output_done;
    
    bool is_kept(Char E) const
    {  
      if (m_kept_delims.length())
        return m_kept_delims.find(E) != string_type::npos;
      else if (m_use_ispunct) {
        return std::ispunct(E) != 0;
      } else
        return false;
    }
    bool is_dropped(Char E) const
    {
      if (m_dropped_delims.length())
        return m_dropped_delims.find(E) != string_type::npos;
      else if (m_use_isspace) {
        return std::isspace(E) != 0;
      } else
        return false;
    }
  };

  //===========================================================================
  // The following class is DEPRECATED, use class char_separators instead.
  //
  // The char_delimiters_separator class, which is a model of
  // TokenizerFunction.  char_delimiters_separator breaks a string
  // into tokens based on character delimiters. There are 2 types of
  // delimiters. returnable delimiters can be returned as
  // tokens. These are often punctuation. nonreturnable delimiters
  // cannot be returned as tokens. These are often whitespace

  // The out of the box GCC 2.95 on cygwin does not have a char_traits class.
#if !defined(BOOST_MSVC) || BOOST_MSVC > 1300
  template <class Char, 
    class Traits = typename std::basic_string<Char>::traits_type >
#else
  template <class Char, 
    class Traits = std::basic_string<Char>::traits_type >
#endif
  class char_delimiters_separator {
  private:  

    typedef std::basic_string<Char,Traits> string_type;
    string_type returnable_;
    string_type nonreturnable_;
    bool return_delims_;
    bool no_ispunct_;
    bool no_isspace_;
    
    bool is_ret(Char E)const
    {  
      if (returnable_.length())
        return  returnable_.find(E) != string_type::npos;
      else{
        if (no_ispunct_) {return false;}
        else{
          int r = std::ispunct(E);
          return r != 0;
        }
      }
    }
    bool is_nonret(Char E)const
    {
      if (nonreturnable_.length())
        return  nonreturnable_.find(E) != string_type::npos;
      else{
        if (no_isspace_) {return false;}
        else{
          int r = std::isspace(E);
          return r != 0;
        }
      }
    }
    
  public:
    explicit char_delimiters_separator(bool return_delims = false, 
                                       const Char* returnable = 0,
                                       const Char* nonreturnable = 0)
      : returnable_(returnable ? returnable : string_type().c_str()),
        nonreturnable_(nonreturnable ? nonreturnable:string_type().c_str()),
        return_delims_(return_delims), no_ispunct_(returnable!=0),
        no_isspace_(nonreturnable!=0) { }
    
    void reset() { }

  public:

     template <typename InputIterator, typename Token>
     bool operator()(InputIterator& next, InputIterator end,Token& tok) {
     tok = Token();
     
     // skip past all nonreturnable delims
     // skip past the returnable only if we are not returning delims
     for (;next!=end && ( is_nonret(*next) || (is_ret(*next) 
       && !return_delims_ ) );++next) { }
     
     if (next == end) {
       return false;
     }
     
     // if we are to return delims and we are one a returnable one
     // move past it and stop
     if (is_ret(*next) && return_delims_) {
       tok+=*next;
       ++next;
     }
     else
       // append all the non delim characters
       for (;next!=end && !is_nonret(*next) && !is_ret(*next);++next)
         tok+=*next;
       
       
     return true;
   }
  };


} //namespace boost


#endif