nsCharSeparatedTokenizer.h

/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=8 sts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#ifndef __nsCharSeparatedTokenizer_h
#define __nsCharSeparatedTokenizer_h

#include "mozilla/Maybe.h"
#include "mozilla/RangedPtr.h"
#include "mozilla/TypedEnumBits.h"

#include "nsCRTGlue.h"
#include "nsTDependentSubstring.h"

// Flags -- only one for now. If we need more, they should be defined to
// be 1 << 1, 1 << 2, etc. (They're masks, and aFlags is a bitfield.)
enum class nsTokenizerFlags {
  Default = 0,
  SeparatorOptional = 1 << 0,
  IncludeEmptyTokenAtEnd = 1 << 1
};

MOZ_MAKE_ENUM_CLASS_BITWISE_OPERATORS(nsTokenizerFlags)

/**
 * This parses a SeparatorChar-separated string into tokens.
 * Whitespace surrounding tokens is not treated as part of tokens, however
 * whitespace inside a token is. If the final token is the empty string, it is
 * not returned by default.
 *
 * Some examples, with SeparatorChar = ',':
 *
 * "foo, bar, baz" ->      "foo" "bar" "baz"
 * "foo,bar,baz" ->        "foo" "bar" "baz"
 * "foo , bar hi , baz" -> "foo" "bar hi" "baz"
 * "foo, ,bar,baz" ->      "foo" "" "bar" "baz"
 * "foo,,bar,baz" ->       "foo" "" "bar" "baz"
 * "foo,bar,baz," ->       "foo" "bar" "baz"
 *
 * The function used for whitespace detection is a template argument.
 * By default, it is NS_IsAsciiWhitespace.
 */
template <typename TDependentSubstringType, bool IsWhitespace(char16_t),
          nsTokenizerFlags Flags = nsTokenizerFlags::Default>
class nsTCharSeparatedTokenizer {
  using CharType = typename TDependentSubstringType::char_type;
  using SubstringType = typename TDependentSubstringType::substring_type;

 public:
  using DependentSubstringType = TDependentSubstringType;

  nsTCharSeparatedTokenizer(const SubstringType& aSource,
                            CharType aSeparatorChar)
      : mIter(aSource.Data(), aSource.Length()),
        mEnd(aSource.Data() + aSource.Length(), aSource.Data(),
             aSource.Length()),
        mSeparatorChar(aSeparatorChar),
        mWhitespaceBeforeFirstToken(false),
        mWhitespaceAfterCurrentToken(false),
        mSeparatorAfterCurrentToken(false) {
    // Skip initial whitespace
    while (mIter < mEnd && IsWhitespace(*mIter)) {
      mWhitespaceBeforeFirstToken = true;
      ++mIter;
    }
  }

  /**
   * Checks if any more tokens are available.
   */
  bool hasMoreTokens() const {
    MOZ_ASSERT(mIter == mEnd || !IsWhitespace(*mIter),
               "Should be at beginning of token if there is one");

    if constexpr (Flags & nsTokenizerFlags::IncludeEmptyTokenAtEnd) {
      return mIter < mEnd || (mIter == mEnd && mSeparatorAfterCurrentToken);
    } else {
      return mIter < mEnd;
    }
  }

  /*
   * Returns true if there is whitespace prior to the first token.
   */
  bool whitespaceBeforeFirstToken() const {
    return mWhitespaceBeforeFirstToken;
  }

  /*
   * Returns true if there is a separator after the current token.
   * Useful if you want to check whether the last token has a separator
   * after it which may not be valid.
   */
  bool separatorAfterCurrentToken() const {
    return mSeparatorAfterCurrentToken;
  }

  /*
   * Returns true if there is any whitespace after the current token.
   */
  bool whitespaceAfterCurrentToken() const {
    return mWhitespaceAfterCurrentToken;
  }

  /**
   * Returns the next token.
   */
  const DependentSubstringType nextToken() {
    mozilla::RangedPtr<const CharType> tokenStart = mIter;
    mozilla::RangedPtr<const CharType> tokenEnd = mIter;

    MOZ_ASSERT(mIter == mEnd || !IsWhitespace(*mIter),
               "Should be at beginning of token if there is one");

    // Search until we hit separator or end (or whitespace, if a separator
    // isn't required -- see clause with 'break' below).
    while (mIter < mEnd && *mIter != mSeparatorChar) {
      // Skip to end of the current word.
      while (mIter < mEnd && !IsWhitespace(*mIter) &&
             *mIter != mSeparatorChar) {
        ++mIter;
      }
      tokenEnd = mIter;

      // Skip whitespace after the current word.
      mWhitespaceAfterCurrentToken = false;
      while (mIter < mEnd && IsWhitespace(*mIter)) {
        mWhitespaceAfterCurrentToken = true;
        ++mIter;
      }
      if constexpr (Flags & nsTokenizerFlags::SeparatorOptional) {
        // We've hit (and skipped) whitespace, and that's sufficient to end
        // our token, regardless of whether we've reached a SeparatorChar.
        break;
      }  // (else, we'll keep looping until we hit mEnd or SeparatorChar)
    }

    mSeparatorAfterCurrentToken = (mIter != mEnd && *mIter == mSeparatorChar);
    MOZ_ASSERT((Flags & nsTokenizerFlags::SeparatorOptional) ||
                   (mSeparatorAfterCurrentToken == (mIter < mEnd)),
               "If we require a separator and haven't hit the end of "
               "our string, then we shouldn't have left the loop "
               "unless we hit a separator");

    // Skip separator (and any whitespace after it), if we're at one.
    if (mSeparatorAfterCurrentToken) {
      ++mIter;

      while (mIter < mEnd && IsWhitespace(*mIter)) {
        mWhitespaceAfterCurrentToken = true;
        ++mIter;
      }
    }

    return Substring(tokenStart.get(), tokenEnd.get());
  }

  auto ToRange() const;

 private:
  mozilla::RangedPtr<const CharType> mIter;
  const mozilla::RangedPtr<const CharType> mEnd;
  const CharType mSeparatorChar;
  bool mWhitespaceBeforeFirstToken;
  bool mWhitespaceAfterCurrentToken;
  bool mSeparatorAfterCurrentToken;
};

constexpr bool NS_TokenizerIgnoreNothing(char16_t) { return false; }

template <bool IsWhitespace(char16_t), typename CharType,
          nsTokenizerFlags Flags = nsTokenizerFlags::Default>
using nsTCharSeparatedTokenizerTemplate =
    nsTCharSeparatedTokenizer<nsTDependentSubstring<CharType>, IsWhitespace,
                              Flags>;

template <bool IsWhitespace(char16_t),
          nsTokenizerFlags Flags = nsTokenizerFlags::Default>
using nsCharSeparatedTokenizerTemplate =
    nsTCharSeparatedTokenizerTemplate<IsWhitespace, char16_t, Flags>;

using nsCharSeparatedTokenizer =
    nsCharSeparatedTokenizerTemplate<NS_IsAsciiWhitespace>;

template <bool IsWhitespace(char16_t),
          nsTokenizerFlags Flags = nsTokenizerFlags::Default>
using nsCCharSeparatedTokenizerTemplate =
    nsTCharSeparatedTokenizerTemplate<IsWhitespace, char, Flags>;

using nsCCharSeparatedTokenizer =
    nsCCharSeparatedTokenizerTemplate<NS_IsAsciiWhitespace>;

/**
 * Adapts a char separated tokenizer for use in a range-based for loop.
 *
 * Use this typically only indirectly, e.g. like
 *
 * for (const auto& token : nsCharSeparatedTokenizer(aText, ' ').ToRange()) {
 *    // ...
 * }
 */
template <typename Tokenizer>
class nsTokenizedRange {
 public:
  using DependentSubstringType = typename Tokenizer::DependentSubstringType;

  explicit nsTokenizedRange(Tokenizer&& aTokenizer)
      : mTokenizer(std::move(aTokenizer)) {}

  struct EndSentinel {};
  struct Iterator {
    explicit Iterator(const Tokenizer& aTokenizer) : mTokenizer(aTokenizer) {
      Next();
    }

    const DependentSubstringType& operator*() const { return *mCurrentToken; }

    Iterator& operator++() {
      Next();
      return *this;
    }

    bool operator==(const EndSentinel&) const {
      return mCurrentToken.isNothing();
    }

    bool operator!=(const EndSentinel&) const { return mCurrentToken.isSome(); }

   private:
    void Next() {
      mCurrentToken.reset();

      if (mTokenizer.hasMoreTokens()) {
        mCurrentToken.emplace(mTokenizer.nextToken());
      }
    }

    Tokenizer mTokenizer;
    mozilla::Maybe<DependentSubstringType> mCurrentToken;
  };

  auto begin() const { return Iterator{mTokenizer}; }
  auto end() const { return EndSentinel{}; }

 private:
  const Tokenizer mTokenizer;
};

template <typename TDependentSubstringType, bool IsWhitespace(char16_t),
          nsTokenizerFlags Flags>
auto nsTCharSeparatedTokenizer<TDependentSubstringType, IsWhitespace,
                               Flags>::ToRange() const {
  return nsTokenizedRange{nsTCharSeparatedTokenizer{*this}};
}

// You should not need to instantiate this class directly.
// Use nsTSubstring::Split instead.
template <typename T>
class nsTSubstringSplitter
    : public nsTokenizedRange<nsTCharSeparatedTokenizerTemplate<
          NS_TokenizerIgnoreNothing, T,
          nsTokenizerFlags::IncludeEmptyTokenAtEnd>> {
 public:
  using nsTokenizedRange<nsTCharSeparatedTokenizerTemplate<
      NS_TokenizerIgnoreNothing, T,
      nsTokenizerFlags::IncludeEmptyTokenAtEnd>>::nsTokenizedRange;
};

extern template class nsTSubstringSplitter<char>;
extern template class nsTSubstringSplitter<char16_t>;

#endif /* __nsCharSeparatedTokenizer_h */