Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Fetching contributors…

Cannot retrieve contributors at this time

723 lines (629 sloc) 26.993 kb
/*
***************************************************************************
* Copyright (C) 1999-2008 International Business Machines Corporation *
* and others. All rights reserved. *
***************************************************************************
**********************************************************************
* Date Name Description
* 10/22/99 alan Creation.
* 11/11/99 rgillam Complete port from Java.
**********************************************************************
*/
#ifndef RBBI_H
#define RBBI_H
#include "unicode/utypes.h"
/**
* \file
* \brief C++ API: Rule Based Break Iterator
*/
#if !UCONFIG_NO_BREAK_ITERATION
#include "unicode/brkiter.h"
#include "unicode/udata.h"
#include "unicode/parseerr.h"
#include "unicode/schriter.h"
#include "unicode/uchriter.h"
struct UTrie;
U_NAMESPACE_BEGIN
/** @internal */
struct RBBIDataHeader;
class RuleBasedBreakIteratorTables;
class BreakIterator;
class RBBIDataWrapper;
class UStack;
class LanguageBreakEngine;
class UnhandledEngine;
struct RBBIStateTable;
/**
*
* A subclass of BreakIterator whose behavior is specified using a list of rules.
* <p>Instances of this class are most commonly created by the factory methods of
* BreakIterator::createWordInstance(), BreakIterator::createLineInstance(), etc.,
* and then used via the abstract API in class BreakIterator</p>
*
* <p>See the ICU User Guide for information on Break Iterator Rules.</p>
*
* <p>This class is not intended to be subclassed. (Class DictionaryBasedBreakIterator
* is a subclass, but that relationship is effectively internal to the ICU
* implementation. The subclassing interface to RulesBasedBreakIterator is
* not part of the ICU API, and may not remain stable.</p>
*
*/
class U_COMMON_API RuleBasedBreakIterator : public BreakIterator {
protected:
/**
* The UText through which this BreakIterator accesses the text
* @internal
*/
UText *fText;
/**
* A character iterator that refers to the same text as the UText, above.
* Only included for compatibility with old API, which was based on CharacterIterators.
* Value may be adopted from outside, or one of fSCharIter or fDCharIter, below.
*/
CharacterIterator *fCharIter;
/**
* When the input text is provided by a UnicodeString, this will point to
* a characterIterator that wraps that data. Needed only for the
* implementation of getText(), a backwards compatibility issue.
*/
StringCharacterIterator *fSCharIter;
/**
* When the input text is provided by a UText, this
* dummy CharacterIterator over an empty string will
* be returned from getText()
*/
UCharCharacterIterator *fDCharIter;
/**
* The rule data for this BreakIterator instance
* @internal
*/
RBBIDataWrapper *fData;
/** Index of the Rule {tag} values for the most recent match.
* @internal
*/
int32_t fLastRuleStatusIndex;
/**
* Rule tag value valid flag.
* Some iterator operations don't intrinsically set the correct tag value.
* This flag lets us lazily compute the value if we are ever asked for it.
* @internal
*/
UBool fLastStatusIndexValid;
/**
* Counter for the number of characters encountered with the "dictionary"
* flag set.
* @internal
*/
uint32_t fDictionaryCharCount;
/**
* When a range of characters is divided up using the dictionary, the break
* positions that are discovered are stored here, preventing us from having
* to use either the dictionary or the state table again until the iterator
* leaves this range of text. Has the most impact for line breaking.
* @internal
*/
int32_t* fCachedBreakPositions;
/**
* The number of elements in fCachedBreakPositions
* @internal
*/
int32_t fNumCachedBreakPositions;
/**
* if fCachedBreakPositions is not null, this indicates which item in the
* cache the current iteration position refers to
* @internal
*/
int32_t fPositionInCache;
/**
*
* If present, UStack of LanguageBreakEngine objects that might handle
* dictionary characters. Searched from top to bottom to find an object to
* handle a given character.
* @internal
*/
UStack *fLanguageBreakEngines;
/**
*
* If present, the special LanguageBreakEngine used for handling
* characters that are in the dictionary set, but not handled by any
* LangugageBreakEngine.
* @internal
*/
UnhandledEngine *fUnhandledBreakEngine;
/**
*
* The type of the break iterator, or -1 if it has not been set.
* @internal
*/
int32_t fBreakType;
protected:
//=======================================================================
// constructors
//=======================================================================
/**
* Constant to be used in the constructor
* RuleBasedBreakIterator(RBBIDataHeader*, EDontAdopt, UErrorCode &);
* which does not adopt the memory indicated by the RBBIDataHeader*
* parameter.
*
* @internal
*/
enum EDontAdopt {
kDontAdopt
};
/**
* Constructor from a flattened set of RBBI data in malloced memory.
* RulesBasedBreakIterators built from a custom set of rules
* are created via this constructor; the rules are compiled
* into memory, then the break iterator is constructed here.
*
* The break iterator adopts the memory, and will
* free it when done.
* @internal
*/
RuleBasedBreakIterator(RBBIDataHeader* data, UErrorCode &status);
/**
* Constructor from a flattened set of RBBI data in memory which need not
* be malloced (e.g. it may be a memory-mapped file, etc.).
*
* This version does not adopt the memory, and does not
* free it when done.
* @internal
*/
RuleBasedBreakIterator(const RBBIDataHeader* data, enum EDontAdopt dontAdopt, UErrorCode &status);
friend class RBBIRuleBuilder;
/** @internal */
friend class BreakIterator;
public:
/** Default constructor. Creates an empty shell of an iterator, with no
* rules or text to iterate over. Object can subsequently be assigned to.
* @stable ICU 2.2
*/
RuleBasedBreakIterator();
/**
* Copy constructor. Will produce a break iterator with the same behavior,
* and which iterates over the same text, as the one passed in.
* @param that The RuleBasedBreakIterator passed to be copied
* @stable ICU 2.0
*/
RuleBasedBreakIterator(const RuleBasedBreakIterator& that);
/**
* Construct a RuleBasedBreakIterator from a set of rules supplied as a string.
* @param rules The break rules to be used.
* @param parseError In the event of a syntax error in the rules, provides the location
* within the rules of the problem.
* @param status Information on any errors encountered.
* @stable ICU 2.2
*/
RuleBasedBreakIterator( const UnicodeString &rules,
UParseError &parseError,
UErrorCode &status);
/**
* This constructor uses the udata interface to create a BreakIterator
* whose internal tables live in a memory-mapped file. "image" is an
* ICU UDataMemory handle for the pre-compiled break iterator tables.
* @param image handle to the memory image for the break iterator data.
* Ownership of the UDataMemory handle passes to the Break Iterator,
* which will be responsible for closing it when it is no longer needed.
* @param status Information on any errors encountered.
* @see udata_open
* @see #getBinaryRules
* @stable ICU 2.8
*/
RuleBasedBreakIterator(UDataMemory* image, UErrorCode &status);
/**
* Destructor
* @stable ICU 2.0
*/
virtual ~RuleBasedBreakIterator();
/**
* Assignment operator. Sets this iterator to have the same behavior,
* and iterate over the same text, as the one passed in.
* @param that The RuleBasedBreakItertor passed in
* @return the newly created RuleBasedBreakIterator
* @stable ICU 2.0
*/
RuleBasedBreakIterator& operator=(const RuleBasedBreakIterator& that);
/**
* Equality operator. Returns TRUE if both BreakIterators are of the
* same class, have the same behavior, and iterate over the same text.
* @param that The BreakIterator to be compared for equality
* @return TRUE if both BreakIterators are of the
* same class, have the same behavior, and iterate over the same text.
* @stable ICU 2.0
*/
virtual UBool operator==(const BreakIterator& that) const;
/**
* Not-equal operator. If operator== returns TRUE, this returns FALSE,
* and vice versa.
* @param that The BreakIterator to be compared for inequality
* @return TRUE if both BreakIterators are not same.
* @stable ICU 2.0
*/
UBool operator!=(const BreakIterator& that) const;
/**
* Returns a newly-constructed RuleBasedBreakIterator with the same
* behavior, and iterating over the same text, as this one.
* Differs from the copy constructor in that it is polymorphic, and
* will correctly clone (copy) a derived class.
* clone() is thread safe. Multiple threads may simultaeneously
* clone the same source break iterator.
* @return a newly-constructed RuleBasedBreakIterator
* @stable ICU 2.0
*/
virtual BreakIterator* clone() const;
/**
* Compute a hash code for this BreakIterator
* @return A hash code
* @stable ICU 2.0
*/
virtual int32_t hashCode(void) const;
/**
* Returns the description used to create this iterator
* @return the description used to create this iterator
* @stable ICU 2.0
*/
virtual const UnicodeString& getRules(void) const;
//=======================================================================
// BreakIterator overrides
//=======================================================================
/**
* <p>
* Return a CharacterIterator over the text being analyzed.
* The returned character iterator is owned by the break iterator, and must
* not be deleted by the caller. Repeated calls to this function may
* return the same CharacterIterator.
* </p>
* <p>
* The returned character iterator must not be used concurrently with
* the break iterator. If concurrent operation is needed, clone the
* returned character iterator first and operate on the clone.
* </p>
* <p>
* When the break iterator is operating on text supplied via a UText,
* this function will fail. Lacking any way to signal failures, it
* returns an CharacterIterator containing no text.
* The function getUText() provides similar functionality,
* is reliable, and is more efficient.
* </p>
*
* TODO: deprecate this function?
*
* @return An iterator over the text being analyzed.
* @stable ICU 2.0
*/
virtual CharacterIterator& getText(void) const;
/**
* Get a UText for the text being analyzed.
* The returned UText is a shallow clone of the UText used internally
* by the break iterator implementation. It can safely be used to
* access the text without impacting any break iterator operations,
* but the underlying text itself must not be altered.
*
* @param fillIn A UText to be filled in. If NULL, a new UText will be
* allocated to hold the result.
* @param status receives any error codes.
* @return The current UText for this break iterator. If an input
* UText was provided, it will always be returned.
* @stable ICU 3.4
*/
virtual UText *getUText(UText *fillIn, UErrorCode &status) const;
/**
* Set the iterator to analyze a new piece of text. This function resets
* the current iteration position to the beginning of the text.
* @param newText An iterator over the text to analyze. The BreakIterator
* takes ownership of the character iterator. The caller MUST NOT delete it!
* @stable ICU 2.0
*/
virtual void adoptText(CharacterIterator* newText);
/**
* Set the iterator to analyze a new piece of text. This function resets
* the current iteration position to the beginning of the text.
* @param newText The text to analyze.
* @stable ICU 2.0
*/
virtual void setText(const UnicodeString& newText);
/**
* Reset the break iterator to operate over the text represented by
* the UText. The iterator position is reset to the start.
*
* This function makes a shallow clone of the supplied UText. This means
* that the caller is free to immediately close or otherwise reuse the
* Utext that was passed as a parameter, but that the underlying text itself
* must not be altered while being referenced by the break iterator.
*
* @param text The UText used to change the text.
* @param status Receives any error codes.
* @stable ICU 3.4
*/
virtual void setText(UText *text, UErrorCode &status);
/**
* Sets the current iteration position to the beginning of the text.
* @return The offset of the beginning of the text.
* @stable ICU 2.0
*/
virtual int32_t first(void);
/**
* Sets the current iteration position to the end of the text.
* @return The text's past-the-end offset.
* @stable ICU 2.0
*/
virtual int32_t last(void);
/**
* Advances the iterator either forward or backward the specified number of steps.
* Negative values move backward, and positive values move forward. This is
* equivalent to repeatedly calling next() or previous().
* @param n The number of steps to move. The sign indicates the direction
* (negative is backwards, and positive is forwards).
* @return The character offset of the boundary position n boundaries away from
* the current one.
* @stable ICU 2.0
*/
virtual int32_t next(int32_t n);
/**
* Advances the iterator to the next boundary position.
* @return The position of the first boundary after this one.
* @stable ICU 2.0
*/
virtual int32_t next(void);
/**
* Moves the iterator backwards, to the last boundary preceding this one.
* @return The position of the last boundary position preceding this one.
* @stable ICU 2.0
*/
virtual int32_t previous(void);
/**
* Sets the iterator to refer to the first boundary position following
* the specified position.
* @param offset The position from which to begin searching for a break position.
* @return The position of the first break after the current position.
* @stable ICU 2.0
*/
virtual int32_t following(int32_t offset);
/**
* Sets the iterator to refer to the last boundary position before the
* specified position.
* @param offset The position to begin searching for a break from.
* @return The position of the last boundary before the starting position.
* @stable ICU 2.0
*/
virtual int32_t preceding(int32_t offset);
/**
* Returns true if the specfied position is a boundary position. As a side
* effect, leaves the iterator pointing to the first boundary position at
* or after "offset".
* @param offset the offset to check.
* @return True if "offset" is a boundary position.
* @stable ICU 2.0
*/
virtual UBool isBoundary(int32_t offset);
/**
* Returns the current iteration position.
* @return The current iteration position.
* @stable ICU 2.0
*/
virtual int32_t current(void) const;
/**
* Return the status tag from the break rule that determined the most recently
* returned break position. For break rules that do not specify a
* status, a default value of 0 is returned. If more than one break rule
* would cause a boundary to be located at some position in the text,
* the numerically largest of the applicable status values is returned.
* <p>
* Of the standard types of ICU break iterators, only word break and
* line break provide status values. The values are defined in
* the header file ubrk.h. For Word breaks, the status allows distinguishing between words
* that contain alphabetic letters, "words" that appear to be numbers,
* punctuation and spaces, words containing ideographic characters, and
* more. For Line Break, the status distinguishes between hard (mandatory) breaks
* and soft (potential) break positions.
* <p>
* <code>getRuleStatus()</code> can be called after obtaining a boundary
* position from <code>next()</code>, <code>previous()</code>, or
* any other break iterator functions that returns a boundary position.
* <p>
* When creating custom break rules, one is free to define whatever
* status values may be convenient for the application.
* <p>
* Note: this function is not thread safe. It should not have been
* declared const, and the const remains only for compatibility
* reasons. (The function is logically const, but not bit-wise const).
* <p>
* @return the status from the break rule that determined the most recently
* returned break position.
*
* @see UWordBreak
* @stable ICU 2.2
*/
virtual int32_t getRuleStatus() const;
/**
* Get the status (tag) values from the break rule(s) that determined the most
* recently returned break position.
* <p>
* The returned status value(s) are stored into an array provided by the caller.
* The values are stored in sorted (ascending) order.
* If the capacity of the output array is insufficient to hold the data,
* the output will be truncated to the available length, and a
* U_BUFFER_OVERFLOW_ERROR will be signaled.
*
* @param fillInVec an array to be filled in with the status values.
* @param capacity the length of the supplied vector. A length of zero causes
* the function to return the number of status values, in the
* normal way, without attemtping to store any values.
* @param status receives error codes.
* @return The number of rule status values from rules that determined
* the most recent boundary returned by the break iterator.
* In the event of a U_BUFFER_OVERFLOW_ERROR, the return value
* is the total number of status values that were available,
* not the reduced number that were actually returned.
* @see getRuleStatus
* @stable ICU 3.0
*/
virtual int32_t getRuleStatusVec(int32_t *fillInVec, int32_t capacity, UErrorCode &status);
/**
* Returns a unique class ID POLYMORPHICALLY. Pure virtual override.
* This method is to implement a simple version of RTTI, since not all
* C++ compilers support genuine RTTI. Polymorphic operator==() and
* clone() methods call this method.
*
* @return The class ID for this object. All objects of a
* given class have the same class ID. Objects of
* other classes have different class IDs.
* @stable ICU 2.0
*/
virtual UClassID getDynamicClassID(void) const;
/**
* Returns the class ID for this class. This is useful only for
* comparing to a return value from getDynamicClassID(). For example:
*
* Base* polymorphic_pointer = createPolymorphicObject();
* if (polymorphic_pointer->getDynamicClassID() ==
* Derived::getStaticClassID()) ...
*
* @return The class ID for all objects of this class.
* @stable ICU 2.0
*/
static UClassID U_EXPORT2 getStaticClassID(void);
/*
* Create a clone (copy) of this break iterator in memory provided
* by the caller. The idea is to increase performance by avoiding
* a storage allocation. Use of this functoin is NOT RECOMMENDED.
* Performance gains are minimal, and correct buffer management is
* tricky. Use clone() instead.
*
* @param stackBuffer The pointer to the memory into which the cloned object
* should be placed. If NULL, allocate heap memory
* for the cloned object.
* @param BufferSize The size of the buffer. If zero, return the required
* buffer size, but do not clone the object. If the
* size was too small (but not zero), allocate heap
* storage for the cloned object.
*
* @param status Error status. U_SAFECLONE_ALLOCATED_WARNING will be
* returned if the the provided buffer was too small, and
* the clone was therefore put on the heap.
*
* @return Pointer to the clone object. This may differ from the stackBuffer
* address if the byte alignment of the stack buffer was not suitable
* or if the stackBuffer was too small to hold the clone.
* @stable ICU 2.0
*/
virtual BreakIterator * createBufferClone(void *stackBuffer,
int32_t &BufferSize,
UErrorCode &status);
/**
* Return the binary form of compiled break rules,
* which can then be used to create a new break iterator at some
* time in the future. Creating a break iterator from pre-compiled rules
* is much faster than building one from the source form of the
* break rules.
*
* The binary data can only be used with the same version of ICU
* and on the same platform type (processor endian-ness)
*
* @param length Returns the length of the binary data. (Out paramter.)
*
* @return A pointer to the binary (compiled) rule data. The storage
* belongs to the RulesBasedBreakIterator object, not the
* caller, and must not be modified or deleted.
* @internal
*/
virtual const uint8_t *getBinaryRules(uint32_t &length);
protected:
//=======================================================================
// implementation
//=======================================================================
/**
* Dumps caches and performs other actions associated with a complete change
* in text or iteration position.
* @internal
*/
virtual void reset(void);
#if 0
/**
* Return true if the category lookup for this char
* indicates that it is in the set of dictionary lookup chars.
* This function is intended for use by dictionary based break iterators.
* @return true if the category lookup for this char
* indicates that it is in the set of dictionary lookup chars.
* @internal
*/
virtual UBool isDictionaryChar(UChar32);
/**
* Get the type of the break iterator.
* @internal
*/
virtual int32_t getBreakType() const;
#endif
/**
* Set the type of the break iterator.
* @internal
*/
virtual void setBreakType(int32_t type);
/**
* Common initialization function, used by constructors and bufferClone.
* (Also used by DictionaryBasedBreakIterator::createBufferClone().)
* @internal
*/
void init();
private:
/**
* This method backs the iterator back up to a "safe position" in the text.
* This is a position that we know, without any context, must be a break position.
* The various calling methods then iterate forward from this safe position to
* the appropriate position to return. (For more information, see the description
* of buildBackwardsStateTable() in RuleBasedBreakIterator.Builder.)
* @param statetable state table used of moving backwards
* @internal
*/
int32_t handlePrevious(const RBBIStateTable *statetable);
/**
* This method is the actual implementation of the next() method. All iteration
* vectors through here. This method initializes the state machine to state 1
* and advances through the text character by character until we reach the end
* of the text or the state machine transitions to state 0. We update our return
* value every time the state machine passes through a possible end state.
* @param statetable state table used of moving forwards
* @internal
*/
int32_t handleNext(const RBBIStateTable *statetable);
protected:
/**
* This is the function that actually implements dictionary-based
* breaking. Covering at least the range from startPos to endPos,
* it checks for dictionary characters, and if it finds them determines
* the appropriate object to deal with them. It may cache found breaks in
* fCachedBreakPositions as it goes. It may well also look at text outside
* the range startPos to endPos.
* If going forward, endPos is the normal Unicode break result, and
* if goind in reverse, startPos is the normal Unicode break result
* @param startPos The start position of a range of text
* @param endPos The end position of a range of text
* @param reverse The call is for the reverse direction
* @internal
*/
int32_t checkDictionary(int32_t startPos, int32_t endPos, UBool reverse);
private:
/**
* This function returns the appropriate LanguageBreakEngine for a
* given character c.
* @param c A character in the dictionary set
* @internal
*/
const LanguageBreakEngine *getLanguageBreakEngine(UChar32 c);
/**
* @internal
*/
void makeRuleStatusValid();
};
//------------------------------------------------------------------------------
//
// Inline Functions Definitions ...
//
//------------------------------------------------------------------------------
inline UBool RuleBasedBreakIterator::operator!=(const BreakIterator& that) const {
return !operator==(that);
}
U_NAMESPACE_END
#endif /* #if !UCONFIG_NO_BREAK_ITERATION */
#endif
Jump to Line
Something went wrong with that request. Please try again.