-
Notifications
You must be signed in to change notification settings - Fork 0
/
codepoint_iterator.hxx
273 lines (261 loc) · 8.87 KB
/
codepoint_iterator.hxx
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
#pragma once
#include <assert.h>
#include <cstdint>
#include <iterator>
namespace beam_me_up {}
namespace bmu = beam_me_up;
namespace beam_me_up {
typedef std::uint8_t u8unit_t;
typedef std::uint16_t u16unit_t;
typedef std::uint32_t u32char_t;
const u32char_t INVALID_CODEPOINT(-1);
namespace detail {
const u32char_t MAX_CODEPOINT(0x0010ffffu);///< Maximum valid value for a Unicode code point
inline bool isSubOctet(u8unit_t val)
{
return ((val & 0xC0) == 0x80);
}
}
template<typename _OutIt>
size_t putUTF8Octets(u32char_t cp, _OutIt it, size_t const outOctetsCount)
{
typedef typename std::iterator_traits<_OutIt>::iterator_category iterator_category;
static_assert(std::is_base_of<std::output_iterator_tag, iterator_category>::value || std::is_base_of<std::random_access_iterator_tag, iterator_category>::value
, "putUTF8Octets requires output iterator");
if (cp > detail::MAX_CODEPOINT)
return 0;
unsigned n = 0;
if (cp < 0x80) // 0xxx xxxxx (septet0), at most 7 bits (0x7f)
n = 1;
else if (cp < 0x800) // 110x xxxx 10xx xxxx (quintet1 sextet0), at most 5 + 6 = 11 bits (0x7ff)
n = 2;
else if (cp < 0x10000) // 1110 xxxx 10xx xxxx 10xx xxxx (quartet2 sextet1 sextet0), at most 4 + 2 * 6 = 16 bits (0xffff)
n = 3;
else // 1111 0xxx 10xx xxxx 10xx xxxx 10xx xxxx (trio3 sextet2 sextet1 sextet0), at most 3 + 3 * 6 = 21 bits (0x1fffff)
n = 4;
if (outOctetsCount < n) // do nothing if there's not enough room
return 0;
u8unit_t seq[4];
for (size_t i = n - 1; i != 0; --i) { //from last until 2nd octet
seq[i] = 0x80 | (cp & 0x3f);
cp >>= 6;
}
static u8unit_t const lead1st[5]{ 0, 0, 0xC0, 0xE0, 0xF0 };
static u8unit_t const mask1st[5]{ 0, 0xff, 0x1f, 0x0f, 0x07 };
seq[0] = lead1st[n] | (cp & mask1st[n]); // 1st octet
for (size_t i = 0; i < n; ++i, ++it) //copy octets to output
*it = seq[i];
return n;
}
template<typename _Container>
inline size_t putUTF8Octets(u32char_t cp, std::back_insert_iterator<_Container> it)
{
return putUTF8Octets(cp, it, 4);
}
//Trail-bytes always start with "10" as the top-bits, with the lower 6 bits being used to store the remaining bits from the Unicode value.
//A trail-byte must always follow a lead-byte - it cannot ever appear on its own. So, a Unicode value in the range 0-127 is represented as-is.
//Values outside of this range (0x80 - 0x10FFFF) are represented using a multibyte sequence, comprising exactly one lead-byte and one-or-more trail-bytes.
template<class _FwdIt>
u32char_t getUTF8Codepoint(_FwdIt it, _FwdIt const end, typename std::iterator_traits<_FwdIt>::difference_type& nOctets)
{
typedef typename std::iterator_traits<_FwdIt>::iterator_category iterator_category;
static_assert(std::is_base_of<std::forward_iterator_tag, iterator_category>::value
, "getUTF8Codepoint requires forward iterator");
u8unit_t const by0(static_cast<u8unit_t>(*it & 0xFF));
if (by0 < 0x80)
nOctets = 1;
else if ((by0 & 0xE0) == 0xC0)
nOctets = 2;
else if ((by0 & 0xF0) == 0xE0)
nOctets = 3;
else if ((by0 & 0xF8) == 0xF0)
nOctets = 4;
else
return INVALID_CODEPOINT;
if (end - it < nOctets)
return INVALID_CODEPOINT;
if (1 == nOctets) // 0xxx xxxxx (septet0), at most 7 bits (0x7f)
return by0;
u8unit_t const by1(static_cast<u8unit_t>(*++it & 0xFF));
if (!detail::isSubOctet(by1))
return INVALID_CODEPOINT;
if (2 == nOctets) // 110x xxxx 10xx xxxx (quintet1 sextet0), at most 5 + 6 = 11 bits (0x7ff)
return ((by0 & 0x1f) << 6) | (by1 & 0x3f);
u8unit_t const by2(static_cast<u8unit_t>(*++it & 0xFF));
if (!detail::isSubOctet(by2))
return INVALID_CODEPOINT;
if (3 == nOctets) // 1110 xxxx 10xx xxxx 10xx xxxx (quartet2 sextet1 sextet0), at most 4 + 2 * 6 = 16 bits (0xffff)
return (u32char_t(by0 & 0x0f) << 12) | (u32char_t(by1 & 0x3f) << 6) | (by2 & 0x3f);
u8unit_t const by3(static_cast<u8unit_t>(*++it & 0xFF));
if (!detail::isSubOctet(by3))
return INVALID_CODEPOINT;
if (4 == nOctets) // 1111 0xxx 10xx xxxx 10xx xxxx 10xx xxxx (trio3 sextet2 sextet1 sextet0), at most 3 + 3 * 6 = 21 bits (0x1fffff)
return (u32char_t(by0 & 0x07) << 18) | (u32char_t(by1 & 0x3f) << 12) | (u32char_t(by2 & 0x3f) << 6) | (by3 & 0x3f);
return INVALID_CODEPOINT;
}
template<typename _OutIt>
size_t putUTF16Words(u32char_t cp, _OutIt it, size_t const outWordsCount)
{
typedef typename std::iterator_traits<_OutIt>::iterator_category iterator_category;
static_assert(std::is_base_of<std::output_iterator_tag, iterator_category>::value || std::is_base_of<std::random_access_iterator_tag, iterator_category>::value
, "putUTF8Octets requires output iterator");
static_assert(sizeof(typename std::iterator_traits<_OutIt>::value_type) == 2, "Expected double-byte values");
if (cp > detail::MAX_CODEPOINT)
return 0;
unsigned n = 0;
if (cp > 0xffff) { //make a surrogate pair
if (outWordsCount < 2)
return 0;
*it = static_cast<u16unit_t>(0xd800u | ((cp - 0x10000) >> 10));
*++it = static_cast<u16unit_t>(0xdc00u | (cp & 0x3ff));
return 2;
}
if (outWordsCount < 1)
return 0;
*it = cp;
return 1;
}
template<class _FwdIt>
u32char_t getUTF16Codepoint(_FwdIt it, _FwdIt const end, typename std::iterator_traits<_FwdIt>::difference_type& nWords)
{
typedef typename std::iterator_traits<_FwdIt>::iterator_category iterator_category;
static_assert(std::is_base_of<std::forward_iterator_tag, iterator_category>::value
, "getUTF16Codepoint requires forward iterator");
static_assert(sizeof(typename std::iterator_traits<_FwdIt>::value_type) == 2, "Expected double-byte values");
u32char_t const w1(*it & 0xFFFF);
u32char_t cp = w1;
nWords = 1;
bool const bIsSurrogatePair = (0xd800u <= w1 && w1 <= 0xdbffu);// Take care of surrogate pairs first
if (bIsSurrogatePair && (it + 1) != end) {
u32char_t const w2(*(it + 1) & 0xFFFF); // don't yet increment iterator just peek next value
if (0xdc00u <= w2 && w2 <= 0xdfffu) { // this is surrogate pair
static u32char_t const mask10 = (1 << 10) - 1;
cp = 0x10000 + (((w1 & mask10) << 10) | (w2 & mask10));
if (cp <= detail::MAX_CODEPOINT) {
++it; // skip already read low surrogate
nWords = 2;
}
else
cp = w1;
}
}
return cp;
}
//rezultat je Unicode Codepoint. Rezultat INVALID_CODEPOINT je indikacija greske i tada je it == end
template <typename _OctetIterator>
class codepoint_iterator : public std::iterator <std::bidirectional_iterator_tag, u32char_t> {
void setInvalid(void)
{
__cp_length = 0;
__cp = INVALID_CODEPOINT;
__it = __end;
}
void dereferenceCodepoint(void);
void goToNextCodepoint(void)
{
std::advance(__it, __cp_length);
dereferenceCodepoint();
}
void goToPrevCodepoint(void)
{
if (__it == __start)
return setInvalid(); // can't go back
while (__it != __start && detail::isSubOctet(*(--__it))); // go back skipping suboctets
if (detail::isSubOctet(*__it))
return setInvalid();
dereferenceCodepoint();
}
public:
explicit codepoint_iterator()
: __cp_length(0)
, __cp(INVALID_CODEPOINT)
, __it()
, __start()
, __end()
{
static_assert(sizeof(typename _OctetIterator::value_type) == 1, "Expected single-byte values");
}
explicit codepoint_iterator(_OctetIterator __it,
_OctetIterator __itstart,
_OctetIterator __itend)
: __cp_length(0)
, __cp(INVALID_CODEPOINT)
, __it(__it)
, __start(__itstart)
, __end(__itend)
{
assert(__it >= __start);
assert(__it <= __end);
dereferenceCodepoint();
}
_OctetIterator base() const
{
return __it;
}
u32char_t operator * () const
{
assert((__cp != INVALID_CODEPOINT && __cp_length != 0 && __it != __end)
|| (__cp == INVALID_CODEPOINT && __cp_length == 0 && __it == __end));
return __cp;
}
operator bool (void) const
{
bool bOK = INVALID_CODEPOINT != __cp && __it != __end;
return INVALID_CODEPOINT != __cp && __it != __end;
}
bool operator ! (void) const
{
return INVALID_CODEPOINT == __cp || __it == __end;
}
bool operator == (codepoint_iterator const& __other) const
{
assert(__start == __other.__start && __end == __other.__end);
return (__it == __other.__it);
}
bool operator != (codepoint_iterator const& __other) const
{
return !(operator == (__other));
}
codepoint_iterator& operator ++ () // prefix
{
goToNextCodepoint();
return *this;
}
codepoint_iterator operator ++ (int) // postfix
{
codepoint_iterator const __tmp(*this);
goToNextCodepoint();
return __tmp;
}
codepoint_iterator& operator -- ()
{
goToPrevCodepoint();
return *this;
}
codepoint_iterator operator -- (int)
{
codepoint_iterator const __tmp(*this);
goToPrevCodepoint();
return __tmp;
}
private:
difference_type __cp_length;
u32char_t __cp;
_OctetIterator __it;
_OctetIterator __start;
_OctetIterator __end;
};
template <typename _OctetIterator>
void codepoint_iterator<_OctetIterator>::dereferenceCodepoint(void)
{
if (__it == __end) {
setInvalid();
return;
}
difference_type cpLen = 0;
__cp = getUTF8Codepoint<_OctetIterator>(__it, __end, cpLen);
__cp_length = cpLen;
if (INVALID_CODEPOINT == __cp)
return setInvalid();
}
}