-
Notifications
You must be signed in to change notification settings - Fork 42
/
NLTokenizer.xml
397 lines (397 loc) · 21.7 KB
/
NLTokenizer.xml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
<Type Name="NLTokenizer" FullName="NaturalLanguage.NLTokenizer">
<TypeSignature Language="C#" Value="public class NLTokenizer : Foundation.NSObject" />
<TypeSignature Language="ILAsm" Value=".class public auto ansi beforefieldinit NLTokenizer extends Foundation.NSObject" />
<TypeSignature Language="DocId" Value="T:NaturalLanguage.NLTokenizer" />
<TypeSignature Language="F#" Value="type NLTokenizer = class
 inherit NSObject" />
<AssemblyInfo>
<AssemblyName>Xamarin.iOS</AssemblyName>
<AssemblyVersion>0.0.0.0</AssemblyVersion>
</AssemblyInfo>
<AssemblyInfo>
<AssemblyName>Xamarin.Mac</AssemblyName>
<AssemblyVersion>0.0.0.0</AssemblyVersion>
</AssemblyInfo>
<Base>
<BaseTypeName>Foundation.NSObject</BaseTypeName>
</Base>
<Interfaces />
<Attributes>
<Attribute>
<AttributeName>Foundation.Register("NLTokenizer", true)</AttributeName>
</Attribute>
<Attribute>
<AttributeName>ObjCRuntime.Introduced(ObjCRuntime.PlatformName.iOS, 12, 0, ObjCRuntime.PlatformArchitecture.All, null)</AttributeName>
</Attribute>
<Attribute>
<AttributeName>ObjCRuntime.Introduced(ObjCRuntime.PlatformName.MacOSX, 10, 14, ObjCRuntime.PlatformArchitecture.Arch64, null)</AttributeName>
</Attribute>
<Attribute>
<AttributeName>ObjCRuntime.Introduced(ObjCRuntime.PlatformName.TvOS, 12, 0, ObjCRuntime.PlatformArchitecture.All, null)</AttributeName>
</Attribute>
<Attribute>
<AttributeName>ObjCRuntime.Introduced(ObjCRuntime.PlatformName.WatchOS, 5, 0, ObjCRuntime.PlatformArchitecture.All, null)</AttributeName>
</Attribute>
</Attributes>
<Docs>
<summary>Breaks a text up into semantic units.</summary>
<remarks>To be added.</remarks>
</Docs>
<Members>
<Member MemberName=".ctor">
<MemberSignature Language="C#" Value="protected NLTokenizer (Foundation.NSObjectFlag t);" />
<MemberSignature Language="ILAsm" Value=".method familyhidebysig specialname rtspecialname instance void .ctor(class Foundation.NSObjectFlag t) cil managed" />
<MemberSignature Language="DocId" Value="M:NaturalLanguage.NLTokenizer.#ctor(Foundation.NSObjectFlag)" />
<MemberSignature Language="F#" Value="new NaturalLanguage.NLTokenizer : Foundation.NSObjectFlag -> NaturalLanguage.NLTokenizer" Usage="new NaturalLanguage.NLTokenizer t" />
<MemberType>Constructor</MemberType>
<AssemblyInfo>
<AssemblyVersion>0.0.0.0</AssemblyVersion>
<AssemblyName>Xamarin.iOS</AssemblyName>
</AssemblyInfo>
<AssemblyInfo>
<AssemblyName>Xamarin.Mac</AssemblyName>
<AssemblyVersion>0.0.0.0</AssemblyVersion>
</AssemblyInfo>
<Attributes>
<Attribute>
<AttributeName>ObjCRuntime.BindingImpl(ObjCRuntime.BindingImplOptions.GeneratedCode | ObjCRuntime.BindingImplOptions.Optimizable)</AttributeName>
</Attribute>
<Attribute>
<AttributeName>System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Advanced)</AttributeName>
</Attribute>
</Attributes>
<Parameters>
<Parameter Name="t" Type="Foundation.NSObjectFlag" />
</Parameters>
<Docs>
<param name="t">Unused sentinel value, pass NSObjectFlag.Empty.</param>
<summary>Constructor to call on derived classes to skip initialization and merely allocate the object.</summary>
<remarks>
<para>This constructor should be called by derived classes when they completely construct the object in managed code and merely want the runtime to allocate and initialize the NSObject. This is required to implement the two-step initialization process that Objective-C uses, the first step is to perform the object allocation, the second step is to initialize the object. When developers invoke the constructor that takes the NSObjectFlag.Empty they take advantage of a direct path that goes all the way up to NSObject to merely allocate the object's memory and bind the Objective-C and C# objects together. The actual initialization of the object is up to the developer.</para>
<para>This constructor is typically used by the binding generator to allocate the object, but prevent the actual initialization to take place. Once the allocation has taken place, the constructor has to initialize the object. With constructors generated by the binding generator this means that it manually invokes one of the "init" methods to initialize the object.</para>
<para>It is the developer's responsibility to completely initialize the object if they chain up using the NSObjectFlag.Empty path.</para>
<para>In general, if the developer's constructor invokes the NSObjectFlag.Empty base implementation, then it should be calling an Objective-C init method. If this is not the case, developers should instead chain to the proper constructor in their class. </para>
<para>The argument value is ignored and merely ensures that the only code that is executed is the construction phase is the basic NSObject allocation and runtime type registration. Typically the chaining would look like this:</para>
<example>
<code lang="csharp lang-csharp"><![CDATA[
//
// The NSObjectFlag merely allocates the object and registers the
// C# class with the Objective-C runtime if necessary, but no actual
// initXxx method is invoked, that is done later in the constructor
//
// This is taken from Xamarin.iOS's source code:
//
[Export ("initWithFrame:")]
public UIView (System.Drawing.RectangleF frame) : base (NSObjectFlag.Empty)
{
// Invoke the init method now.
var initWithFrame = new Selector ("initWithFrame:").Handle;
if (IsDirectBinding)
Handle = ObjCRuntime.Messaging.IntPtr_objc_msgSend_CGRect (this.Handle, initWithFrame, frame);
else
Handle = ObjCRuntime.Messaging.IntPtr_objc_msgSendSuper_CGRect (this.SuperHandle, initWithFrame, frame);
}
]]></code>
</example>
</remarks>
</Docs>
</Member>
<Member MemberName=".ctor">
<MemberSignature Language="C#" Value="public NLTokenizer (NaturalLanguage.NLTokenUnit unit);" />
<MemberSignature Language="ILAsm" Value=".method public hidebysig specialname rtspecialname instance void .ctor(valuetype NaturalLanguage.NLTokenUnit unit) cil managed" />
<MemberSignature Language="DocId" Value="M:NaturalLanguage.NLTokenizer.#ctor(NaturalLanguage.NLTokenUnit)" />
<MemberSignature Language="F#" Value="new NaturalLanguage.NLTokenizer : NaturalLanguage.NLTokenUnit -> NaturalLanguage.NLTokenizer" Usage="new NaturalLanguage.NLTokenizer unit" />
<MemberType>Constructor</MemberType>
<AssemblyInfo>
<AssemblyVersion>0.0.0.0</AssemblyVersion>
<AssemblyName>Xamarin.iOS</AssemblyName>
</AssemblyInfo>
<AssemblyInfo>
<AssemblyName>Xamarin.Mac</AssemblyName>
<AssemblyVersion>0.0.0.0</AssemblyVersion>
</AssemblyInfo>
<Attributes>
<Attribute>
<AttributeName>Foundation.Export("initWithUnit:")</AttributeName>
</Attribute>
<Attribute>
<AttributeName>ObjCRuntime.BindingImpl(ObjCRuntime.BindingImplOptions.GeneratedCode | ObjCRuntime.BindingImplOptions.Optimizable)</AttributeName>
</Attribute>
<Attribute>
<AttributeName>ObjCRuntime.DesignatedInitializer</AttributeName>
</Attribute>
</Attributes>
<Parameters>
<Parameter Name="unit" Type="NaturalLanguage.NLTokenUnit" />
</Parameters>
<Docs>
<param name="unit">The unit into which the tokenizer will separate text.</param>
<summary>Creates a new tokenizer that breaks text up into the specified semantic <paramref name="unit" />s.</summary>
<remarks>To be added.</remarks>
</Docs>
</Member>
<Member MemberName=".ctor">
<MemberSignature Language="C#" Value="protected internal NLTokenizer (IntPtr handle);" />
<MemberSignature Language="ILAsm" Value=".method familyorassemblyhidebysig specialname rtspecialname instance void .ctor(native int handle) cil managed" />
<MemberSignature Language="DocId" Value="M:NaturalLanguage.NLTokenizer.#ctor(System.IntPtr)" />
<MemberSignature Language="F#" Value="new NaturalLanguage.NLTokenizer : nativeint -> NaturalLanguage.NLTokenizer" Usage="new NaturalLanguage.NLTokenizer handle" />
<MemberType>Constructor</MemberType>
<AssemblyInfo>
<AssemblyVersion>0.0.0.0</AssemblyVersion>
<AssemblyName>Xamarin.iOS</AssemblyName>
</AssemblyInfo>
<AssemblyInfo>
<AssemblyName>Xamarin.Mac</AssemblyName>
<AssemblyVersion>0.0.0.0</AssemblyVersion>
</AssemblyInfo>
<Attributes>
<Attribute>
<AttributeName>ObjCRuntime.BindingImpl(ObjCRuntime.BindingImplOptions.GeneratedCode | ObjCRuntime.BindingImplOptions.Optimizable)</AttributeName>
</Attribute>
<Attribute>
<AttributeName>System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Advanced)</AttributeName>
</Attribute>
</Attributes>
<Parameters>
<Parameter Name="handle" Type="System.IntPtr" />
</Parameters>
<Docs>
<param name="handle">Pointer (handle) to the unmanaged object.</param>
<summary>A constructor used when creating managed representations of unmanaged objects; Called by the runtime.</summary>
<remarks>
<para>This constructor is invoked by the runtime infrastructure (<see cref="M:ObjCRuntime.Runtime.GetNSObject(System.IntPtr)" />) to create a new managed representation for a pointer to an unmanaged Objective-C object. Developers should not invoke this method directly, instead they should call the GetNSObject method as it will prevent two instances of a managed object to point to the same native object.</para>
</remarks>
</Docs>
</Member>
<Member MemberName="ClassHandle">
<MemberSignature Language="C#" Value="public override IntPtr ClassHandle { get; }" />
<MemberSignature Language="ILAsm" Value=".property instance native int ClassHandle" />
<MemberSignature Language="DocId" Value="P:NaturalLanguage.NLTokenizer.ClassHandle" />
<MemberSignature Language="F#" Value="member this.ClassHandle : nativeint" Usage="NaturalLanguage.NLTokenizer.ClassHandle" />
<MemberType>Property</MemberType>
<AssemblyInfo>
<AssemblyVersion>0.0.0.0</AssemblyVersion>
<AssemblyName>Xamarin.iOS</AssemblyName>
</AssemblyInfo>
<AssemblyInfo>
<AssemblyName>Xamarin.Mac</AssemblyName>
<AssemblyVersion>0.0.0.0</AssemblyVersion>
</AssemblyInfo>
<ReturnValue>
<ReturnType>System.IntPtr</ReturnType>
</ReturnValue>
<Docs>
<summary>The handle for this class.</summary>
<value>The pointer to the Objective-C class.</value>
<remarks>Each Xamarin.iOS class mirrors an unmanaged Objective-C class. This value contains the pointer to the Objective-C class, it is similar to calling objc_getClass with the object name.</remarks>
</Docs>
</Member>
<Member MemberName="EnumerateTokens">
<MemberSignature Language="C#" Value="public virtual void EnumerateTokens (Foundation.NSRange range, NaturalLanguage.NLTokenizerEnumerateContinuationHandler handler);" />
<MemberSignature Language="ILAsm" Value=".method public hidebysig newslot virtual instance void EnumerateTokens(valuetype Foundation.NSRange range, class NaturalLanguage.NLTokenizerEnumerateContinuationHandler handler) cil managed" />
<MemberSignature Language="DocId" Value="M:NaturalLanguage.NLTokenizer.EnumerateTokens(Foundation.NSRange,NaturalLanguage.NLTokenizerEnumerateContinuationHandler)" />
<MemberSignature Language="F#" Value="abstract member EnumerateTokens : Foundation.NSRange * NaturalLanguage.NLTokenizerEnumerateContinuationHandler -> unit
override this.EnumerateTokens : Foundation.NSRange * NaturalLanguage.NLTokenizerEnumerateContinuationHandler -> unit" Usage="nLTokenizer.EnumerateTokens (range, handler)" />
<MemberType>Method</MemberType>
<AssemblyInfo>
<AssemblyVersion>0.0.0.0</AssemblyVersion>
<AssemblyName>Xamarin.iOS</AssemblyName>
</AssemblyInfo>
<AssemblyInfo>
<AssemblyName>Xamarin.Mac</AssemblyName>
<AssemblyVersion>0.0.0.0</AssemblyVersion>
</AssemblyInfo>
<Attributes>
<Attribute>
<AttributeName>Foundation.Export("enumerateTokensInRange:usingBlock:")</AttributeName>
</Attribute>
<Attribute>
<AttributeName>ObjCRuntime.BindingImpl(ObjCRuntime.BindingImplOptions.GeneratedCode | ObjCRuntime.BindingImplOptions.Optimizable)</AttributeName>
</Attribute>
</Attributes>
<ReturnValue>
<ReturnType>System.Void</ReturnType>
</ReturnValue>
<Parameters>
<Parameter Name="range" Type="Foundation.NSRange" />
<Parameter Name="handler" Type="NaturalLanguage.NLTokenizerEnumerateContinuationHandler">
<Attributes>
<Attribute FrameworkAlternate="xamarin-ios-sdk-12">
<AttributeName>ObjCRuntime.BlockProxy(typeof(ObjCRuntime.Trampolines/NIDNLTokenizerEnumerateContinuationHandler))</AttributeName>
</Attribute>
</Attributes>
</Parameter>
</Parameters>
<Docs>
<param name="range">The lexical range over which to get tokens.</param>
<param name="handler">A handler to run on each token.</param>
<summary>Enumerates tokens for the specified range in the text.</summary>
<remarks>To be added.</remarks>
</Docs>
</Member>
<Member MemberName="GetTokenRange">
<MemberSignature Language="C#" Value="public virtual Foundation.NSRange GetTokenRange (nuint characterIndex);" />
<MemberSignature Language="ILAsm" Value=".method public hidebysig newslot virtual instance valuetype Foundation.NSRange GetTokenRange(valuetype System.nuint characterIndex) cil managed" />
<MemberSignature Language="DocId" Value="M:NaturalLanguage.NLTokenizer.GetTokenRange(System.nuint)" />
<MemberSignature Language="F#" Value="abstract member GetTokenRange : nuint -> Foundation.NSRange
override this.GetTokenRange : nuint -> Foundation.NSRange" Usage="nLTokenizer.GetTokenRange characterIndex" />
<MemberType>Method</MemberType>
<AssemblyInfo>
<AssemblyVersion>0.0.0.0</AssemblyVersion>
<AssemblyName>Xamarin.iOS</AssemblyName>
</AssemblyInfo>
<AssemblyInfo>
<AssemblyName>Xamarin.Mac</AssemblyName>
<AssemblyVersion>0.0.0.0</AssemblyVersion>
</AssemblyInfo>
<Attributes>
<Attribute>
<AttributeName>Foundation.Export("tokenRangeAtIndex:")</AttributeName>
</Attribute>
<Attribute>
<AttributeName>ObjCRuntime.BindingImpl(ObjCRuntime.BindingImplOptions.GeneratedCode | ObjCRuntime.BindingImplOptions.Optimizable)</AttributeName>
</Attribute>
</Attributes>
<ReturnValue>
<ReturnType>Foundation.NSRange</ReturnType>
</ReturnValue>
<Parameters>
<Parameter Name="characterIndex" Type="System.nuint" />
</Parameters>
<Docs>
<param name="characterIndex">The index of a character that is covered by a token.</param>
<summary>Gets the range of the token that covers the specified character index.</summary>
<returns>The range of the token that covers the specified character index.</returns>
<remarks>To be added.</remarks>
</Docs>
</Member>
<Member MemberName="GetTokens">
<MemberSignature Language="C#" Value="public virtual Foundation.NSValue[] GetTokens (Foundation.NSRange range);" />
<MemberSignature Language="ILAsm" Value=".method public hidebysig newslot virtual instance class Foundation.NSValue[] GetTokens(valuetype Foundation.NSRange range) cil managed" />
<MemberSignature Language="DocId" Value="M:NaturalLanguage.NLTokenizer.GetTokens(Foundation.NSRange)" />
<MemberSignature Language="F#" Value="abstract member GetTokens : Foundation.NSRange -> Foundation.NSValue[]
override this.GetTokens : Foundation.NSRange -> Foundation.NSValue[]" Usage="nLTokenizer.GetTokens range" />
<MemberType>Method</MemberType>
<AssemblyInfo>
<AssemblyVersion>0.0.0.0</AssemblyVersion>
<AssemblyName>Xamarin.iOS</AssemblyName>
</AssemblyInfo>
<AssemblyInfo>
<AssemblyName>Xamarin.Mac</AssemblyName>
<AssemblyVersion>0.0.0.0</AssemblyVersion>
</AssemblyInfo>
<Attributes>
<Attribute>
<AttributeName>Foundation.Export("tokensForRange:")</AttributeName>
</Attribute>
<Attribute>
<AttributeName>ObjCRuntime.BindingImpl(ObjCRuntime.BindingImplOptions.GeneratedCode | ObjCRuntime.BindingImplOptions.Optimizable)</AttributeName>
</Attribute>
</Attributes>
<ReturnValue>
<ReturnType>Foundation.NSValue[]</ReturnType>
</ReturnValue>
<Parameters>
<Parameter Name="range" Type="Foundation.NSRange" />
</Parameters>
<Docs>
<param name="range">The range for which to return all tokens.</param>
<summary>Tokenizes the specified range of text.</summary>
<returns>Tokens for the specified range.</returns>
<remarks>To be added.</remarks>
</Docs>
</Member>
<Member MemberName="SetLanguage">
<MemberSignature Language="C#" Value="public void SetLanguage (NaturalLanguage.NLLanguage language);" />
<MemberSignature Language="ILAsm" Value=".method public hidebysig instance void SetLanguage(valuetype NaturalLanguage.NLLanguage language) cil managed" />
<MemberSignature Language="DocId" Value="M:NaturalLanguage.NLTokenizer.SetLanguage(NaturalLanguage.NLLanguage)" />
<MemberSignature Language="F#" Value="member this.SetLanguage : NaturalLanguage.NLLanguage -> unit" Usage="nLTokenizer.SetLanguage language" />
<MemberType>Method</MemberType>
<AssemblyInfo>
<AssemblyVersion>0.0.0.0</AssemblyVersion>
<AssemblyName>Xamarin.iOS</AssemblyName>
</AssemblyInfo>
<AssemblyInfo>
<AssemblyName>Xamarin.Mac</AssemblyName>
<AssemblyVersion>0.0.0.0</AssemblyVersion>
</AssemblyInfo>
<Attributes>
<Attribute>
<AttributeName>ObjCRuntime.BindingImpl(ObjCRuntime.BindingImplOptions.GeneratedCode | ObjCRuntime.BindingImplOptions.Optimizable)</AttributeName>
</Attribute>
</Attributes>
<ReturnValue>
<ReturnType>System.Void</ReturnType>
</ReturnValue>
<Parameters>
<Parameter Name="language" Type="NaturalLanguage.NLLanguage" />
</Parameters>
<Docs>
<param name="language">The language value to set.</param>
<summary>Sets the language that the tokenizer will use when processing the string.</summary>
<remarks>To be added.</remarks>
</Docs>
</Member>
<Member MemberName="String">
<MemberSignature Language="C#" Value="public virtual string String { get; set; }" />
<MemberSignature Language="ILAsm" Value=".property instance string String" />
<MemberSignature Language="DocId" Value="P:NaturalLanguage.NLTokenizer.String" />
<MemberSignature Language="F#" Value="member this.String : string with get, set" Usage="NaturalLanguage.NLTokenizer.String" />
<MemberType>Property</MemberType>
<AssemblyInfo>
<AssemblyVersion>0.0.0.0</AssemblyVersion>
<AssemblyName>Xamarin.iOS</AssemblyName>
</AssemblyInfo>
<AssemblyInfo>
<AssemblyName>Xamarin.Mac</AssemblyName>
<AssemblyVersion>0.0.0.0</AssemblyVersion>
</AssemblyInfo>
<Attributes>
<Attribute>
<AttributeName>ObjCRuntime.BindingImpl(ObjCRuntime.BindingImplOptions.GeneratedCode | ObjCRuntime.BindingImplOptions.Optimizable)</AttributeName>
</Attribute>
<Attribute>
<AttributeName>get: Foundation.Export("string", ObjCRuntime.ArgumentSemantic.Strong)</AttributeName>
</Attribute>
<Attribute>
<AttributeName>set: Foundation.Export("setString:", ObjCRuntime.ArgumentSemantic.Strong)</AttributeName>
</Attribute>
</Attributes>
<ReturnValue>
<ReturnType>System.String</ReturnType>
</ReturnValue>
<Docs>
<summary>Gets or sets the string to process.</summary>
<value>The string to process.</value>
<remarks>To be added.</remarks>
</Docs>
</Member>
<Member MemberName="Unit">
<MemberSignature Language="C#" Value="public virtual NaturalLanguage.NLTokenUnit Unit { get; }" />
<MemberSignature Language="ILAsm" Value=".property instance valuetype NaturalLanguage.NLTokenUnit Unit" />
<MemberSignature Language="DocId" Value="P:NaturalLanguage.NLTokenizer.Unit" />
<MemberSignature Language="F#" Value="member this.Unit : NaturalLanguage.NLTokenUnit" Usage="NaturalLanguage.NLTokenizer.Unit" />
<MemberType>Property</MemberType>
<AssemblyInfo>
<AssemblyVersion>0.0.0.0</AssemblyVersion>
<AssemblyName>Xamarin.iOS</AssemblyName>
</AssemblyInfo>
<AssemblyInfo>
<AssemblyName>Xamarin.Mac</AssemblyName>
<AssemblyVersion>0.0.0.0</AssemblyVersion>
</AssemblyInfo>
<Attributes>
<Attribute>
<AttributeName>ObjCRuntime.BindingImpl(ObjCRuntime.BindingImplOptions.GeneratedCode | ObjCRuntime.BindingImplOptions.Optimizable)</AttributeName>
</Attribute>
<Attribute>
<AttributeName>get: Foundation.Export("unit")</AttributeName>
</Attribute>
</Attributes>
<ReturnValue>
<ReturnType>NaturalLanguage.NLTokenUnit</ReturnType>
</ReturnValue>
<Docs>
<summary>Gets the semantic unit of the tokens that this tokenizer returns.</summary>
<value>The semantic unit of the tokens that this tokenizer returns.</value>
<remarks>To be added.</remarks>
</Docs>
</Member>
</Members>
</Type>