forked from ChatScript/ChatScript
-
Notifications
You must be signed in to change notification settings - Fork 12
/
Copy pathtokenSystem.cpp
2339 lines (2134 loc) · 90 KB
/
tokenSystem.cpp
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#include "common.h"
#ifdef INFORMATION
SPACES space \t \r \n
PUNCTUATIONS , | - (see also ENDERS)
ENDERS . ; : ? ! -
BRACKETS () [ ] { } < >
ARITHMETICS % * + - ^ = / .
SYMBOLS $ # @ ~
CONVERTERS & `
//NORMALS A-Z a-z 0-9 _ and sometimes /
#endif
int inputNest = 0;
#define MAX_BURST 400
static char burstWords[MAX_BURST][MAX_WORD_SIZE]; // each token burst from a text string
static unsigned int burstLimit = 0; // index of burst words
uint64 tokenFlags; // what tokenization saw
char* wordStarts[MAX_SENTENCE_LENGTH]; // current sentence tokenization (always points to D->word values or allocated values)
int wordCount; // how many words/tokens in sentence
bool capState[MAX_SENTENCE_LENGTH];
bool originalCapState[MAX_SENTENCE_LENGTH]; // was input word capitalized by user
void ResetTokenSystem()
{
tokenFlags = 0;
wordStarts[0] = AllocateHeap((char*)"");
wordCount = 0;
memset(wordStarts,0,sizeof(char*)*MAX_SENTENCE_LENGTH); // reinit for new volley - sharing of word space can occur throughout this volley
wordStarts[0] = ""; // underflow protection
ClearWhereInSentence();
memset(concepts, 0, sizeof(concepts)); // concept chains per word
memset(topics, 0, sizeof(concepts)); // concept chains per word
}
void DumpResponseControls(uint64 val)
{
if (val & RESPONSE_UPPERSTART) Log(STDTRACELOG,(char*)"RESPONSE_UPPERSTART ");
if (val & RESPONSE_REMOVESPACEBEFORECOMMA) Log(STDTRACELOG,(char*)"RESPONSE_REMOVESPACEBEFORECOMMA ");
if (val & RESPONSE_ALTERUNDERSCORES) Log(STDTRACELOG,(char*)"RESPONSE_ALTERUNDERSCORES ");
if (val & RESPONSE_REMOVETILDE) Log(STDTRACELOG, (char*)"RESPONSE_REMOVETILDE ");
if (val & RESPONSE_NOCONVERTSPECIAL) Log(STDTRACELOG, (char*)"RESPONSE_NOCONVERTSPECIAL ");
if (val & RESPONSE_CURLYQUOTES) Log(STDTRACELOG, (char*)"RESPONSE_CURLYQUOTES ");
}
void DumpTokenControls(uint64 val)
{
if ((val & DO_SUBSTITUTE_SYSTEM) == DO_SUBSTITUTE_SYSTEM) Log(STDTRACELOG,(char*)"DO_SUBSTITUTE_SYSTEM ");
else // partials
{
if (val & DO_ESSENTIALS) Log(STDTRACELOG,(char*)"DO_ESSENTIALS ");
if (val & DO_SUBSTITUTES) Log(STDTRACELOG,(char*)"DO_SUBSTITUTES ");
if (val & DO_CONTRACTIONS) Log(STDTRACELOG,(char*)"DO_CONTRACTIONS ");
if (val & DO_INTERJECTIONS) Log(STDTRACELOG,(char*)"DO_INTERJECTIONS ");
if (val & DO_BRITISH) Log(STDTRACELOG,(char*)"DO_BRITISH ");
if (val & DO_SPELLING) Log(STDTRACELOG,(char*)"DO_SPELLING ");
if (val & DO_TEXTING) Log(STDTRACELOG,(char*)"DO_TEXTING ");
if (val & DO_NOISE) Log(STDTRACELOG,(char*)"DO_NOISE ");
}
if (val & DO_PRIVATE) Log(STDTRACELOG,(char*)"DO_PRIVATE ");
// reserved
if (val & DO_NUMBER_MERGE) Log(STDTRACELOG,(char*)"DO_NUMBER_MERGE ");
if (val & DO_PROPERNAME_MERGE) Log(STDTRACELOG,(char*)"DO_PROPERNAME_MERGE ");
if (val & DO_DATE_MERGE) Log(STDTRACELOG,(char*)"DO_DATE_MERGE ");
if (val & NO_PROPER_SPELLCHECK) Log(STDTRACELOG,(char*)"NO_PROPER_SPELLCHECK ");
if (val & NO_LOWERCASE_PROPER_MERGE) Log(STDTRACELOG,(char*)"NO_LOWERCASE_PROPER_MERGE ");
if (val & DO_SPELLCHECK) Log(STDTRACELOG,(char*)"DO_SPELLCHECK ");
if (val & DO_INTERJECTION_SPLITTING) Log(STDTRACELOG,(char*)"DO_INTERJECTION_SPLITTING ");
if (val & DO_SPLIT_UNDERSCORE) Log(STDTRACELOG,(char*)"DO_SPLIT_UNDERSCORE ");
if (val & MARK_LOWER) Log(STDTRACELOG,(char*)"MARK_LOWER ");
if ((val & DO_PARSE) == DO_PARSE) Log(STDTRACELOG,(char*)"DO_PARSE ");
else if (val & DO_POSTAG) Log(STDTRACELOG,(char*)"DO_POSTAG ");
if (val & NO_IMPERATIVE) Log(STDTRACELOG,(char*)"NO_IMPERATIVE ");
if (val & NO_WITHIN) Log(STDTRACELOG,(char*)"NO_WITHIN ");
if (val & NO_SENTENCE_END) Log(STDTRACELOG,(char*)"NO_SENTENCE_END ");
if (val & NO_HYPHEN_END) Log(STDTRACELOG,(char*)"NO_HYPHEN_END ");
if (val & NO_COLON_END) Log(STDTRACELOG,(char*)"NO_COLON_END ");
if (val & NO_SEMICOLON_END) Log(STDTRACELOG,(char*)"NO_SEMICOLON_END ");
if (val & STRICT_CASING) Log(STDTRACELOG,(char*)"STRICT_CASING ");
if (val & ONLY_LOWERCASE) Log(STDTRACELOG,(char*)"ONLY_LOWERCASE ");
if (val & TOKEN_AS_IS) Log(STDTRACELOG,(char*)"TOKEN_AS_IS ");
if (val & SPLIT_QUOTE) Log(STDTRACELOG,(char*)"SPLIT_QUOTE ");
if (val & LEAVE_QUOTE) Log(STDTRACELOG,(char*)"LEAVE_QUOTE ");
if (val & UNTOUCHED_INPUT) Log(STDTRACELOG,(char*)"UNTOUCHED_INPUT ");
if (val & NO_FIX_UTF) Log(STDTRACELOG,(char*)"NO_FIX_UTF ");
if (val & NO_CONDITIONAL_IDIOM) Log(STDTRACELOG, (char*)"NO_CONDITIONAL_IDIOM ");
}
void DumpTokenFlags(char* msg)
{
Log(STDTRACELOG,(char*)"%s TokenFlags: ",msg);
// DID THESE
if (tokenFlags & DO_ESSENTIALS) Log(STDTRACELOG,(char*)"DO_ESSENTIALS ");
if (tokenFlags & DO_SUBSTITUTES) Log(STDTRACELOG,(char*)"DO_SUBSTITUTES ");
if (tokenFlags & DO_CONTRACTIONS) Log(STDTRACELOG,(char*)"DO_CONTRACTIONS ");
if (tokenFlags & DO_INTERJECTIONS) Log(STDTRACELOG,(char*)"DO_INTERJECTIONS ");
if (tokenFlags & DO_BRITISH) Log(STDTRACELOG,(char*)"DO_BRITISH ");
if (tokenFlags & DO_SPELLING) Log(STDTRACELOG,(char*)"DO_SPELLING ");
if (tokenFlags & DO_TEXTING) Log(STDTRACELOG,(char*)"DO_TEXTING ");
if (tokenFlags & DO_PRIVATE) Log(STDTRACELOG,(char*)"DO_PRIVATE ");
// reserved
if (tokenFlags & DO_NUMBER_MERGE) Log(STDTRACELOG,(char*)"NUMBER_MERGE ");
if (tokenFlags & DO_PROPERNAME_MERGE) Log(STDTRACELOG,(char*)"PROPERNAME_MERGE ");
if (tokenFlags & DO_DATE_MERGE) Log(STDTRACELOG,(char*)"DATE_MERGE ");
if (tokenFlags & DO_SPELLCHECK) Log(STDTRACELOG,(char*)"SPELLCHECK ");
// FOUND THESE
if (tokenFlags & NO_HYPHEN_END) Log(STDTRACELOG,(char*)"HYPHEN_END ");
if (tokenFlags & NO_COLON_END) Log(STDTRACELOG,(char*)"COLON_END ");
if (tokenFlags & PRESENT) Log(STDTRACELOG,(char*)"PRESENT ");
if (tokenFlags & PAST) Log(STDTRACELOG,(char*)"PAST ");
if (tokenFlags & FUTURE) Log(STDTRACELOG,(char*)"FUTURE ");
if (tokenFlags & PERFECT) Log(STDTRACELOG,(char*)"PERFECT ");
if (tokenFlags & PRESENT_PERFECT) Log(STDTRACELOG,(char*)"PRESENT_PERFECT ");
if (tokenFlags & CONTINUOUS) Log(STDTRACELOG,(char*)"CONTINUOUS ");
if (tokenFlags & PASSIVE) Log(STDTRACELOG,(char*)"PASSIVE ");
if (tokenFlags & QUESTIONMARK) Log(STDTRACELOG,(char*)"QUESTIONMARK ");
if (tokenFlags & EXCLAMATIONMARK) Log(STDTRACELOG,(char*)"EXCLAMATIONMARK ");
if (tokenFlags & PERIODMARK) Log(STDTRACELOG,(char*)"PERIODMARK ");
if (tokenFlags & USERINPUT) Log(STDTRACELOG,(char*)"USERINPUT ");
if (tokenFlags & FAULTY_PARSE) Log(STDTRACELOG,(char*)"FAULTY_PARSE ");
if (tokenFlags & COMMANDMARK) Log(STDTRACELOG,(char*)"COMMANDMARK ");
if (tokenFlags & QUOTATION) Log(STDTRACELOG,(char*)"QUOTATION ");
if (tokenFlags & IMPLIED_YOU) Log(STDTRACELOG,(char*)"IMPLIED_YOU ");
if (tokenFlags & NOT_SENTENCE) Log(STDTRACELOG,(char*)"NOT_SENTENCE ");
if (inputNest) Log(STDTRACELOG,(char*)" ^input ");
if (tokenFlags & NO_CONDITIONAL_IDIOM) Log(STDTRACELOG, (char*)"CONDITIONAL_IDIOM ");
Log(STDTRACELOG,(char*)"\r\n");
}
// BUG see if . allowed in word
int ValidPeriodToken(char* start, char* end, char next,char next2) // token with period in it - classify it
{ // TOKEN_INCLUSIVE means completes word TOKEN_EXCLUSIVE not part of word. TOKEN_INCOMPLETE means embedded in word but word not yet done
size_t len = end - start;
if (IsAlphaUTF8(next) && tokenControl & TOKEN_AS_IS) return TOKEN_INCOMPLETE;
if (IsDigit(next)) return TOKEN_INCOMPLETE;
if (len > 100) return TOKEN_EXCLUSIVE; // makes no sense
if (len == 2) // letter period combo like H.
{
char* next = SkipWhitespace(start + 2);
if (IsUpperCase(*next) || !*next) return TOKEN_INCLUSIVE; // Letter period like E. before a name
}
if (IsWhiteSpace(next) && IsDigit(*start)) return TOKEN_EXCLUSIVE; // assume no one uses double period without a digit after it.
if (FindWord(start,len)) return TOKEN_INCLUSIVE; // nov. recognized by system for later use
if (IsMadeOfInitials(start,end) == ABBREVIATION) return TOKEN_INCLUSIVE; // word of initials is ok
if (IsUrl(start,end))
{
if (!IsAlphaUTF8(*(end-1))) return TOKEN_INCOMPLETE; // bruce@job.net]
return TOKEN_INCLUSIVE; // swallow URL as a whole
}
if (!strnicmp((char*)"no.",start,3) && IsDigit(next)) return TOKEN_INCLUSIVE; // no.8
if (!strnicmp((char*)"no.",start,3)) return TOKEN_INCLUSIVE; // sentence: No.
if (!IsDigit(*start) && len > 3 && *(end-3) == '.') return TOKEN_INCLUSIVE; // p.a._system
if (FindWord(start,len-1)) return TOKEN_EXCLUSIVE; // word exists independent of it
// is part of a word but word not yet done
if (IsFloat(start,end,numberStyle) && IsDigit(next)) return TOKEN_INCOMPLETE; // decimal number9
if (*start == '$' && IsFloat(start+1,end,numberStyle) && IsDigit(next)) return TOKEN_INCOMPLETE; // decimal number9 or money
if (IsNumericDate(start,end)) return TOKEN_INCOMPLETE; // swallow period date as a whole - bug . after it?
if ( next == '-') return TOKEN_INCOMPLETE; // like N.J.-based
if (IsAlphaUTF8(next)) return TOKEN_INCOMPLETE; // "file.txt"
// not part of word, will be stand alone token.
return TOKEN_EXCLUSIVE;
}
////////////////////////////////////////////////////////////////////////
// BURSTING CODE
////////////////////////////////////////////////////////////////////////
int BurstWord(char* word, int contractionStyle)
{
#ifdef INFORMATION
BurstWord, at a minimum, separates the argument into words based on internal whitespace and internal sentence punctuation.
This is done for storing "sentences" as fact callArgumentList.
Movie titles extend this to split off possessive endings of nouns. Bob's becomes Bob_'s.
Movie titles may contain contractions. These are not split, but two forms of the title have to be stored, the
original and one spot contractions have be expanded, which refines to the original.
And in full burst mode it splits off contractions as well (why- who uses it).
#endif
// concept and class names do not burst, regular or quoted, nor do we waste time if word is 1-2 characters, or if quoted string and NOBURST requested
if (!word[1] || !word[2] || *word == '~' || (*word == '\'' && word[1] == '~' ) || (contractionStyle & NOBURST && *word == '"'))
{
strcpy(burstWords[0],word);
return 1;
}
// make it safe to write on the data while separating things
char* copy = AllocateBuffer();
strcpy(copy, word);
word = copy;
unsigned int base = 0;
// eliminate quote kind of things around it
if (*word == '"' || *word == '\'' || *word == '*' || *word == '.')
{
size_t len = strlen(word);
if (len > 2 && word[len-1] == *word) // start and end same and has something between
{
word[len-1] = 0; // remove trailing quote
++word;
}
}
bool underscoreSeen = false;
char* start = word;
while (*++word) // locate spaces of words, and 's 'd 'll
{
if (*word == ' ' || *word == '_' || *word == '`' || (*word == '-' && contractionStyle == HYPHENS)) // these bound words for sure
{
if (*word == '_' || *word == '`') underscoreSeen = true;
if (!word[1]) break; // end of coming up.
char* end = word;
int len = end-start;
char* prior = (end-1); // ptr to last char of word
char priorchar = *prior;
// separate punctuation from token except if it is initials or abbrev of some kind
if (priorchar == ',' || IsPunctuation(priorchar) & ENDERS) // - : ; ? ! ,
{
char next = *end;
char next2 = (next) ? *SkipWhitespace(end+1) : 0;
if (len <= 1){;}
else if (priorchar == '.' && ValidPeriodToken(start,end,next,next2) != TOKEN_EXCLUSIVE){;} // dont want to burst titles or abbreviations period from them
else // punctuation not a part of token
{
*prior = 0; // not a singleton character, remove it
--len; // better not be here with -fore (len = 0)
}
}
// copy off the word we burst
strncpy(burstWords[base],start,len);
burstWords[base++][len] = 0;
if (base > (MAX_BURST - 5)) break; // protect excess
// add trailing punctuation if any was removed
if (!*prior)
{
*burstWords[base] = priorchar;
burstWords[base++][1] = 0;
}
// now resume after
start = word + 1;
while (*start == ' ' || *start == '_' || *start == '`') ++start; // skip any excess blanks of either kind
word = start - 1;
}
else if (*word == '\'' && contractionStyle & (POSSESSIVES|CONTRACTIONS)) // possible word boundary by split of contraction or possession
{
int split = 0;
if (word[1] == 0 || word[1] == ' ' || word[1] == '_') split = 1; // ' at end of word
else if (word[1] == 's' && (word[2] == 0 || word[2] == ' ' || word[2] == '_')) split = 2; // 's at end of word
else if (!(contractionStyle & CONTRACTIONS)) {;} // only accepting possessives
else if (word[1] == 'm' && (word[2] == 0 || word[2] == ' ' || word[2] == '_')) split = 2; // 'm at end of word
else if (word[1] == 't' && (word[2] == 0 || word[2] == ' ' || word[2] == '_')) split = 2; // 't at end of word
else if ((word[1] == 'r' || word[1] == 'v') && word[2] == 'e' && (word[3] == 0 || word[3] == ' ' || word[3] == '_')) split = 3; // 're 've
else if (word[1] == 'l' && word[2] == 'l' && (word[3] == 0 || word[3] == ' ' || word[3] == '_')) split = 3; // 'll
if (split)
{
// swallow any word before
if (*start != '\'')
{
int len = word - start;
strncpy(burstWords[base],start,len);
burstWords[base++][len] = 0;
start = word;
}
// swallow apostrophe chunk as unique word, aim at the blank after it
word += split;
int len = word - start;
strncpy(burstWords[base],start,len);
burstWords[base++][len] = 0;
start = word;
if (!*word) break; // we are done, show we are at end of line
if (base > MAX_BURST - 5) break; // protect excess
++start; // set start to go for next word+
}
}
}
// now handle end of last piece
if (start && *start && *start != ' ' && *start != '_') strcpy(burstWords[base++],start); // a trailing 's or ' won't have any followup word left
if (!base && underscoreSeen) strcpy(burstWords[base++],(char*)"_");
else if (!base) strcpy(burstWords[base++],start);
FreeBuffer();
burstLimit = base; // note legality of burst word accessor GetBurstWord
return base;
}
char* GetBurstWord(unsigned int n) // 0-based
{
if (n >= burstLimit)
{
ReportBug((char*)"Bad burst n %d",n)
return "";
}
return burstWords[n];
}
char* JoinWords(unsigned int n,bool output) //
{
char* limit;
char* joinBuffer = InfiniteStack(limit,"JoinWords"); // transient
*joinBuffer = 0;
char* at = joinBuffer;
for (unsigned int i = 0; i < n; ++i)
{
char* hold = burstWords[i];
if (!hold) break;
if (!output && (*hold == ',' || *hold == '?' || *hold == '!' || *hold == ':')) // for output, dont space before punctuation
{
if (joinBuffer != at) *--at = 0; // remove the understore before it
}
size_t len = strlen(hold);
if ((len + 4 + (at - joinBuffer)) >= maxBufferSize) break; // avoid overflow
strcpy(at,hold);
at += len;
if (i != (n-1)) strcpy(at++,(char*)"_");
}
if (strlen(joinBuffer) >= (MAX_WORD_SIZE-1))
{
ReportBug("Joinwords was too big %d %s",strlen(joinBuffer),joinBuffer);
joinBuffer[MAX_WORD_SIZE-1] = 0; // safety truncation
}
CompleteBindStack(); // we'd like to leave this infinite but string copy by caller may be into infinite as well
return joinBuffer;
}
////////////////////////////////////////////////////////////////////////
// BASIC TOKENIZING CODE
////////////////////////////////////////////////////////////////////////
static char* HandleQuoter(char* ptr,char** words, int& count)
{
char c = *ptr; // kind of quoter
char* end = ptr;
while (1)
{
end = strchr(end + 1, c); // find matching end?
if (!end) return NULL;
if (end[1] == '"') end++; // skip over "" in quote
else break;
}
if (tokenControl & LEAVE_QUOTE) return end+1;
char pastEnd = IsPunctuation(end[1]); // what comes AFTER quote
if (!(pastEnd & (SPACES|PUNCTUATIONS|ENDERS))) return NULL; // doesnt end cleanly
// if quote has a tailing comma or period, move it outside of the end - "Pirates of the Caribbean,(char*)" -- violates NOMODIFY clause if any
char priorc = *(end-1);
if (priorc == ',' || priorc == '.')
{
*(end-1) = *end;
*end-- = priorc;
}
if (c == '*') // stage direction notation, erase it and return to normal processing
{
*ptr = ' ';
*end = ' '; // erase the closing * of a stage direction -- but violates a nomodify clause
return ptr; // skip opening *
}
// strip off the quotes if quoted words are only alphanumeric single words (emphasis quoting)
char* at = ptr;
while (++at < end)
{
if (!IsAlphaUTF8OrDigit(*at) ) // worth quoting, unless it is final char and an ender
{
if (at == (end-1) && IsPunctuation(*at) & ENDERS);
else // store string as properly tokenized, NOT as a string.
{
char* limit;
char* buf = InfiniteStack(limit,"HandleQuoter"); // transient
++end; // subsume the closing marker
strncpy(buf,ptr,end-ptr);
buf[end-ptr] = 0;
buf[MAX_WORD_SIZE - 25] = 0; // force safe limit
++count;
words[count] = AllocateHeap(buf);
ReleaseInfiniteStack();
if (!words[count]) words[count] = AllocateHeap((char*)"a"); // safe replacement
return end;
}
}
}
++count;
words[count] = AllocateHeap(ptr+1,end-ptr-1); // stripped quotes off simple word
if (!words[count]) words[count] = AllocateHeap((char*)"a"); // safe replacement
return end + 1;
}
static WORDP UnitSubstitution(char* buffer)
{
char value[MAX_WORD_SIZE];
char* at = buffer - 1;
while (IsDigit(*++at) || *at == '.' || *at == ','); // skip past number
strcpy(value, "?`");
strcat(value + 2, at); // presume word after number is not big
while ((at = strchr(value, '.'))) memmove(at, at + 1, strlen(at)); // remove abbreviation periods
WORDP D = FindWord(value, 0, STANDARD_LOOKUP);
if (!D)
{
size_t len = strlen(value);
if (value[len-1] == 's') D = FindWord(value, len-1, STANDARD_LOOKUP);
}
uint64 allowed = tokenControl & (DO_SUBSTITUTE_SYSTEM | DO_PRIVATE);
return (D && allowed & D->internalBits) ? D : NULL; // allowed transform
}
static char* FindWordEnd(char* ptr, char* priorToken, char** words, int &count, bool nomodify, bool oobStart, bool oobJson)
{
char* start = ptr;
char c = *ptr;
unsigned char kind = IsPunctuation(c);
char* end = NULL;
static bool quotepending = false;
// OOB which has { or [ inside starter, must swallow all as one string lest reading JSON blow token limit on sentence. And we can do jsonparse.
if (oobStart && oobJson) // support JSON parsing
{
if (count == 0 && *ptr == '[') return ptr + 1; // start of oob
int level = 0;
char* jsonStart = ptr;
--ptr;
bool quote = false;
while (*++ptr)
{
if (*ptr == '"' && *(ptr - 1) != '\\') quote = !quote;
if (quote) {} // ignore content for level counting
else if (*ptr == '{' || *ptr == '[')
{
if (*(ptr - 1) != '\\') ++level;
}
else if (*ptr == '}' || *ptr == ']')
{
if (*(ptr - 1) != '\\') --level;
if (level == 0)
{
if (tokenControl & JSON_DIRECT_FROM_OOB) // allow full json no tokenlimit
{
ARGUMENT(1) = "TRANSIENT SAFE";
ARGUMENT(2) = jsonStart;
char word[MAX_WORD_SIZE];
FunctionResult result = JSONParseCode(word);
++count;
if (result == NOPROBLEM_BIT) words[count] = AllocateHeap(word); // insert json object
else words[count] = AllocateHeap((char*)"bad json");
}
return ptr + 1;
}
}
}
return ptr;
}
// OOB only separates ( [ { ) ] } - the rest remain joined as given
if (oobStart)
{
if (*ptr == '(' || *ptr == ')' || *ptr == '[' || *ptr == ']' || *ptr == '{' || *ptr == '}' || *ptr == ',') return ptr + 1;
bool quote = false;
--ptr;
while (*++ptr)
{
if (*ptr == '"' && *(ptr - 1) != '\\') quote = !quote;
if (quote) continue;
if (*ptr != ' ' && *ptr != '(' && *ptr != ')' && *ptr != '[' && *ptr != ']' && *ptr != '{' && *ptr != '}') continue;
break;
}
return ptr;
}
if (kind & QUOTERS) // quoted strings
{
if (c == '\'' && ptr[1] == 's' && !IsAlphaUTF8(ptr[2])) return ptr + 2; // 's directly
if (c == '"')
{
if (tokenControl & SPLIT_QUOTE)
{
char* end = strchr(ptr + 1, '"');
if (end) // strip the quotes
{
*ptr = ' ';
*end = ' ';
}
else return ptr + 1; // split up quote marks
}
else // see if merely highlighting a word
{
char* word = AllocateStack(NULL,INPUT_BUFFER_SIZE,false,false);
char* tail = ReadCompiledWord(ptr, word);
char* close = strchr(word + 1, '"');
ReleaseStack(word);
if (close && !strchr(word, ' ')) // we dont need quotes
{
if (tokenControl & LEAVE_QUOTE) return tail;
*ptr = ' '; // kill off starting dq
ptr[close - word] = ' '; // kill off closing dq
return ptr;
}
}
}
if (c == '\'' && tokenControl & SPLIT_QUOTE) // 'enemies of the state'
{
if (quotepending) quotepending = false;
else if (strchr(ptr + 1, '\'')) quotepending = true;
if (quotepending) return ptr + 1;
else if (ptr[1] == ' ' || ptr[1] == '.' || ptr[1] == ',') return ptr + 1;
}
if (c == '\'' && !(tokenControl & TOKEN_AS_IS) && !IsAlphaUTF8(ptr[1]) && !IsDigit(ptr[1])) return ptr + 1; // is this quote or apostrophe - for penntag dont touch it - for 've leave it alone also leave '82 alone
else if (c == '\'' && tokenControl & TOKEN_AS_IS) { ; } // for penntag dont touch it - for 've leave it alone also leave '82 alone
else if (c == '"' && tokenControl & TOKEN_AS_IS) return ptr + 1;
else
{
char* end = HandleQuoter(ptr, words, count);
if (end) return end;
}
if (!IsDigit(ptr[1])) return ptr + 1; // just return isolated quote
}
char token[MAX_WORD_SIZE];
ReadCompiledWord(ptr, token);
size_t l = strlen(token);
if (*ptr == '?') return ptr + 1; // we dont have anything that should join after ? but ) might start emoticon
if (*ptr == 0xc2 && ptr[1] == 0xbf) return ptr + 2; // inverted spanish ?
if (*ptr == 0xc2 && ptr[1] == 0xa1) return ptr + 2; // inverted spanish !
if (IsAlphaUTF8(*ptr) && ptr[1] == '.' && ptr[2] == ' ' && IsUpperCase(*ptr)) return ptr + 2; // single letter abbreviaion period like H.
if (*ptr == '.' && ptr[1] == '.' && ptr[2] == '.' && ptr[3] != '.') return ptr + 3; // ...
if (*ptr == '-' && ptr[1] == '-' && ptr[2] == '-') ptr[2] = ' '; // change excess --- to space
if (*ptr == '-' && ptr[1] == '-' && (ptr[2] == ' ' || IsAlphaUTF8(ptr[2]))) return ptr + 2; // the -- break
if (*ptr == ';' && ptr[1] != ')' && ptr[1] != '(') return ptr + 1; // semicolon not emoticon
if (*ptr == ',' && ptr[1] != ':') return ptr + 1; // comma not emoticon
if (*ptr == '|') return ptr + 1;
if (*ptr == '(' || *ptr == '[' || *ptr == '{') return ptr + 1;
// if we actually have this token in dictionary, accept it. (eg abbreviations, etc)
WORDP Z = FindWord(token); // either case
if (Z && !IS_NEW_WORD(Z) && token[l-1] != '?' && token[l - 1] != '!') return ptr + l; // not generated by user input
// if token ends in period and does not start with digit (not float) and word we know,
// return prior
if (*token == '.' && !IsInteger(token + 1, false, numberStyle) && FindWord(token + 1)) return ptr + 1; // sentence end then word we know
if (token[l - 1] == '.' && FindWord(token, l - 1)) return ptr + l - 1;
// if this was 93302-42345 then we need to keep - separate, not as minus
if (*token == '-' && IsInteger(token + 1, false, numberStyle) && IsInteger(priorToken, false, numberStyle))
{
return ptr + 1;
}
// find current token which has | after it and separate it, like myba,atat,joha
char* pipe = strchr(token + 1, '|');
if (pipe)
{
*pipe = 0; // break apart token
}
// check for float
if (strchr(token, numberPeriod) || strchr(token, 'e') || strchr(token, 'E'))
{
char* at = token;
// check for a currency symbol
char* number = token;
char* currency = 0;
if ((currency = (char*)GetCurrency((unsigned char*)at, number))) at = number; // if currency, find number start of it
bool seenExponent = false;
bool seenPeriod = false;
while (*++at && (IsDigit(*at) || *at == ',' || *at == '.' || (!seenExponent && (*at == 'e' || *at == 'E')) || *at == '-' || *at == '+'))
{
if (currency && at == currency) break; // seen enough if reached a currency suffix
if (*at == 'e' || *at == 'E') seenExponent = true; // exponent can only appear once, 10e4euros
// period AFTER float like 1.0. w space or end
if (*at == numberPeriod && IsDigit(*(at-1)) && seenPeriod && !at[1])
{
return ptr + (at - token);
}
if (*at == numberPeriod) seenPeriod = true;
}
// may be units or currency attached, so dont split that apart
if (IsFloat(number, at, numberStyle) && !UnitSubstitution(at)) // $50. is not a float, its end of sentene
{
if (currency && at == currency) at += strlen(currency);
if (*at == '%') ++at;
if (*at == 'k' || *at == 'K' || *at == 'm' || *at == 'M' || *at == 'B' || *at == 'b')
{
if (!at[1]) ++at;
}
return ptr + (at - token);
}
}
// check for negative number
if (*token == '-' && IsDigit(token[1]))
{
char* at = token;
while (*++at && (IsDigit(*at) || *at == '.' || *at == ',')) { ; }
if (!*at) {
// might be at the year part of a date 10-1-1992
if (count > 2 && IsDigit(*priorToken) && *words[count - 1] == '-' && IsDigit(*words[count - 2])) { ; }
else return ptr + strlen(token);
}
}
// check for ordinary integers whose commas may be confusing
if ((IsDigit(token[0])|| IsDigit(token[1])) && IsDigitWord(token, numberStyle, true)) return ptr + strlen(token);
// check for date
if (IsDate(token)) return ptr + strlen(token);
// check for two numbers separated by a hyphen
char* hyp = strchr(token, '-');
if (hyp && IsDigit(*token))
{
char* at = hyp;
while (*++at && IsDigit(*at)) { ; }
char* at1 = hyp;
while (--at1 != token && IsDigit(*--at1)) { ; }
if (at1 == token && *at == 0) return ptr + (hyp - token);
}
if (hyp && !strchr(hyp+1,'-')) // - used as measure separator
{
if ((hyp[1] == 'x' || hyp[1] == 'X') && hyp[1] == '-') // measure like 2ft-x-5ft
{
ptr[hyp - token] = ' ';
if (hyp[2] == '-') ptr[hyp + 2 - token] = ' ';
return ptr + (hyp - token);
}
else if ((IsDigit(*token) || (*token == '.' && IsDigit(token[1]))) && IsAlphaUTF8(hyp[1]) && !(tokenControl & TOKEN_AS_IS)) // break apart measures like 4-ft except when penntag strict casing
{
ptr[hyp - token] = ' ';
return ptr + (hyp - token); // treat as space
}
else if (hyp[1] == '-') return ptr + (hyp - token); // the anyways-- break
}
// find current token which has comma after it and separate it, like myba,atat,joha
char* comma = strchr(token + 1, ',');
if (comma)
{
*comma = 0; // break apart token
comma = ptr + (comma - token);
}
// Things that are normally separated as single character tokens
char next = ptr[1];
if (c == '=' && next == '=') // swallow headers == ==== ===== etc
{
while (*++ptr == '='){;}
return ptr;
}
else if (c == '\'' && next == '\'' && ptr[2] == '\'' && ptr[3] == '\'') return ptr + 4; // '''' marker
else if (c == '\'' && next == '\'' && ptr[2] == '\'') return ptr + 3; // ''' marker
else if (c == '\'' && next == '\'') return ptr + 2; // '' marker
// arithmetic operator between numbers - . won't be seen because would have been swallowed already if part of a float,
else if ((kind & ARITHMETICS || c == 'x' || c == 'X' || c == '/') && IsDigit(*priorToken) && IsDigit(next))
{
return ptr+1; // separate operators from number
}
// normal punctuation separation
else if (c == '.' && IsDigit(ptr[1])); // double start like .24
else if (c == '.' && (ptr[1] == '"' || ptr[1] == '\'')) return ptr + 1; // let it end after closing quote
if (c == '.' && ptr[1] == '.' && ptr[2] == '.') // stop at .. or ... stand alone punctuation
{
if (tokenControl & TOKEN_AS_IS)
return ptr + 3;
return ptr+1;
}
else if (*ptr == numberComma)
{
if (IsDigit(ptr[1]) && IsDigit(ptr[2]) && IsDigit(ptr[3]) && ptr != start && IsDigit(ptr[-1])) { ; } // 1,000 is legal
else return ptr + 1;
}
else if (kind & (ENDERS|PUNCTUATIONS) && ((unsigned char)IsPunctuation(ptr[1]) == SPACES || ptr[1] == 0)) return ptr+1;
// read an emoticon
char emote[MAX_WORD_SIZE];
int index = 0;
int letters = 0;
char* at = ptr-1;
if (!IsAlphaUTF8OrDigit(at[1])) while (*++at && *at != ' ') // dont check on T?
{
emote[index++] = *at;
if (IsAlphaUTF8(*at) || IsDigit(*at)) ++letters;
if (letters > 1) break; // to many to be emoticon
if (*at == '?' || *at == '!' || *at == '.' || *at == ',')
{
letters = 5;
break; // punctuation we dont want to lose
}
}
if (letters < 2 && (at-ptr) >= 2 && emote[0] != '.' && emote[0] != ',' && emote[0] != '?' && emote[0] != '!' ) // presumed emoticon
{
return at;
}
if (comma && IsDigit(*(comma-1)) && !IsDigit(comma[1])) return comma; // $7 99
if (comma && IsDigit(comma[1]) && IsDigit(comma[2]) && IsDigit(comma[3]) && IsDigit(comma[4])) return comma; // 25,2019
if (comma && !IsCommaNumberSegment(comma+1,NULL)) return comma; // 25,2 rest of word is not valid comma segments
if (kind & BRACKETS && ( (c != '>' && c != '<') || next != '=') )
{
if (c == '<' && next == '/') return ptr + 2; // keep html together </
if (c == '[' && next == '[') return ptr + 2; // keep html together [[
if (c == ']' && next == ']') return ptr + 2; // keep html together ]]
if (c == '{' && next == '{') return ptr + 2; // keep html together {{
if (c == '}' && next == '}') return ptr + 2; // keep html together }}
return ptr+1; // keep all brackets () [] {} <> separate but <= and >= are operations
}
// find "normal" word end, including all touching nonwhitespace, keeping periods (since can be part of word) but not ? or ! which cant
end = ptr;
char* stopper = NULL;
char* fullstopper = NULL;
if (*ptr != ':' && *ptr != ';') while (*++end && !IsWhiteSpace(*end) && *end != '!' && *end != '?')
{
if (*end == numberComma)
{
if (!IsDigit(end[1]) || !IsDigit(* (end-1))) // not comma within a number
{
if (!fullstopper) fullstopper = end;
if (!stopper) stopper = end;
}
continue;
}
if (*end == ';' && !stopper) stopper = end;
if (*end == '-' && !(tokenControl & TOKEN_AS_IS) && !stopper) stopper = end; // alternate possible end (e.g. 8.4-ounce)
if (*end == ';' && !fullstopper) fullstopper = end; // alternate possible end (e.g. 8.4-ounce)
if (end[0] == '.' && end[1] == '.' && end[2] == '.') break; // ...
}
if (comma && end > comma && (!IsDigit(comma[1]) ||!IsDigit(comma[-1]))) end = comma;
if (end == ptr) ++end; // must shift at least 1
// possessive ending? swallow whole token like "K-9's"
if (*(end - 1) == 's' && (end - ptr) > 2 && *(end - 2) == '\'') return end - 2;
WORDP X = FindWord(ptr,end-ptr,PRIMARY_CASE_ALLOWED);
// avoid punctuation so we can detect emoticons
if (X && !(X->properties & PUNCTUATION) && (X->properties & PART_OF_SPEECH || X->systemFlags & PATTERN_WORD || X->internalBits & HAS_SUBSTITUTE)) // we know this word (with exceptions)
{
// if ' follows a number, make it feet
if (*ptr == '\'' && (end-ptr) == 1)
{
if (IsDigit(*priorToken))
{
++count;
words[count] = AllocateHeap((char*)"foot");
return end;
}
}
// but No. must not be recognized unless followed by a digit
else if (!strnicmp(ptr,(char*)"no.",end-ptr))
{
char* at = end;
if (*at) while (*++at && *at == ' ');
if (IsDigit(*at)) return end;
}
else return end;
}
if (IsUpperCase(*ptr))
{
X = FindWord(ptr,end-ptr,LOWERCASE_LOOKUP);
// avoid punctuation so we can detect emoticons
if (X && !(X->properties & PUNCTUATION) && (X->properties & PART_OF_SPEECH || X->systemFlags & PATTERN_WORD || X->internalBits & HAS_SUBSTITUTE)) // we know this word (with exceptions)
{
// No. must not be recognized unless followed by a digit
if (!strnicmp(ptr,(char*)"no.",end-ptr))
{
char* at = end;
if (*at) while (*++at && *at == ' ');
if (IsDigit(*at)) return end;
}
else return end;
}
}
// could it be email or web address?
char* atsign = strchr(ptr,'@'); // possible email?
if (atsign && atsign < end)
{
char* period = strchr(atsign+1,'.');
if (period && period < end && IsAlphaUTF8(ptr[end-ptr-1]) && IsAlphaUTF8(ptr[end-ptr-2])) // can be domain data
{
// find end of email text word
while (*++period)
{
if (!IsAlphaUTF8(*period)) return period;
}
return end;
}
}
// e-mail, needs to not see - as a stopper.
WORDP W = (fullstopper) ? FindWord(ptr,fullstopper-ptr) : NULL;
if (*end && IsDigit(end[1]) && IsDigit(*(end-1))) W = NULL; // if , separating digits, DONT break at it 4,000 even though we recognize subpiece
if (W && (W->properties & PART_OF_SPEECH || W->systemFlags & PATTERN_WORD)) return fullstopper; // recognize word at more splits
// recognize subword? now in case - is a stopper
if (stopper)
{
W = ((stopper-ptr) > 1 && ((*stopper != '-' && *stopper != '/') || !IsAlphaUTF8(stopper[1]))) ? FindWord(ptr,stopper-ptr) : NULL;
if (*stopper == '-' && (IsAlphaUTF8(end[1]) || IsDigit(end[1]))) W = NULL; // but don't split - in a name or word or think like jo-5
else if (*stopper && IsDigit(stopper[1]) && IsDigit(*(stopper-1))) W = NULL; // if , separating digits, DONT break at it 4,000 even though we recognize subpiece
if (W && (W->properties & PART_OF_SPEECH || W->systemFlags & PATTERN_WORD)) return stopper; // recognize word at more splits
}
int lsize = strlen(token);
while (IsPunctuation(token[lsize-1])) token[--lsize] = 0; // remove trailing punctuation
char* after = start + lsize;
// see if we have 25,2015
size_t tokenlen = strlen(token);
if (tokenlen == 7 && IsDigit(token[0]) && IsDigit(token[1]) && token[2] == numberComma && IsDigit(token[3]))
return ptr + 2;
if (tokenlen == 6 && IsDigit(token[0]) && token[1] == numberComma && IsDigit(token[2])) // 2,2015
return ptr + 1;
if (!strnicmp(token,"https://",8) || !strnicmp(token,"http://",7)) return after;
if (*priorToken != '/' && IsFraction(token)) return after; // fraction?
// check for place number
char* place = ptr;
while (IsDigit(*place)) ++place;
if (!stricmp(language, "english") && (!stricmp(place,"st") || !stricmp(place,"nd") || !stricmp(place,"rd"))) return end;
else if (!stricmp(language, "french") && (!stricmp(place, "er") || !stricmp(place, "ere") || !stricmp(place, "ère") || !stricmp(place, "nd") || !stricmp(place, "nde") || !stricmp(place, "eme") || !stricmp(place, "ème"))) return end;
int len = end - ptr;
char next2;
if (*ptr == '/') return ptr+1; // split of things separated
while (++ptr && !IsWordTerminator(*ptr)) // now scan to find end of token one by one, stopping where appropriate
{
c = *ptr;
if (c == '|') break;
kind = IsPunctuation(c);
next = ptr[1];
if (c == numberComma)
{
if (!IsDigit(ptr[1]) || !IsDigit(*(ptr-1))) break; // comma obviously not in a number
// must have 3 digits after comma
if (!IsDigit(ptr[2]) || !IsDigit(ptr[3])) break;
}
if (c == '\'' && next == '\'') break; // '' marker or ''' or ''''
else if (c == '=' && next == '=') break; // swallow headers == ==== ===== etc
next2 = (next) ? *SkipWhitespace(ptr+2) : 0; // start of next token
if (c == '-' && next == '-') break; // -- in middle is a break regardless
if (tokenControl & TOKEN_AS_IS) {;}
else
{
if (c == '\'') // possessive ' or 's - we separate ' or 's into own word
{
if (next == ',' || IsWhiteSpace(next) || next == ';' || next == '.' || next == '!' || next == '?') // trailing plural?
{
break;
}
if (!IsAlphaUTF8OrDigit(next)) break; // ' not within a word, ending it
if (((next == 's') || ( next == 'S')) && !IsAlphaUTF8OrDigit(ptr[2])) // 's becomes separate - can be WRONG when used as contraction like speaker's but we cant know
{
ptr[1] = 's'; // in case uppercase flaw
break;
}
// ' as particle ellision
if ((ptr - start) == 1 && (*start == 'd' || *start == 'c' || *start == 'j' || *start == 'l' || *start == 's' || *start == 't' || *start == 'm' || *start == 'n')) return ptr + 1; // break off d' argent and other foreign particles
else if (!stricmp(language, "french"))
{
if ((ptr - start) == 1 && (*start == 'D' || *start == 'C' || *start == 'J' || *start == 'L' || *start == 'S' || *start == 'T' || *start == 'M' || *start == 'N')) return ptr + 1; // break off french particles in upper case
else if ((ptr - start) == 2 && (*start == 'q' || *start == 'Q') && *(start + 1) == 'u') return ptr + 1; // break off qu'
else if ((ptr - start) == 5 && (*start == 'j' || *start == 'J') && *(start + 1) == 'u' && *(start + 2) == 's' && *(start + 3) == 'q' && *(start + 4) == 'u') return ptr + 1; // break off jusqu'
else if ((ptr - start) == 6 && (*start == 'l' || *start == 'L') && *(start + 1) == 'o' && *(start + 2) == 'r' && *(start + 3) == 's' && *(start + 4) == 'q' && *(start + 5) == 'u') return ptr + 1; // break off lorsqu'
else if ((ptr - start) == 6 && (*start == 'p' || *start == 'P') && *(start + 1) == 'u' && *(start + 2) == 'i' && *(start + 3) == 's' && *(start + 4) == 'q' && *(start + 5) == 'u') return ptr + 1; // break off puisqu'
}
// 12'6" or 12'. or 12'
if (IsDigit(*start) && !IsAlphaUTF8(next)) return ptr + 1; // 12' swallow ' into number word
}
else if (ptr != start && c == ':' && IsDigit(next) && IsDigit(*(ptr-1)) && len > 1) // time 10:30 or odds 1:3
{
if (!strnicmp(end-2,(char*)"am",2)) return end-2;
else if (!strnicmp(end-2,(char*)"pm",2)) return end-2;
else if (len > 2 && !strnicmp(end-3,(char*)"a.m",3)) return end-3;
else if (len > 2 && !strnicmp(end-3,(char*)"p.m",3)) return end-3;
else if (len > 3 && !strnicmp(end-4,(char*)"a.m.",4)) return end-4;
else if (len > 3 && !strnicmp(end-4,(char*)"p.m.",4)) return end-4;
else if (ptr[2] == ' ' || !ptr[2]) return ptr+2;
else if ((ptr[3] == ' ' || !ptr[3]) && IsDigit(ptr[2])) return ptr+3;
}
// number before things? 8months but not 24% And dont split 1.23 or time words 10:30 and 30:20:20. dont break 6E
if (IsDigit(*start) && IsDigit(*(ptr-1)) && !IsDigit(c) && c != '%' && c != '.' && c != ':' && ptr[1] && ptr[2] && ptr[1] != ' ' && ptr[2] != ' ')
{
if (c == 's' && ptr[1] == 't'){;} // 1st
else if (c == 'n' && ptr[1] == 'd'){;} // 2nd
else if (c == 'r' && ptr[1] == 'd'){;} // 3rd
else if (c == 't' && ptr[1] == 'h'){;} // 5th
else // break apart known word but not single value or non-word
{
char word[MAX_WORD_SIZE];
ReadCompiledWord(ptr-1,word); // what is the word
if (FindWord(word,0)) return ptr; // we know this second word after the digit
}
}
if ( c == ']' || c == ')') break; //closers
if ((c == 'x' || c== 'X') && IsDigit(*start) && IsDigit(next)) break; // break 4x4
}
if (kind & BRACKETS) break; // separate brackets
if (kind & (PUNCTUATIONS|ENDERS|QUOTERS) && IsWordTerminator(next))
{
if (c == '-' && *ptr == '-' && next == ' ') return ptr + 1;
if (tokenControl & TOKEN_AS_IS && next == ' ' && ptr[1] && !IsWhiteSpace(ptr[2])) return ptr + 1; // our token ends and there is more text to come
if (!(tokenControl & TOKEN_AS_IS)) break; // funny things at end of word
}
if (c == '/') return ptr; // separate out / items like john/bob or 12/21/45 or 1/2
if (c == ';') return ptr; // separate semicolons
// special interpretations of period
if (c == '.')
{
int x = ValidPeriodToken(start,end,next,next2);
if (x == TOKEN_INCLUSIVE) return end;
else if (x == TOKEN_INCOMPLETE) continue;
else break;
}
}
if (*(ptr-1) == '"' && start != (ptr-1)) --ptr;// trailing double quote stuck on something else
return ptr;
}
char* Tokenize(char* input,int &mycount,char** words,bool all,bool nomodify,bool oobStart) // return ptr to stuff to continue analyzing later
{ // all is true if to pay no attention to end of sentence -- eg for a quoted string
// nomodify is true on analyzing outputs into sentences, because user format may be fixed
char* ptr = SkipWhitespace(input);
int count = 0;
char* html = input;
bool oobJson = false;
unsigned int quoteCount = 0;
char priorToken[MAX_WORD_SIZE] = {0};
int nest = 0;
unsigned int paren = 0;
//AdjustUTF8(ptr, ptr - 1);
if (tokenControl == UNTOUCHED_INPUT)
{
while (ALWAYS) {
input = SkipWhitespace(input);
char* space = strchr(input,' '); // find separator
if (space) {
++count;
words[count] = AllocateHeap(input,space-input); // the token
input = space;
}
else if (*input) {
++count;
words[count] = AllocateHeap(input); // the token
input += strlen(input);
break;
}
else break;
}
mycount = count;
ptr = input;
goto SAFETY;
}
// convert html data
while ((html = strstr(html,(char*)"&#")) != 0) //  
{
if (IsDigit(html[2]) && IsDigit(html[3]) && html[4] == ';')
{
*html = (char)atoi(html+2);
memmove(html+1,html+5,strlen(html+4));
}
else ++html;
}
html = input;
while ((html = strstr(html,(char*)""")) != 0) // "
{
*html = '"';
memmove(html+1,html+6,strlen(html+5));
}
html = input;
while ((html = strchr(html,'\\')) != 0) // \" remove this