#include "common.h"

#include "cs_jp.h"

#ifdef INFORMATION

SPACES space \t \r \n

PUNCTUATIONS, | -(see also ENDERS)

ENDERS .; : ? !-

BRACKETS() [] {} < >

ARITHMETICS % *+-^ = / .

SYMBOLS $ # @ ~

CONVERTERS & `

//NORMALS A-Z a-z 0-9 _ and sometimes /

#endif

static WORDP subresult;

int inputNest = 0;

int actualTokenCount = 0;

char burstWords[MAX_BURST][MAX_WORD_SIZE]; // each token burst from a text string

static unsigned int burstLimit = 0; // index of burst words

static WORDP lastMatch = NULL;

static int lastMatchLocation = 0;

uint64 tokenFlags; // what tokenization saw

char* wordStarts[MAX_SENTENCE_LENGTH]; // current sentence tokenization (always points to D->word values or allocated values)

unsigned int wordCount; // how many words/tokens in sentence

bool capState[MAX_SENTENCE_LENGTH];

void ResetTokenSystem()

{

tokenFlags = 0;

wordCount = 0;

memset(wordStarts,0,sizeof(char*)*MAX_SENTENCE_LENGTH); // reinit for new volley - sharing of word space can occur throughout this volley

wordStarts[0] = ""; // underflow protection

ClearWhereInSentence();

memset(concepts, 0, sizeof(concepts)); // concept chains per word

memset(topics, 0, sizeof(concepts)); // concept chains per word

}

void DumpResponseControls(uint64 val)

{

if (val & RESPONSE_UPPERSTART) Log(USERLOG,"RESPONSE_UPPERSTART ");

if (val & RESPONSE_REMOVESPACEBEFORECOMMA) Log(USERLOG,"RESPONSE_REMOVESPACEBEFORECOMMA ");

if (val & RESPONSE_ALTERUNDERSCORES) Log(USERLOG,"RESPONSE_ALTERUNDERSCORES ");

if (val & RESPONSE_REMOVETILDE) Log(USERLOG,"RESPONSE_REMOVETILDE ");

if (val & RESPONSE_NOCONVERTSPECIAL) Log(USERLOG,"RESPONSE_NOCONVERTSPECIAL ");

if (val & RESPONSE_CURLYQUOTES) Log(USERLOG,"RESPONSE_CURLYQUOTES ");

}

void DumpTokenControls(uint64 val)

{

if ((val & DO_SUBSTITUTE_SYSTEM) == DO_SUBSTITUTE_SYSTEM) Log(USERLOG,"DO_SUBSTITUTE_SYSTEM ");

else // partials

{

if (val & DO_ESSENTIALS) Log(USERLOG,"DO_ESSENTIALS ");

if (val & DO_SUBSTITUTES) Log(USERLOG,"DO_SUBSTITUTES ");

if (val & DO_CONTRACTIONS) Log(USERLOG,"DO_CONTRACTIONS ");

if (val & DO_INTERJECTIONS) Log(USERLOG,"DO_INTERJECTIONS ");

if (val & DO_BRITISH) Log(USERLOG,"DO_BRITISH ");

if (val & DO_SPELLING) Log(USERLOG,"DO_SPELLING ");

if (val & DO_TEXTING) Log(USERLOG,"DO_TEXTING ");

if (val & DO_NOISE) Log(USERLOG,"DO_NOISE ");

}

if (val & DO_PRIVATE) Log(USERLOG,"DO_PRIVATE ");

// reserved

if (val & DO_NUMBER_MERGE) Log(USERLOG,"DO_NUMBER_MERGE ");

if (val & DO_PROPERNAME_MERGE) Log(USERLOG,"DO_PROPERNAME_MERGE ");

if (val & DO_DATE_MERGE) Log(USERLOG,"DO_DATE_MERGE ");

if (val & NO_PROPER_SPELLCHECK) Log(USERLOG,"NO_PROPER_SPELLCHECK ");

if (val & NO_LOWERCASE_PROPER_MERGE) Log(USERLOG,"NO_LOWERCASE_PROPER_MERGE ");

if (val & DO_SPELLCHECK) Log(USERLOG,"DO_SPELLCHECK ");

if (val & DO_INTERJECTION_SPLITTING) Log(USERLOG,"DO_INTERJECTION_SPLITTING ");

if (val & DO_SPLIT_UNDERSCORE) Log(USERLOG,"DO_SPLIT_UNDERSCORE ");

if (val & MARK_LOWER) Log(USERLOG,"MARK_LOWER ");

if ((val & DO_PARSE) == DO_PARSE) Log(USERLOG,"DO_PARSE ");

else if (val & DO_POSTAG) Log(USERLOG,"DO_POSTAG ");

if ( val & JSON_DIRECT_FROM_OOB) Log(USERLOG, "JSON_DIRECT_FROM_OOB ");

if (val & NO_IMPERATIVE) Log(USERLOG,"NO_IMPERATIVE ");

if (val & NO_WITHIN) Log(USERLOG,"NO_WITHIN ");

if (val & NO_SENTENCE_END) Log(USERLOG,"NO_SENTENCE_END ");

if (val & NO_HYPHEN_END) Log(USERLOG,"NO_HYPHEN_END ");

if (val & NO_COLON_END) Log(USERLOG,"NO_COLON_END ");

if (val & NO_SEMICOLON_END) Log(USERLOG,"NO_SEMICOLON_END ");

if (val & STRICT_CASING) Log(USERLOG,"STRICT_CASING ");

if (val & ONLY_LOWERCASE) Log(USERLOG,"ONLY_LOWERCASE ");

if (val & TOKEN_AS_IS) Log(USERLOG,"TOKEN_AS_IS ");

if (val & SPLIT_QUOTE) Log(USERLOG,"SPLIT_QUOTE ");

if (val & LEAVE_QUOTE) Log(USERLOG,"LEAVE_QUOTE ");

if (val & UNTOUCHED_INPUT) Log(USERLOG,"UNTOUCHED_INPUT ");

if (val & NO_FIX_UTF) Log(USERLOG,"NO_FIX_UTF ");

if (val & NO_CONDITIONAL_IDIOM) Log(USERLOG,"NO_CONDITIONAL_IDIOM ");

}

void DumpTokenFlags(char* msg)

{

Log(USERLOG,"%s TokenFlags: ",msg);

// DID THESE

if (tokenFlags & DO_ESSENTIALS) Log(USERLOG,"DO_ESSENTIALS ");

if (tokenFlags & DO_SUBSTITUTES) Log(USERLOG,"DO_SUBSTITUTES ");

if (tokenFlags & DO_CONTRACTIONS) Log(USERLOG,"DO_CONTRACTIONS ");

if (tokenFlags & DO_INTERJECTIONS) Log(USERLOG,"DO_INTERJECTIONS ");

if (tokenFlags & DO_BRITISH) Log(USERLOG,"DO_BRITISH ");

if (tokenFlags & DO_SPELLING) Log(USERLOG,"DO_SPELLING ");

if (tokenFlags & DO_TEXTING) Log(USERLOG,"DO_TEXTING ");

if (tokenFlags & DO_PRIVATE) Log(USERLOG,"DO_PRIVATE ");

// reserved

if (tokenFlags & DO_NUMBER_MERGE) Log(USERLOG,"NUMBER_MERGE ");

if (tokenFlags & DO_PROPERNAME_MERGE) Log(USERLOG,"PROPERNAME_MERGE ");

if (tokenFlags & DO_DATE_MERGE) Log(USERLOG,"DATE_MERGE ");

if (tokenFlags & DO_SPELLCHECK) Log(USERLOG,"SPELLCHECK ");

// FOUND THESE

if (tokenFlags & NO_HYPHEN_END) Log(USERLOG,"HYPHEN_END ");

if (tokenFlags & NO_COLON_END) Log(USERLOG,"COLON_END ");

if (tokenFlags & PRESENT) Log(USERLOG,"PRESENT ");

if (tokenFlags & PAST) Log(USERLOG,"PAST ");

if (tokenFlags & FUTURE) Log(USERLOG,"FUTURE ");

if (tokenFlags & PERFECT) Log(USERLOG,"PERFECT ");

if (tokenFlags & PRESENT_PERFECT) Log(USERLOG,"PRESENT_PERFECT ");

if (tokenFlags & CONTINUOUS) Log(USERLOG,"CONTINUOUS ");

if (tokenFlags & PASSIVE) Log(USERLOG,"PASSIVE ");

if (tokenFlags & QUESTIONMARK) Log(USERLOG,"QUESTIONMARK ");

if (tokenFlags & EXCLAMATIONMARK) Log(USERLOG,"EXCLAMATIONMARK ");

if (tokenFlags & PERIODMARK) Log(USERLOG,"PERIODMARK ");

if (tokenFlags & IMPLIED_SUBJECT) Log(USERLOG,"IMPLIED_SUBJECT ");

if (tokenFlags & USERINPUT) Log(USERLOG,"USERINPUT ");

if (tokenFlags & FAULTY_PARSE) Log(USERLOG,"FAULTY_PARSE ");

if (tokenFlags & COMMANDMARK) Log(USERLOG,"COMMANDMARK ");

if (tokenFlags & QUOTATION) Log(USERLOG,"QUOTATION ");

if (tokenFlags & IMPLIED_YOU) Log(USERLOG,"IMPLIED_YOU ");

if (tokenFlags & NOT_SENTENCE) Log(USERLOG,"NOT_SENTENCE ");

if (inputNest) Log(USERLOG," ^input ");

if (tokenFlags & NO_CONDITIONAL_IDIOM) Log(USERLOG,"CONDITIONAL_IDIOM ");

Log(USERLOG,"\r\n");

}

// BUG see if . allowed in word

int ValidPeriodToken(char* start, char* end, char next,char next2) // token with period in it - classify it

{ // TOKEN_INCLUSIVE means completes word TOKEN_EXCLUSIVE not part of word. TOKEN_INCOMPLETE means embedded in word but word not yet done

size_t len = end - start;

if (IsAlphaUTF8(next) && tokenControl & TOKEN_AS_IS) return TOKEN_INCOMPLETE;

if (IsDigit(next)) return TOKEN_INCOMPLETE;

if (len > 100) return TOKEN_EXCLUSIVE; // makes no sense

if (len == 2) // letter period combo like H.

{

char* next1 = SkipWhitespace(start + 2);

if (IsUpperCase(*next1) || !*next1) return TOKEN_INCLUSIVE; // Letter period like E. before a name

}

if (IsWhiteSpace(next) && IsDigit(*start)) return TOKEN_EXCLUSIVE; // assume no one uses double period without a digit after it.

if (FindWord(start,len)) return TOKEN_INCLUSIVE; // nov. recognized by system for later use

if (IsMadeOfInitials(start,end) == ABBREVIATION) return TOKEN_INCLUSIVE; // word of initials is ok

if (IsUrl(start,end))

{

if (!IsAlphaUTF8(*(end-1))) return TOKEN_INCOMPLETE; // bruce@job.net]

return TOKEN_INCLUSIVE; // swallow URL as a whole

}

if (!strnicmp((char*)"no.",start,3) && IsDigit(next)) return TOKEN_INCLUSIVE; // no.8

if (!strnicmp((char*)"no.",start,3)) return TOKEN_INCLUSIVE; // sentence: No.

if (!IsDigit(*start) && len > 3 && *(end-3) == '.') return TOKEN_INCLUSIVE; // p.a._system

if (FindWord(start,len-1)) return TOKEN_EXCLUSIVE; // word exists independent of it

// is part of a word but word not yet done

if (IsFloat(start,end,numberStyle) && IsDigit(next)) return TOKEN_INCOMPLETE; // decimal number9

if (*start == '$' && IsFloat(start+1,end,numberStyle) && IsDigit(next)) return TOKEN_INCOMPLETE; // decimal number9 or money

if (IsNumericDate(start,end)) return TOKEN_INCOMPLETE; // swallow period date as a whole - bug . after it?

if ( next == '-') return TOKEN_INCOMPLETE; // like N.J.-based

if (IsAlphaUTF8(next)) return TOKEN_INCOMPLETE; // "file.txt"

// not part of word, will be stand alone token.

return TOKEN_EXCLUSIVE;

}

////////////////////////////////////////////////////////////////////////

// BURSTING CODE

////////////////////////////////////////////////////////////////////////

int BurstWord(const char* word, int contractionStyle)

{

#ifdef INFORMATION

BurstWord, at a minimum, separates the argument into words based on internal whitespace and internal sentence punctuation.

This is done for storing "sentences" as fact callArgumentList.

Movie titles extend this to split off possessive endings of nouns. Bob's becomes Bob_'s.

Movie titles may contain contractions. These are not split, but two forms of the title have to be stored, the

original and one spot contractions have be expanded, which refines to the original.

And in full burst mode it splits off contractions as well (why- who uses it).

#endif

// concept and class names do not burst, regular or quoted, nor do we waste time if word is 1-2 characters, or if quoted string and NOBURST requested

if (!word[1] || !word[2] || *word == '~' || (*word == '\'' && word[1] == '~' ) || (contractionStyle & NOBURST && *word == '"'))

{

strcpy(burstWords[0],word);

return 1;

}

// make it safe to write on the data while separating things

char* copy = AllocateBuffer("burst");

strcpy(copy, word);

unsigned int base = 0;

// eliminate quote kind of things around it

if (*copy == '"' || *copy == '\'') // used to also be || *copy == '*' || *copy == '.'

{

size_t len = strlen(copy);

if (len == 3 && *copy == '.' && copy[1] == '.' && copy[2] == '.'); // keep ellipsis

else if (len > 2 && copy[len - 1] == *copy) // start and end same and has something between

{

copy[len-1] = 0; // remove trailing quote

++copy;

}

}

bool underscoreSeen = false;

char* start = copy;

while (*++copy) // locate spaces of copys, and 's 'd 'll

{

if (*copy == ' ' || *copy == '_' || *copy == '`' || (*copy == '-' && contractionStyle == HYPHENS)) // these bound copys for sure

{

if (*copy == '_' || *copy == '`') underscoreSeen = true;

if (!copy[1]) break; // end of coming up.

char* end = copy;

int len = end-start;

char* prior = (end-1); // ptr to last char of copy

char priorchar = *prior;

// separate punctuation from token except if it is initials or abbrev of some kind

if (priorchar == ',' || IsPunctuation(priorchar) & ENDERS) // - : ; ? ! ,

{

char next = *end;

char next2 = (next) ? *SkipWhitespace(end+1) : 0;

if (len <= 1){;}

else if (priorchar == '.' && ValidPeriodToken(start,end,next,next2) != TOKEN_EXCLUSIVE){;} // dont want to burst titles or abbreviations period from them

else // punctuation not a part of token

{

*prior = 0; // not a singleton character, remove it

--len; // better not be here with -fore (len = 0)

}

}

// copy off the copy we burst

strncpy(burstWords[base],start,len);

burstWords[base++][len] = 0;

if (base > (MAX_BURST - 5)) break; // protect excess

// add trailing punctuation if any was removed

if (!*prior)

{

*burstWords[base] = priorchar;

burstWords[base++][1] = 0;

}

// now resume after

start = copy + 1;

while (*start == ' ' || *start == '_' || *start == '`') ++start; // skip any excess blanks of either kind

copy = start - 1;

}

else if (*copy == '\'' && contractionStyle & (POSSESSIVES|CONTRACTIONS)) // possible copy boundary by split of contraction or possession

{

int split = 0;

if (copy[1] == 0 || copy[1] == ' ' || copy[1] == '_') split = 1; // ' at end of copy

else if (copy[1] == 's' && (copy[2] == 0 || copy[2] == ' ' || copy[2] == '_')) split = 2; // 's at end of copy

else if (!(contractionStyle & CONTRACTIONS)) {;} // only accepting possessives

else if (copy[1] == 'm' && (copy[2] == 0 || copy[2] == ' ' || copy[2] == '_')) split = 2; // 'm at end of copy

else if (copy[1] == 't' && (copy[2] == 0 || copy[2] == ' ' || copy[2] == '_')) split = 2; // 't at end of copy

else if ((copy[1] == 'r' || copy[1] == 'v') && copy[2] == 'e' && (copy[3] == 0 || copy[3] == ' ' || copy[3] == '_')) split = 3; // 're 've

else if (copy[1] == 'l' && copy[2] == 'l' && (copy[3] == 0 || copy[3] == ' ' || copy[3] == '_')) split = 3; // 'll

if (split)

{

// swallow any copy before

if (*start != '\'')

{

int len = copy - start;

strncpy(burstWords[base],start,len);

burstWords[base++][len] = 0;

start = copy;

}

// swallow apostrophe chunk as unique copy, aim at the blank after it

copy += split;

int len = copy - start;

strncpy(burstWords[base],start,len);

burstWords[base++][len] = 0;

start = copy;

if (!*copy) break; // we are done, show we are at end of line

if (base > MAX_BURST - 5) break; // protect excess

++start; // set start to go for next copy+

}

}

}

// now handle end of last piece

if (start && *start && *start != ' ' && *start != '_') strcpy(burstWords[base++],start); // a trailing 's or ' won't have any followup copy left

if (!base && underscoreSeen) strcpy(burstWords[base++],(char*)"_");

else if (!base && start) strcpy(burstWords[base++],start);

FreeBuffer("burst");

burstLimit = base; // note legality of burst copy accessor GetBurstcopy

return base;

}

char* GetBurstWord(unsigned int n) // 0-based

{

if (n >= burstLimit)

{

ReportBug((char*)"Bad burst n %d",n);

return "";

}

return burstWords[n];

}

char* JoinWords(unsigned int n,bool output,char* joinBuffer) //

{

char* limit;

bool given = (joinBuffer) ? true : false;

if (!joinBuffer) joinBuffer = InfiniteStack(limit,"JoinWords"); // transient maybe

*joinBuffer = 0;

char* at = joinBuffer;

for (unsigned int i = 0; i < n; ++i)

{

char* hold = burstWords[i];

if (!hold) break;

if (!output && (*hold == ',' || *hold == '?' || *hold == '!' || *hold == ':')) // for output, dont space before punctuation

{

if (joinBuffer != at) *--at = 0; // remove the understore before it

}

size_t len = strlen(hold);

if ((len + 4 + (at - joinBuffer)) >= maxBufferSize) break; // avoid overflow

strcpy(at,hold);

at += len;

if (i != (n-1)) strcpy(at++,(char*)"_");

}

if (strlen(joinBuffer) >= (MAX_WORD_SIZE-1))

{

joinBuffer[MAX_WORD_SIZE - 1] = 0; // safety truncation

ReportBug("Joinwords was too big %d %s...",strlen(joinBuffer),joinBuffer);

}

if (!given) CompleteBindStack(); // we'd like to leave this infinite but string copy by caller may be into infinite as well

return joinBuffer;

}

////////////////////////////////////////////////////////////////////////

// BASIC TOKENIZING CODE

////////////////////////////////////////////////////////////////////////

static char* HandleQuoter(char* ptr,char** words, int& count)

{

char c = *ptr; // kind of quoter

char* end = ptr;

while (1)

{

end = strchr(end + 1, c); // find matching end?

if (!end) return NULL;

if (end[1] == '"') end++; // skip over "" in quote

else break;

}

if (tokenControl & LEAVE_QUOTE) return end+1;

char pastEnd = IsPunctuation(end[1]); // what comes AFTER quote

if (!(pastEnd & (SPACES|PUNCTUATIONS|ENDERS))) return NULL; // doesnt end cleanly

// if quote has a tailing comma or period, move it outside of the end - "Pirates of the Caribbean,(char*)" -- violates NOMODIFY clause if any

char priorc = *(end-1);

if (priorc == ',' || priorc == '.')

{

*(end-1) = *end;

*end-- = priorc;

}

if (c == '*') // stage direction notation, erase it and return to normal processing

{

*ptr = ' ';

*end = ' '; // erase the closing * of a stage direction -- but violates a nomodify clause

return ptr; // skip opening *

}

// strip off the quotes if quoted words are only alphanumeric single words (emphasis quoting)

char* at = ptr;

while (++at < end)

{

if (!IsAlphaUTF8OrDigit(*at) ) // worth quoting, unless it is final char and an ender

{

if (at == (end-1) && IsPunctuation(*at) & ENDERS);

else // store string as properly tokenized, NOT as a string.

{

char* limit;

char* buf = InfiniteStack(limit,"HandleQuoter"); // transient

++end; // subsume the closing marker

strncpy(buf,ptr,end-ptr);

buf[end-ptr] = 0;

buf[MAX_WORD_SIZE - 25] = 0; // force safe limit

++count;

words[count] = AllocateHeap(buf);

ReleaseInfiniteStack();

if (!words[count]) words[count] = AllocateHeap((char*)"a"); // safe replacement

return end;

}

}

}

++count;

if ((end - ptr) <= 1) words[count] = AllocateHeap((char*)"a"); // protection from erroneous

else words[count] = AllocateHeap(ptr+1,end-ptr-1); // stripped quotes off simple word

if (!words[count]) words[count] = AllocateHeap((char*)"a"); // safe replacement

if (!words[count]) --count; // flush it

return end + 1;

}

WORDP ApostropheBreak(char* aword)

{

char word[MAX_WORD_SIZE];

if (strlen(aword) > (MAX_WORD_SIZE - 2)) return NULL;

*word = '*';

strcpy(word + 1, aword);

WORDP D = FindWord(word);

if (D)

{

if (D->systemFlags & HAS_SUBSTITUTE)

{

WORDP X = GetSubstitute(D);

uint64 allowed = tokenControl & (DO_SUBSTITUTE_SYSTEM | DO_PRIVATE);

return (allowed) ? X : NULL; // allowed to break

}

}

return NULL;

}

static WORDP UnitSubstitution(char* buffer, unsigned int i)

{

char value[MAX_WORD_SIZE];

char* at = buffer - 1;

if (IsSign(*(at + 1)) ) ++at; // negative units

while (IsDigit(*++at) || *at == '.' || *at == ','); // skip past number

strcpy(value, "?|"); // SUBSTITUTE_SEPARATOR used

// also consider next word not conjoined

if (!*at && i > 0 && i < wordCount)

{

strcat(value + 2, wordStarts[i + 1]); // presume word after number is not big

}

else strcat(value + 2, at); // presume word after number is not big

while ((at = strchr(value, '.'))) memmove(at, at + 1, strlen(at)); // remove abbreviation periods

WORDP D = FindWord(value, 0, STANDARD_LOOKUP);

if (!D)

{

size_t len = strlen(value);

if (value[len-1] == 's') D = FindWord(value, len-1, STANDARD_LOOKUP);

}

uint64 allowed = tokenControl & (DO_SUBSTITUTE_SYSTEM | DO_PRIVATE);

if (D && allowed & D->internalBits) return D ; // allowed transform

return NULL;

}

static WORDP PosSubstitution(char* buffer, int i)

{

char value[3 * MAX_WORD_SIZE];

strcpy(value, "?=");

strcat(value + 2, buffer); // presume word after word is not big

char* at = value;

while ((at = strchr(at, '_'))) *at++ = '`';// make safe form that we used for _

WORDP D = FindWord(value, 0, STANDARD_LOOKUP);

uint64 allowed = tokenControl & (DO_SUBSTITUTE_SYSTEM | DO_PRIVATE);

if (D && allowed & D->internalBits) return D; // allowed transform

return NULL;

}

static char spawnWord[100];

static char* FindWordEnd(char* ptr, char* priorToken, char** words, int& count, bool& oobStart, int& oobJson)

{

char* start = ptr;

char c = *ptr;

unsigned char kind = IsPunctuation(c);

char* end = NULL;

static bool quotepending = false;

bool isEnglish = (!stricmp(current_language, "english") ? true : false);

bool isFrench = (!stricmp(current_language, "french") ? true : false);

bool isJapanese = ((!stricmp(current_language, "japanese") || !stricmp(current_language, "chinese"))? true : false);

bool isSpanish = (!stricmp(current_language, "spanish") ? true : false);

// OOB which has { or [ inside starter, must swallow all as one string lest reading JSON blow token limit on sentence. And we can do jsonparse.

if (oobJson) // support JSON parsing

{

if (count == 0 && (*ptr == '[' || *ptr == '{')) return ptr + 1; // start of oob [ token

int level = 0;

char* jsonStart = ptr;

--ptr;

bool quote = false;

char* why = strstr(ptr, "why");

while (*++ptr)

{

if (*ptr == '\\') // escaped character, skip over (protect against escaped dquote)

{

ptr += 1;

continue;

}

if (*ptr == '"')

quote = !quote;

if (quote)

continue; // ignore content for level counting

if (*ptr == '{' || *ptr == '[')

++level;

else if (*ptr == '}' || *ptr == ']')

{

if (--level == 0)

{

if (tokenControl & JSON_DIRECT_FROM_OOB) // allow full json

{

// don't let parser be confused by user utterance, e.g. if ends in a quote

char* closer = ptr + 1;

char close = *closer;

*closer = 0;

char word[MAX_WORD_SIZE] = "";

uint64 oldbot = myBot;

myBot = 0; // universal access to this transient json

FunctionResult result = InternalCall("^JSONParseCode", JSONParseCode, (char*)"TRANSIENT SAFE", jsonStart, NULL, word);

myBot = oldbot;

++count;

*closer = close;

if (result == NOPROBLEM_BIT) words[count] = AllocateHeap(word); // insert json object

else words[count] = AllocateHeap((char*)"bad-json");

}

oobJson = false;

return ptr + 1;

}

}

}

if (level > 2 && tokenControl & JSON_DIRECT_FROM_OOB)

{

ReportBug("Possible failure detecting JSON oob");

}

oobJson = 0; // give up

return ptr;

}

// OOB only separates ( [ { ) ] } - the rest remain joined as given

if (oobStart)

{

if (*ptr == '(' || *ptr == ')' || *ptr == '[' || *ptr == ']' || *ptr == '{' || *ptr == '}' || *ptr == ',') return ptr + 1;

bool quote = false;

--ptr;

while (*++ptr)

{

if (*ptr == '"' && *(ptr - 1) != '\\') quote = !quote;

if (quote) continue;

if (*ptr != ' ' && *ptr != '(' && *ptr != ')' && *ptr != '[' && *ptr != ']' && *ptr != '{' && *ptr != '}') continue;

break;

}

return ptr;

}

#ifdef PRIVATE_CODE

// Check for private hook function to find the end of the next word

static HOOKPTR fnTokenize = FindHookFunction((char*)"TokenizeWord");

if (fnTokenize)

{

char* end = ((TokenizeWordHOOKFN)fnTokenize)(ptr, words, count);

if (end && end > ptr) return end;

}

#endif

char utfcharacter[10];

char* endchar = IsUTF8(ptr, utfcharacter); // return after this character if it is valid

if (utfcharacter[1] ) // even in english mode, tolerate jp/zh punctuation to convert

{

unsigned char japanletter[8];

char* prior = ptr;

endchar = ptr;

// find end of utf8 stuff

while (endchar = IsUTF8(endchar, utfcharacter))

{

int kind = 0;

// swap terminal punctuation to english

if (IsJapanese(prior, (unsigned char*)&japanletter, kind) && kind == JAPANESE_PUNCTUATION)

{

if (japanletter[2] == 'F' && japanletter[3] == 'F' && japanletter[4] == '0' && japanletter[5] == '1') // full width !

{

if (prior == ptr)

{

strcpy(spawnWord, "!");

return ptr + 3;

}

else return prior;

}

else if (japanletter[2] == 'F' && japanletter[3] == 'F' && japanletter[4] == '0' && japanletter[5] == 'E') // full width .

{

if (prior == ptr)

{

strcpy(spawnWord, ".");

return ptr + 3;

}

else return prior;

}

else if (japanletter[2] == '3' && japanletter[3] == '0' && japanletter[4] == '0' && japanletter[5] == '2') // full width . chinese?

{

if (prior == ptr)

{

strcpy(spawnWord, ".");

return ptr + 3;

}

else return prior;

}

else if (japanletter[2] == 'F' && japanletter[3] == 'F' && japanletter[4] == '1' && japanletter[5] == 'F') // full width ?

{

if (prior == ptr)

{

strcpy(spawnWord, "?");

return ptr + 3;

}

else return prior;

}

// swap terminal punctuation to english

if (japanletter[0] == 0xef && japanletter[1] == 0xbc && japanletter[2] == 0x9f) //japan ？efbc9f

{

if (prior == ptr)

{

strcpy(spawnWord, "?");

return ptr + 3;

}

else return prior;

}

if (japanletter[0] == 0xe3 && japanletter[1] == 0x80 && japanletter[2] == 0x82) //japan 。e38082

{

if (prior == ptr)

{

strcpy(spawnWord, ".");

return ptr + 3;

}

else return prior;

}

if (japanletter[0] == 0xef && japanletter[1] == 0xbc && japanletter[2] == 0x82) //japan ！efbc81

{

if (prior == ptr)

{

strcpy(spawnWord, "!");

return ptr + 3;

}

else return prior;

}

}

prior = endchar;

}

}

// large repeat punctuation

if (*ptr == ptr[1] && ptr[1] == ptr[2] && ptr[2] == ptr[3] && IsPunctuation(*ptr))

{

c = *ptr;

char* at = ptr + 3;

while (*++at == c) *at = ' '; // eradicate junk

}

// ellipsis

if (!strncmp(ptr, ". . . ", 6))

{

memcpy(ptr, "... ", 6);

}

// punctuation inside closing quote. flip them

if (ptr[1] == '\'' && (*ptr == '.' || *ptr == '?' || *ptr == '!'))

{

ptr[1] = *ptr;

*ptr = '\'';

}

// special break on token

if (*ptr == '\'')

{

char word[MAX_WORD_SIZE];

ReadCompiledWord(ptr, word);

WORDP X = ApostropheBreak(word);

if (X) return ptr + strlen(word); // allow token

}

// break on article prefix l' and j' and t' and m' and s'

if ((*ptr == 'l' || *ptr == 'L' || *ptr == 'j' || *ptr == 'J' || *ptr == 't' || *ptr == 'T' || *ptr == 'm' || *ptr == 'M' || *ptr == 's' || *ptr == 'S')

&& ptr[1] == '\'')

{

return ptr + 2;

}

// break on article prefix qu'

if ((*ptr == 'Q' || *ptr == 'q' ) && ptr[1] == 'u' && ptr[2] == '\'')

{

return ptr + 3;

}

char token[MAX_WORD_SIZE];

ReadCompiledWord(ptr, token);

#ifdef PRIVATE_CODE

// Check for private hook function to check a token following local rules

static HOOKPTR fnIsToken = FindHookFunction((char*)"IsValidTokenWord");

if (fnIsToken)

{

if (((IsValidTokenWordHOOKFN) fnIsToken)(token))

{

return ptr + strlen(token);

}

}

#endif

// try emoiji pieces (since multiple utf8 emojis might be used)

char emoji[10];

if (IsUTF8(token, emoji) && emoji[1]) // utf characters will be >1 byte

{

WORDP E = FindWord(emoji, 0);

if (E && E->properties & EMOJI) return ptr + strlen(emoji);

}

WORDP EMO = FindWord(token);

if (EMO && EMO->properties & EMOJI) return ptr + strlen(EMO->word);

// serial no.

if (!stricmp(token, "no.") && !stricmp(priorToken, "serial"))

{

strcpy(spawnWord, "number");

return ptr + 3;

}

if (kind & QUOTERS) // quoted strings (but let *sigh* be emoji before this)

{

if (c == '\'' && ptr[1] == 's' && !IsAlphaUTF8(ptr[2])) return ptr + 2; // 's directly

if (c == '"')

{

if (tokenControl & SPLIT_QUOTE)

{

char* end1 = strchr(ptr + 1, '"');

if (end1) // strip the quotes and try agin

{

*ptr = ' ';

*end1 = ' ';

return ptr;

}

else return ptr + 1; // split up quote marks

}

else // see if merely highlighting a word

{

char* word = AllocateStack(NULL, maxBufferSize, false, 0);

ReadCompiledWord(ptr, word);

char* close = strchr(word + 1, '"');

ReleaseStack(word);

if (close && !strchr(word, ' ')) // we dont need quotes

{

int wordLen = close - word;

if (tokenControl & LEAVE_QUOTE) return ptr + wordLen + 1; // leave what is after the quotes e.g. a comma

*ptr = ' '; // kill off starting dq

ptr[wordLen] = ' '; // kill off closing dq

return ptr;

}

}

}

if (c == '\'' && tokenControl & SPLIT_QUOTE) // 'enemies of the state'

{

if (quotepending) quotepending = false;

else if (strchr(ptr + 1, '\'')) quotepending = true;

if (quotepending) return ptr + 1;

else if (ptr[1] == ' ' || ptr[1] == '.' || ptr[1] == ',') return ptr + 1;

}

if (c == '\'' && !(tokenControl & TOKEN_AS_IS) && !IsAlphaUTF8(ptr[1]) && !IsDigit(ptr[1])) return ptr + 1; // is this quote or apostrophe - for penntag dont touch it - for 've leave it alone also leave '82 alone

else if (c == '\'' && tokenControl & TOKEN_AS_IS) { ; } // for penntag dont touch it - for 've leave it alone also leave '82 alone

else if (c == '"' && tokenControl & TOKEN_AS_IS) return ptr + 1;

else if (c == '*' && ptr[1] == '.' && (IsLowerCase(ptr[2]) || IsDigit(ptr[2]))) {

char ext[MAX_WORD_SIZE];

ReadCompiledWord(ptr + 2, ext);

if (IsFileExtension(ext)) {

return ptr + strlen(ext) + 2;

}

}

else

{

char* end1 = HandleQuoter(ptr, words, count);

if (end1) return end1;

}

if (!IsDigit(ptr[1])) return ptr + 1; // just return isolated quote

}

// check if url or email address

if (IsMail(token))

{

char* atsign = strchr(token,'@');

char* period = strchr(atsign+1,'.');

char* emailEnd = atsign;

while (*++emailEnd && !IsInvalidEmailCharacter(*emailEnd)); // fred,andy@kore.com

if (period && period < emailEnd && IsAlphaUTF8(ptr[emailEnd-token-1]) && IsAlphaUTF8(ptr[emailEnd-token-2])) // top level domain is alpha

{

// find end of email domain, can be letters or numbers or hyphen

// there maybe be several parts to the domain

while (*++period && period < emailEnd)

{

if (!IsAlphaUTF8OrDigit(*period) && *period != '-' && *period != '.') return ptr + (period - token);

}

return ptr + (emailEnd - token);

}

}

size_t urlLen = strlen(token);

if (IsUrl(token, token + urlLen))

{

char* urlEnd = ptr + urlLen - 1;

// stop at trailing character that is likely to be the next token

if (*urlEnd == ',' || *urlEnd == ';' || *urlEnd == '|' || *urlEnd == '<' || *urlEnd == '>' || *urlEnd == '{' || *urlEnd == '(' || *urlEnd == '[' || *urlEnd == '?') --urlLen;

return ptr + urlLen;

}

// copyright, registered, trademark

char* atend = strchr(token, '@');

if (atend && atend != token)

{

if ((atend[1] == 't' || atend[1] == 's') && atend[2] == 'm' && (!atend[3] || IsPunctuation(atend[3])))

{

if (atend[3]) --urlLen; // discard punc

return ptr + urlLen - 3;

}

else if ((atend[1] == 'r' || atend[1] == 'c') && (!atend[2] || IsPunctuation(atend[2])))

{

if (atend[2]) --urlLen; // discard punc

return ptr + urlLen - 2;

}

}

WORDP X = FindWord(token);

size_t xx = strlen(token);

if (X && X->properties & EMOJI) return ptr + xx;

if (X && !IsPureNumber(token) && token[xx - 1] != '?' && token[xx - 1] != '!' && token[xx - 1] != ',' && token[xx - 1] != ';' && token[xx - 1] != ':') // we know the word and it cant be a number

{

if (!IS_NEW_WORD(X) || (X->systemFlags & PATTERN_WORD)) // if we just created it and not to protect testpattern

{

if (X->properties || (X->systemFlags & PATTERN_WORD)) // meaningful word, not merely phrase header like a. xxx

return ptr + xx;

}

}

// embedded punctuation

char* embed = strchr(token, '?');

if (embed && embed != token && embed[1] && !IsUrl(token, embed)) *embed = 0; // break off love?i, but not ? to introduce the query string in an URL

embed = strchr(token, ')');

if (embed && embed != token ) *embed = 0; // break off 61.3)

if (embed && embed == token && embed[1]) embed[1] = 0; // break off )box.

embed = strchr(token, '.');

if (embed && embed != token && IsAlphaUTF8(embed[1]))

{// break off probable 2 words. BUT U.S. Cellular should not be broken.

size_t l = strlen(token);

int front = (embed - token);

if (front > 4 && embed[1] && !IsFileExtension(embed+1))

{

*embed = 0;

}

}

if (*token == '.' && IsAlphaUTF8(token[1])) token[1] = 0; // break off .he

// if this was 93302-42345 then we need to keep - separate, not as minus

if (*token == '-' && IsInteger(token + 1, false, numberStyle) && IsInteger(priorToken, false, numberStyle))

{

return ptr + 1;

}

// could be in the middle of splitting two times, 2pm-3 or 2:30-3:30

if (*token == '-' && ParseTime(priorToken, NULL, NULL)) return ptr + 1;

if (strlen(token) > 1 && IsDigit(*priorToken) && *(ptr - 1) != ' ' && (*token == 'x' || *token == 'X') && IsDigit(*(token + 1))) return ptr + 1; // continuing a 4x4 split, not 4 X4's

X = FindWord(token); // in case token embedded changed

xx = strlen(token);

if (X && X->properties & EMOJI) return ptr + xx;

if (X && !IsDigit(*token) && token[xx - 1] != '?' && token[xx - 1] != '!' && token[xx - 1] != ',' && token[xx - 1] != ';' && token[xx - 1] != ':') // we know the word and it cant be a number

{

if (!IS_NEW_WORD(X) || (X->systemFlags & PATTERN_WORD)) // if we just created it and not to protect testpattern

{

if (X->properties || (X->systemFlags & PATTERN_WORD))

return ptr + xx;

}

}

char* slash = strchr(token, '/');

if (slash) // dont break up word like km/h

{

if (slash == token) return ptr + 1;

char* slash1 = strchr(slash + 1, '/'); // keep possible date?

if (!slash1) // split it off if not date info

{

*slash = 0;

// not dual number fraction like 1 / 4 or 50 / 50

if (IsDigit(*token) && IsNumber(token) && IsDigit(slash[1]) && IsNumber(slash + 1))

{

*slash = '/'; // let be a token

}

}

}

size_t l = strlen(token);

// ends in question or exclaim

if (token[l - 1] == '!' || token[l - 1] == '?')

{

if (!strcmp(token, ".?")) // some people type both

{

strcpy(spawnWord, "?"); // insert json object

return ptr + 2;

}

if (l > 1) token[--l] = 0; // remove it from token

}

if (*ptr == '?') return ptr + 1; // we dont have anything that should join after ? but ) might start emoticon

if (*ptr == 0xc2 && ptr[1] == 0xbf) return ptr + 2; // inverted spanish ?

if (*ptr == 0xc2 && ptr[1] == 0xa1) return ptr + 2; // inverted spanish !

if (IsAlphaUTF8(*ptr) && ptr[1] == '.' && ptr[2] == ' ' && IsUpperCase(*ptr)) return ptr + 2; // single letter abbreviaion period like H.

if (*ptr == '.' && ptr[1] == '.' && ptr[2] == '.' && ptr[3] != '.') return ptr + 3; // ...

if (*ptr == '-' && ptr[1] == '-' && ptr[2] == '-') ptr[2] = ' '; // change excess --- to space

if (*ptr == '-' && ptr[1] == '-' && (ptr[2] == ' ' || IsAlphaUTF8(ptr[2]))) return ptr + 2; // the -- break

if (*ptr == ';' && ptr[1] != ')' && ptr[1] != '(') return ptr + 1; // semicolon not emoticon

if (*ptr == ',' && ptr[1] != ':') return ptr + 1; // comma not emoticon

if (*ptr == '|') return ptr + 1;

if (*ptr == '(' || *ptr == '[' || *ptr == '{') return ptr + 1;

// if we actually have this token in dictionary, accept it. (eg abbreviations, etc)

WORDP Z = FindWord(token); // either case

if (Z && !IS_NEW_WORD(Z) && token[l - 1] != '?' && token[l - 1] != '!' && token[l - 1] != ',')

{

if (IsDigit(*token) && token[l - 1] == '.') {} // assume no numbers end in . 4. becomes 4 .

else if (Z->properties || Z->systemFlags & PATTERN_WORD) return ptr + l; // not generated by user input

}

// if token ends in period and does not start with digit (not float) and word we know,

// return prior

char* q = strchr(token, '?');

if (q)

{

if (q[1] && !q[2]) return ptr + l; // don?t or it?s

if ((*token == 'i' || *token== 'I') && token[1] == '?' && token[2]) return ptr + l; // I?d or i?ve

}

if (*token == '.' && !IsInteger(token + 1, false, numberStyle) && FindWord(token + 1))

{

if (token[1] != '?') return ptr + 1; // sentence end then word we know

strcpy(spawnWord, "?");

return ptr+2; // delete the period

}

if (token[l - 1] == '.' && FindWord(token, l - 1)) return ptr + l - 1;

// find current token which has | after it and separate it, like myba,atat,joha

char* pipe = strchr(token + 1, '|');

if (pipe)

{

*pipe = 0; // break apart token

}

// check for apostrophe

char* apost = strchr(token, '\'');

if (apost && ApostropheBreak(apost))

{

return ptr + (apost - token);

}

// see if there is a known currency symbol in the token

char* currencynumber = token;

char* currency = (char*)GetCurrency((unsigned char*)token, currencynumber);

// check for float

if (strchr(token, numberPeriod) || strchr(token, 'e') || strchr(token, 'E'))

{

// use currency if found

char* number = currencynumber;

char* at = number;

bool seenExponent = false;

bool seenPeriod = false;

while (*++at && (IsDigit(*at) || *at == ',' || *at == '.' || (!seenExponent && (*at == 'e' || *at == 'E')) || IsSign(*at)))

{

if (currency && at == currency) break; // seen enough if reached a currency suffix

if (*at == 'e' || *at == 'E') seenExponent = true; // exponent can only appear once, 10e4euros

// period AFTER float like 1.0. w space or end

if (*at == numberPeriod && IsDigit(*(at-1)) && seenPeriod && !at[1])

{

return ptr + (at - token);

}

if (*at == numberPeriod) seenPeriod = true;

}

// may be units or currency attached, so dont split that apart

if (IsFloat(number, at, numberStyle) && !UnitSubstitution(at,0)) // $50. is not a float, its end of sentene

{

if (currency && at == currency) at += strlen(currency);

if (*at == '%') ++at;

if (*at == 'k' || *at == 'K' || *at == 'm' || *at == 'M' || *at == 'B' || *at == 'b')

{

if (!at[1]) ++at;

}

return ptr + (at - token);

}

}

// check for negative number

if (*currencynumber == '-' && IsDigit(currencynumber[1]))

{

char* at = currencynumber;

while (*++at && (IsDigit(*at) || *at == '.' || *at == ',')) { ; }

if (!*at) {