#include "common.h"

typedef struct InternationalPhones

{

const char* countrydigits;

const char* concept;

const int countrycode;

const int areacode;

const int exchangecode; // variable digits counts are put here

const int subscriber;

} InternationalPhones;

InternationalPhones phoneData[] = {

{"1", "~usinternational",1,3,3,4},

{"44","~ukinternational",2,0,3,6},

{"49","~deinternational",2,0,3,4},

{"81","~jpinternational",2,0,3,4},

{"34","~esinternational",2,3,3,3},

{"52","~mxinternational",2,2,4,4},

NULL

};

typedef struct JapanCharInfo

{

const char* charword; // japanese 3 byte character

const unsigned char convert; // ascii equivalent

} JapanCharInfo;

bool outputapicall = false;

char* currentInput = NULL; // latest input we are processing from GetNextInput for error reporting

static HEAPREF startSupplementalInput = NULL;

unsigned int startSentence;

bool moreToComeQuestion = false; // is there a ? in later sentences

bool moreToCome = false; // are there more sentences pending

unsigned int endSentence;

bool fullfloat = false; // 2 float digits or all

int numberStyle = AMERICAN_NUMBERS;

FILE* docOut = NULL;

bool showBadUTF = false; // log bad utf8 characer words

static bool blockComment = false;

bool singleSource = false; // in ReadDocument treat each line as an independent sentence

bool newline = false;

int docSampleRate = 0;

int docSample = 0;

uint64 docVolleyStartTime = 0;

unsigned char hasHighChar = 0;

char conditionalCompile[MAX_CONDITIONALS + 1][50];

int conditionalCompiledIndex = 0;

char numberComma = ',';

char numberPeriod = '.';

char tmpWord[MAX_WORD_SIZE]; // globally visible scratch word

char* userRecordSourceBuffer = 0; // input source for reading is this text stream of user file

unsigned char toHex[16] = { '0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F' };

int BOM = NOBOM; // current ByteOrderMark

static char holdc = 0; // holds extra character from readahead

unsigned char utf82extendedascii[128];

unsigned char extendedascii2utf8[128] =

{

0,0xbc,0xa9,0xa2,0xa4,0xa0,0xa5, 0xa7,0xaa,0xab, 0xa8,0xaf,0xae,0xac,0x84,0x85,0x89,0xa6,0x86,0xb4,

0xb6,0xb2,0xbb,0xb9,0xbf,0x96,0x9c,0xa2,0xa3,0xa5, 0x00,0xa1,0xad,0xb3,0xba,0xb1,0x91,

};

typedef struct NUMBERDECODE

{

int word; // word of a number index

unsigned int length; // length of word

int64 value; // value of word

int realNumber; // the type: DIGIT_NUMBER(1), CURRENCY_NUMBER, PLACETYPE_NUMBER, ROMAN_NUMBER, WORD_NUMBER,

} NUMBERDECODE;

static NUMBERDECODE* numberValues[MAX_MULTIDICT]; // multilanguage

static JapanCharInfo japanSet[] =

{

// ranged characters - only substituted in all variables

{"\xEF\xBC\xA1",(unsigned char)'A'}, // "Ａ"

{"\xEF\xBC\xBA",(unsigned char)'Z'}, //"Ｚ"

{"\xEF\xBD\x81",(unsigned char)'a'}, // "ａ"

{"\xEF\xBD\x9A",(unsigned char)'z'}, // "ｚ"

{"\xEF\xBC\x90",(unsigned char)'0'}, // "０"

{"\xEF\xBC\x99",(unsigned char)'9' },// "９"

// one-off characters always substituted in variables

{"\xE3\x80\x82",(unsigned char)'.'}, // "。" ideographic_full_stop, code point U+3002

{"\xEF\xBC\x8E", (unsigned char)'.'}, // "．" fullwidth full stop, code point U+FF0E

{"\xEF\xBC\x8D", (unsigned char)'-'},// "－" fullwidth hyphen-minus, code point U+FF0D uft8

{"\xEF\xBC\xBF", (unsigned char)'_'}, // "＿"fullwidth low line, code point U+FF3F

{"\xEF\xBC\x8E", (unsigned char)'.'}, // "。" full width stop

{"\xEF\xBD\xA1", (unsigned char)'.'}, // "｡" jp half-period stop

// From https://en.wikipedia.org/wiki/List_of_Japanese_typographic_symbols

{0,0}

};

static JapanCharInfo japanControlSet[] = // always change pattern and output

{

{ "\xE3\x80\x80",(unsigned char)' ' },// "　" ideographic space, code point U+3000

{ "\xEF\xBC\x88", (unsigned char)'(' }, // "（", jp open paren, U+FF08

{ "\xEF\xBC\x89", (unsigned char)')' }, // "）", jp close paren, U+FF09

{ "\xEF\xBD\x9B", (unsigned char)'{' }, // "｛", jp open curly brace, U+FF5B

{ "\xEF\xBD\x9D", (unsigned char)'}' }, // "｝", jp close curly brace, U+FF5D

{ "\xEF\xBC\xBB", (unsigned char)'(' }, // "［", jp open bracket, U+FF3B

{ "\xEF\xBC\xBD", (unsigned char)')' }, // "］", jp close bracket, U+FF3D

{ 0,0 }

};

typedef struct CURRENCYDECODE

{

int word; // word index of currency abbrev

int concept; // word index of concept to mark it

} CURRENCYDECODE;

static CURRENCYDECODE* currencies[MAX_MULTIDICT];

static int* monthnames[MAX_MULTIDICT];

static int* topleveldomains;

static int* emojishortnames;

unsigned char toLowercaseData[256] = // convert upper to lower case

{ // encoder/decoder https://mothereff.in/utf-8

// table of values site:

// https://www.utf8-chartable.de/unicode-utf8-table.pl?start=12288

0,1,2,3,4,5,6,7,8,9, 10,11,12,13,14,15,16,17,18,19,

20,21,22,23,24,25,26,27,28,29, 30,31,32,33,34,35,36,37,38,39,

40,41,42,43,44,45,46,47,48,49, 50,51,52,53,54,55,56,57,58,59,

60,61,62,63,64,'a','b','c','d','e',

'f','g','h','i','j','k','l','m','n','o',

'p','q','r','s','t','u','v','w','x','y',

'z',91,92,93,94,95,96,97,98,99, 100,101,102,103,104,105,106,107,108,109,

110,111,112,113,114,115,116,117,118,119, 120,121,122,123,124,125,126,127,128,129,

130,131,132,133,134,135,136,137,138,139, 140,141,142,143,144,145,146,147,148,149,

150,151,152,153,154,155,156,157,158,159, 160,161,162,163,164,165,166,167,168,169,

170,171,172,173,174,175,176,177,178,179, 180,181,182,183,184,185,186,187,188,189,

190,191,192,193 ,194 ,195 ,196 ,197 ,198 ,199 , 200 ,201 ,202 ,203 ,204 ,205 ,206 ,207 ,208 ,209 ,

210 ,211,212 ,213 ,214 ,215 ,216 ,217 ,218 ,219 , 220 ,221 ,222,223,224,225,226,227,228,229,

230,231,232,233,234,235,236,237,238,239, 240,241,242,243,244,245,246,247,248,249,

250,251,252,253,254,255

};

unsigned char toUppercaseData[256] = // convert lower to upper case

{

0,1,2,3,4,5,6,7,8,9, 10,11,12,13,14,15,16,17,18,19,

20,21,22,23,24,25,26,27,28,29, 30,31,32,33,34,35,36,37,38,39,

40,41,42,43,44,45,46,47,48,49, 50,51,52,53,54,55,56,57,58,59,

60,61,62,63,64,'A','B','C','D','E',

'F','G','H','I','J','K','L','M','N','O',

'P','Q','R','S','T','U','V','W','X','Y',

'Z',91,92,93,94,95,96,'A','B','C', 'D','E','F','G','H','I','J','K','L','M',

'N','O','P','Q','R','S','T','U','V','W', 'X','Y','Z',123,124,125,126,127,128,129,

130,131,132,133,134,135,136,137,138,139, 140,141,142,143,144,145,146,147,148,149,

150,151,152,153,154,155,156,157,158,159, 160,161,162,163,164,165,166,167,168,169,

170,171,172,173,174,175,176,177,178,179, 180,181,182,183,184,185,186,187,188,189,

190,191,192-32,193 ,194 ,195 ,196 ,197 ,198 ,199 , 200 ,201 ,202 ,203 ,204 ,205 ,206 ,207 ,208 ,209 ,

210 ,211 ,212 ,213 ,214 ,215 ,216 ,217 ,218 ,219 , 220 ,221 ,222-32,223,224,225,226,227,228,229,

230,231,232,233,234,235,236,237,238,239, 240,241,242,243,244,245,246,247,248,249,

250,251,252,253,254,255

};

unsigned char isVowelData[256] = // english vowels

{

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,

0,0,0,0,0,'a',0,0,0,'e', 0,0,0,'i',0,0,0,0,0,'o',

0,0,0,0,0,'u',0,0,0,'y', 0,0,0,0,0,0,0,'a',0,0,

0,'e',0,0,0,'i',0,0,0,0, 0,'o',0,0,0,0,0,'u',0,0,

0,'y',0,0,0,0,0,0,0,0,

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,

0,0,0,0,0,0

};

signed char nestingData[256] = // the matching bracket things: () [] {}

{

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,

1,-1,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0, // ()

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,

0,0,0,0,0,0,0,0,0,0, 0,1,0,-1,0,0,0,0,0,0, // [ ]

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,

0,0,0,1,0,-1,0,0,0,0, // { }

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,

0,0,0,0,0,0

};

unsigned char legalNaming[256] = // what we allow in script-created names (like ~topicname or $uservar)

{

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,'\'',

0,0,0,0,0,'-','.',0,'0','1', '2','3','4','5','6','7','8','9',0,0,

0,0,0,0,0,'A','B','C','D','E', 'F','G','H','I','J','K','L','M','N','O',

'P','Q','R','S','T','U','V','W','X','Y', 'Z',0,0,0,0,'_',0,'A','B','C',

'D','E','F','G','H','I','J','K','L','M',

'N','O','P','Q','R','S','T','U','V','W', 'X','Y','Z',0,0,0,0,0,1,1,

1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,

1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,

1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,

1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,

1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,

1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,

1,1,1,1,1,1,

};

unsigned char punctuation[256] = // direct lookup of punctuation -- // / is normal because can be a numeric fraction

{

ENDERS,0,0,0,0,0,0,0,0,SPACES, // 10 null \t

SPACES,0,0,SPACES,0,0,0,0,0,0, // 20 \n \r

0,0,0,0,0,0,0,0,0,0, // 30

0,0,SPACES,ENDERS,QUOTERS,SYMBOLS,SYMBOLS,ARITHMETICS,CONVERTERS,QUOTERS, // 40 space ! " # $ % & '

BRACKETS,BRACKETS,ARITHMETICS | QUOTERS , ARITHMETICS,PUNCTUATIONS, ARITHMETICS | ENDERS,ARITHMETICS | ENDERS,ARITHMETICS,0,0, // () * + , - . /

0,0,0,0,0,0,0,0,ENDERS,ENDERS, // 60 : ;

BRACKETS,ARITHMETICS,BRACKETS,ENDERS,SYMBOLS,0,0,0,0,0, // 70 < = > ? @

0,0,0,0,0,0,0,0,0,0, // 80

0,0,0,0,0,0,0,0,0,0, // 90

0,BRACKETS,0,BRACKETS,ARITHMETICS,0,CONVERTERS,0,0,0, // 100 [ \ ] ^ ` _ is normal \ is unusual

0,0,0,0,0,0,0,0,0,0, // 110

0,0,0,0,0,0,0,0,0,0, // 120

0,0,0,BRACKETS,PUNCTUATIONS,BRACKETS,SYMBOLS,0,0,0, // 130 { | } ~

0,0,0,0,0,0,0,0,0,0, // 140

0,0,0,0,0,0,0,0,0,0, // 150

0,0,0,0,0,0,0,0,0,0, // 160

0,0,0,0,0,0,0,0,0,0, // 170

0,0,0,0,0,0,0,0,0,0, // 180

0,0,0,0,0,0,0,0,0,0, // 190

0,0,0,0,0,0,0,0,0,0, // 200

0,0,0,0,0,0,0,0,0,0, // 210

0,0,0,0,0,0,0,0,0,0, // 220

0,0,0,0,0,0,0,0,0,0, // 230

0,0,0,0,0,0,0,0,0,0, // 240

0,0,0,0,0,0,0,0,0,0, // 250

0,0,0,0,0,0,

};

unsigned char realPunctuation[256] = // punctuation characters

{

0,0,0,0,0,0,0,0,0,0, // 10

0,0,0,0,0,0,0,0,0,0, // 20

0,0,0,0,0,0,0,0,0,0, // 30

0,0,0,PUNCTUATIONS,0,0,0,0,0,0, // 40 !

0,0,0 , 0,PUNCTUATIONS, 0,PUNCTUATIONS,0,0,0, // , .

0,0,0,0,0,0,0,0,PUNCTUATIONS,PUNCTUATIONS, // 60 : ;

0,0,0,PUNCTUATIONS,0,0,0,0,0,0, // ?

0,0,0,0,0,0,0,0,0,0, // 80

0,0,0,0,0,0,0,0,0,0, // 90

0,0,0,0,0,0,0,0,0,0, // 100

0,0,0,0,0,0,0,0,0,0, // 110

0,0,0,0,0,0,0,0,0,0, // 120

0,0,0,0,0,0,0,0,0,0, // 130

0,0,0,0,0,0,0,0,0,0, // 140

0,0,0,0,0,0,0,0,0,0, // 150

0,0,0,0,0,0,0,0,0,0, // 160

0,0,0,0,0,0,0,0,0,0, // 170

0,0,0,0,0,0,0,0,0,0, // 180

0,0,0,0,0,0,0,0,0,0, // 190

0,0,0,0,0,0,0,0,0,0, // 200

0,0,0,0,0,0,0,0,0,0, // 210

0,0,0,0,0,0,0,0,0,0, // 220

0,0,0,0,0,0,0,0,0,0, // 230

0,0,0,0,0,0,0,0,0,0, // 240

0,0,0,0,0,0,0,0,0,0, // 250

0,0,0,0,0,0,

};

unsigned char isAlphabeticDigitData[256] = // non-digit number starter (+-.) == 1 isdigit == 2 isupper == 3 leower == 4 isletter >= 3 utf8 == 5

{

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,1,1,0,0,0, // # and $

0,0,0,VALIDNONDIGIT,0,VALIDNONDIGIT,VALIDNONDIGIT,0,VALIDDIGIT,VALIDDIGIT, VALIDDIGIT,VALIDDIGIT,VALIDDIGIT,VALIDDIGIT,VALIDDIGIT,VALIDDIGIT,VALIDDIGIT,VALIDDIGIT,0,0, // + and . and -

0,0,0,0,0,VALIDUPPER,VALIDUPPER,VALIDUPPER,VALIDUPPER,VALIDUPPER,

VALIDUPPER,VALIDUPPER,VALIDUPPER,VALIDUPPER,VALIDUPPER,VALIDUPPER,VALIDUPPER,VALIDUPPER,VALIDUPPER,VALIDUPPER,

VALIDUPPER,VALIDUPPER,VALIDUPPER,VALIDUPPER,VALIDUPPER,VALIDUPPER,VALIDUPPER,VALIDUPPER,VALIDUPPER,VALIDUPPER,

VALIDUPPER,0,0,0,0,0,0,4,4,4,

VALIDLOWER,VALIDLOWER,VALIDLOWER,VALIDLOWER,VALIDLOWER,VALIDLOWER,VALIDLOWER,VALIDLOWER,VALIDLOWER,VALIDLOWER,

VALIDLOWER,VALIDLOWER,VALIDLOWER,VALIDLOWER,VALIDLOWER,VALIDLOWER,VALIDLOWER,VALIDLOWER,VALIDLOWER,VALIDLOWER,

VALIDLOWER,VALIDLOWER,VALIDLOWER,0,0,0,0,0, VALIDUTF8,VALIDUTF8,

VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8, VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,

VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8, VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,

VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8, VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,

VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8, VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,

VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8, VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,

VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8, VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,

VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8,VALIDUTF8

};

unsigned int roman[256] = // values of roman numerals

{

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0, //20

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0, //40

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0, //60

0,0,0,0,0,0,0,100,500,0, 0,0,0,1,0,0,50,1000,0,0, //80 C(67)=100 D(68)=500 I(73)=1 L(76)=50 M(77)=1000

0,0,0,0,0,0,5,0,10,0, 0,0,0,0,0,0,0,0,0,0, //100 V(86)=5 X(88)=10

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,

};

unsigned char isComparatorData[256] = // = < > & ? ! %

{

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0, //20

0,0,0,0,0,0,0,0,0,0, 0,0,0,1,0,0,0,0,1,0, //40 33=! 38=&

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0, //60

1,1,1,1,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0, //80 < = > ?

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0, //100

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0, //120

0,0,0,0,0,0,0,0,0,0,

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,

0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,

0,0,0,0,0,0

};

unsigned char decimalMarkData[3] = // American, Indian, French

{

'.','.',','

};

unsigned char digitGroupingData[3] = // American, Indian, French

{

',',',','.'

};

/////////////////////////////////////////////

// STARTUP

/////////////////////////////////////////////

void Clear_CRNL(char* incoming)

{

char* at = incoming;

while ((at = strchr(at, '\r')))

{

*at = ' ';

if (at[1] == '\n') at[1] = ' '; // if had one, likely have the other

}

at = incoming;

while ((at = strchr(at, '\n'))) *at = ' ';

}

void InitUniversalTextUtilities()

{

ClearNumbers();

memset(utf82extendedascii, 0, 128);

extendedascii2utf8[0xe1 - 128] = 0x9f; // german fancy ss

for (unsigned int i = 0; i < 128; ++i)

{

unsigned char c = extendedascii2utf8[i];

if (c) c -= 128; // remove top bit

utf82extendedascii[c] = (unsigned char)i;

}

char word[MAX_WORD_SIZE];

char* limit;

char* stack;

int* data;

FILE* in;

size_t size;

char file[SMALL_WORD_SIZE];

// universal system livedata

sprintf(word, (char*)"%s/systemessentials.txt", systemFolder);

ReadSubstitutes(word, 0, NULL, DO_ESSENTIALS, true);

sprintf(word, (char*)"%s/queries.txt", systemFolder);

ReadQueryLabels(word);

// most up to date list of all TLDs at http://data.iana.org/TLD/tlds-alpha-by-domain.txt

sprintf(file, (char*)"%s/topleveldomains.txt", systemFolder);

in = FopenStaticReadOnly(file);

if (in)

{

stack = InfiniteStack(limit, "inittlds");

data = (int*)stack;

while (ReadALine(readBuffer, in) >= 0) // top level domain

{

if (*readBuffer == '#' || *readBuffer == 0) continue;

ReadCompiledWord(readBuffer, word);

*data++ = Heap2Index(AllocateHeap(word, 0));

}

*data++ = 0; // terminal value for end detection

size = (char*)data - stack;

size /= 8; // 64bit chunks

topleveldomains = (int*)AllocateHeap(stack, size + 1, 8);

ReleaseInfiniteStack();

FClose(in);

}

// create the empty word

StoreWord("", AS_IS);

}

void Translate_UTF16_2_UTF8(char* input)

{

char* output = input--;

while (*++input)

{

char c = *input;

if (c == '\\' && input[1] == 'u' && IsHexDigit(input[2]) && IsHexDigit(input[3]) && IsHexDigit(input[4]) && IsHexDigit(input[5]))

{

char* utf8 = UTF16_2_UTF8(input + 1, false);

if (utf8)

{

size_t len = strlen(utf8);

input += 5; // skip what we read

// shrink to fit hole

memmove(input + len, input + 6, strlen(input + 5));

strncpy(input, utf8, len);

input += len - 1;

}

}

// else leave input/output alone

}

}

char* UTF16_2_UTF8(char* in,bool withinquote)

{ // u0000 notation

static char answer[20];

*answer = 'x'; //default

answer[1] = 0;

//First code point Last code point Byte 1 Byte 2 Byte 3 Byte 4

// direct utf16 to ascii converts

int n = atoi(in + 1);

if (n == 2019 || n == 2018)

{

return "\'";// left and right curly ' to normal '

}

char d = in[4];

if (n == 201 && (d == 'c' || d == 'd' || d == 'C' || d == 'D'))

{

if (withinquote) return "\\\""; // escape it

return "\""; // left and right curly dq to normal dq

}

if (n == 22) return "\\\""; // std ascii

if (n == 27) return "\\'"; // std quote if (n == 27) return "'"; // std quote if (n == 27) return "\\'"; // std quote if (n == 27) return "'"; // std quote

if (n == 2013 || n == 2014) // preserve m and n dash for now for regression test

{

return NULL; // decline to change

}

// U + 0080 U + 07FF 110xxxxx 10xxxxxx

// U + 0800 U + FFFF 1110xxxx 10xxxxxx 10xxxxxx

// U + 10000[nb 2]U + 10FFFF 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx

if (*in != 'u' ) return answer; // not legal

// 20AC is binary 0010 0000 1010 1100.

unsigned char c = in[1];

if (c < 'a' && c >= 'A') c -= 'A' + 'a'; // flip to lowercase if uppercase

unsigned char hex1 = (IsDigit(c)) ? (c - '0') : (10 + c - 'a');

c = in[2];

if (c < 'a' && c >= 'A') c -= 'A' + 'a';

unsigned char hex2 = (IsDigit(c)) ? (c - '0') : (10 + c - 'a');

c = in[3];

if (c < 'a' && c >= 'A') c -= 'A' + 'a';

unsigned char hex3 = (IsDigit(c)) ? (c - '0') : (10 + c - 'a');

c = in[4];

if (c < 'a' && c >= 'A') c -= 'A' + 'a';

unsigned char hex4 = (IsDigit(c)) ? (c - '0') : (10 + c - 'a');

// private use codepoints

// The main range in the BMP is U + E000..U + F8FF, containing 6, 400 private - use characters.That range is often referred to as the Private Use Area(PUA).But there are also two large ranges of supplementary private - use characters, consisting of most of the code points on Planes 15 and 16: U + F0000..U + FFFFD and U + 100000..U + 10FFFD.Together those ranges allocate another 131, 068 private - use characters.Altogether, then, there are 137, 468 private - use characters in Unicode.

if (hex1 == 0x0d || hex1 == 0x0e) return NULL; // utf16 emoji which may be considered unallocated for utf8

if (hex1 == 0x0f && hex2 < 0x09) return NULL; // private utf16

if (hex1 == 0 && hex2 == 0 && hex3 <= 7) // U + 00 00 00 7F 0xxxxxxx

{// 1 byte utf (normal ascii)

*answer = (hex3 << 4) + hex4;

answer[1] = 0;

}

else if (hex1 == 0 && hex2 <= 7) // 2 byte message 0080-07FF 110xxxxx 10xxxxxx

{

answer[0] = 0xc0 | (hex2 << 2) | (hex3 >> 2); // 3+2 = 5 bit

answer[1] = 0x80 | ((hex3 & 0x03) << 4) | hex4; // 2+4 = 6bit

answer[2] = 0;

}

else if ((hex1 == 8 && hex2 >= 8) || hex1) // 3 byte message 0800-FFFF 1110xxxx 10xxxxxx 10xxxxxx

{

answer[0] = 0xe0 | hex1; // 4 = 4 bit

answer[1] = 0x80 | (hex2 << 2) | (hex3 >> 2); // 4 + 2bit = 6 bit

answer[2] = 0x80 | ((hex3 & 0x03) << 4) | hex4; // 2+4 = 6 bit

answer[3] = 0;

}

else if ((hex1 == 0 && hex2 == 0) || (hex1 == 0 && hex2 <= 7)) // 4 byte message

{// WRONG? BUG

// 10000-10FFFF 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx

// disable

answer[0] = 'x';

answer[1] = 0;

}

else

{

*answer = 'x'; //default

answer[1] = 'x'; //default

answer[2] = 0;

}

return answer;

}

bool IsComparator(char* word)

{

return (!stricmp(word, (char*)"&=") || !stricmp(word, (char*)"|=") || !stricmp(word, (char*)"^=") || !stricmp(word, (char*)"=") || !stricmp(word, (char*)"+=") || !stricmp(word, (char*)"-=") || !stricmp(word, (char*)"/=") || !stricmp(word, (char*)"*="));

}

void ClearNumbers()

{

memset(numberValues,0,sizeof(numberValues)); // heap is discarded, need to discard this

memset(currencies, 0, sizeof(currencies));

}

char* ReadTokenMass(char* ptr, char* word)

{

ptr = SkipWhitespace(ptr);

while (*ptr && *ptr != ' ') *word++ = *ptr++; // find end of word

*word = 0;

return ptr;

}

bool LegalVarChar(char at)

{

return (IsAlphaUTF8OrDigit(at) || at == '_' || at == '-');

}

void InitTextUtilitiesByLanguage(char* lang)

{

char word[MAX_WORD_SIZE];

char value[MAX_WORD_SIZE];

char kind[MAX_WORD_SIZE];

char* limit;

char* stack;

int* data;

size_t size;

char file[SMALL_WORD_SIZE];

sprintf(file, (char*)"%s/%s/numbers.txt", livedataFolder, lang);

FILE* in = FopenStaticReadOnly(file); // LIVEDATA substitutes or from script TOPIC world

if (in)

{

stack = InfiniteStack(limit, "initnumbers");

data = (int*)stack;

// WORD_NUMBER is a word that implies a number value, like dozen

// ORDINAL_NUMBER is word_number and a real number that is a place number like first, second, etc

// REAL_NUMBER is a word that directly represents a number, like two

// FRACTION_NUMBER is a word that implies a faction value like half

while (ReadALine(readBuffer, in) >= 0) // once 1 REALNUMBER

{

if (*readBuffer == '#' || *readBuffer == 0) continue;

char* ptr = ReadCompiledWord(readBuffer, word);

ptr = ReadCompiledWord(ptr, value);

ptr = ReadCompiledWord(ptr, kind);

int64 val;

ReadInt64(value, val);

int kindval = 0;

uint64 flags = ADJECTIVE | NOUN | ADJECTIVE_NUMBER | NOUN_NUMBER;

// if (!stricmp(kind, "DIGIT_NUMBER")) kindval = DIGIT_NUMBER;

// else

if (!stricmp(kind, "FRACTION_NUMBER")) kindval = FRACTION_NUMBER; // a fractional value like quarter

else if (!stricmp(kind, "WORD_NUMBER")) kindval = WORD_NUMBER; //implies a number value, like dozen

else if (!stricmp(kind, "REAL_NUMBER")) kindval = REAL_NUMBER; //directly represents a number, like two

else if (!stricmp(kind, "ORDINAL_NUMBER")) //place number like first

{

kindval = PLACETYPE_NUMBER;

flags |= PLACE_NUMBER;

}

else myexit("Bad number table");

char* w = StoreFlaggedWord(word, AS_IS | flags, NOUN_NODETERMINER)->word;

int index = Heap2Index(w);

*data++ = index;

*data++ = strlen(word);

*(int64*)data = val; // int64 value

data += 2;

*data++ = kindval;

*data++ = 0; // alignment to keep block at int64 style

}

*data++ = 0; // terminal value for end detection

size = (char*)data - stack;

size /= 8; // 64bit chunks

numberValues[languageIndex] = (NUMBERDECODE*)AllocateHeap(stack, size + 1, 8);

ReleaseInfiniteStack();

FClose(in);

}

sprintf(file, (char*)"%s/%s/currencies.txt", livedataFolder, lang);

in = FopenStaticReadOnly(file);

if (in)

{

stack = InfiniteStack(limit, "initcurrency");

data = (int*)stack;

while (ReadALine(readBuffer, in) >= 0) // $ ~dollar

{

if (*readBuffer == '#' || *readBuffer == 0) continue;

char* ptr = ReadCompiledWord(readBuffer, word);

MakeLowerCase(word);

ptr = ReadCompiledWord(ptr, kind);

MakeLowerCase(kind);

if (*kind != '~') myexit("Bad currency table");

char* w = StoreWord(word, AS_IS | CURRENCY)->word; // currency name

int index = Heap2Index(w);

*data++ = index;

w = StoreWord(kind, AS_IS)->word; // currency name

index = Heap2Index(w);

*data++ = index;

}

*data++ = 0; // terminal value for end detection

size = (char*)data - stack;

size /= 8; // 64bit chunks

currencies[languageIndex] = (CURRENCYDECODE*)AllocateHeap(stack, size + 1, 8);

ReleaseInfiniteStack();

FClose(in);

}

sprintf(file, (char*)"%s/%s/months.txt", livedataFolder, lang);

in = FopenStaticReadOnly(file);

if (in)

{

stack = InfiniteStack(limit, "initmonths");

data = (int*)stack;

while (ReadALine(readBuffer, in) >= 0) // month name or abbrev

{

if (*readBuffer == '#' || *readBuffer == 0) continue;

ReadCompiledWord(readBuffer, word);

char* w = StoreWord(word, AS_IS)->word; // month name

int index = Heap2Index(w);

*data++ = index;

}

*data++ = 0; // terminal value for end detection

size = (char*)data - stack;

size /= 8; // 64bit chunks

monthnames[languageIndex] = (int*)AllocateHeap(stack, size + 1, 8);

ReleaseInfiniteStack();

FClose(in);

}

// emoji short names, see https://unicode.org/Public/emoji/13.0/emoji-sequences.txt

// or https://raw.githubusercontent.com/omnidan/node-emoji/master/lib/emoji.json

sprintf(file, (char*)"%s/emojishortnames.txt", systemFolder);

in = FopenStaticReadOnly(file);

if (in)

{

stack = InfiniteStack(limit, "initemoji");

data = (int*)stack;

while (ReadALine(readBuffer, in) >= 0) // short name

{

if (*readBuffer == '#' || *readBuffer == 0) continue;

ReadCompiledWord(readBuffer, word);

// strip any colons

char* ptr = word;

if (*ptr == ':')

{

++ptr;

size_t len = strlen(ptr) - 1;

if (ptr[len] == ':') {

ptr[len] = 0;

}

}

*data++ = Heap2Index(AllocateHeap(ptr, 0));

}

*data++ = 0; // terminal value for end detection

size = (char*)data - stack;

size /= 8; // 64bit chunks

emojishortnames = (int*)AllocateHeap(stack, size + 1, 8);

ReleaseInfiniteStack();

fclose(in);

}

}

char* FindJMSeparator(char* ptr, char c)

{

char* at = ptr - 1;

bool quote = false;

while (*++at)

{

if (*at == '"') quote = !quote;

if (!quote && *at == c) return at; // found terminator not in quotes

}

return NULL;

}

void CopyParam(char* to, char* from, unsigned int limit)

{

size_t len = strlen(from);

if (*from == '"')

{

++from;

--len;

if (from[len - 1] == '"') --len;

}

if (len > limit) len = limit - 1; // truncate silently

for (unsigned int i = 0; i < len; ++i)

{

if (from[i] == '\\') ++i;

*to++ = from[i];

}

*to = 0;

}

bool IsDate(char* original)

{

// Jan-26-2017 or 2017/MAY/26 also jan-02 or jan-2017 or 2017-jan or 02-jan BUT NOT 02-2017

// 24/15/2007 etc

size_t len = strlen(original);

if (!IsDigit(original[0]) && !IsDigit(original[len - 1])) return false;

// date must either start or end with a digit.

char separator = 0;

int digitcount = 0;

bool digit4 = false;

int separatorcount = 0;

char* alpha = NULL;

int alphacount = 0;

bool acceptalpha = true;

--original;

while (*++original)

{

if (IsDigit(*original))

{

if (acceptalpha == false) return false; // cannot mix alpha and digit

++digitcount; // count digit block size

}

else if (IsAlphaUTF8(*original) && *original != '-' && *original != '.' && *original != '/')

{

if (acceptalpha)

{

alpha = original; // 1st one seen

acceptalpha = false;

}

}

else if (!separator) separator = *original;

else if (*original != separator && separatorcount == 2 && !alpha) return true; //when there is already a date but still have different seperators.(eg: 18-02-2019.)

else if (*original != separator) return false; // cannot be

if (separator && separator != '/' && separator != '-' && separator != '.') return false; // 10:30:00-05:00 is not a date

if (*original == separator) // start a new block

{

if (digitcount == 4) // only 1 year is allowed

{

if (digit4) return false; // only 1 of these

digit4 = true;

}

else if (digitcount > 2) return false; // not legal count, must be 1,2 or 4 digits

if (acceptalpha == false && ++alphacount > 1) return false; // only one text month is allowed

acceptalpha = true;

digitcount = 0;

if (++separatorcount > 2) return false; // only 2 separators (3 components) allowed

}

}

if (!separator) return false;

if (!alpha)

{

return (separatorcount == 2);

}

if (separatorcount < 1) return false; // 2 or 3 allowed

// confirm its a month

int* name = monthnames[languageIndex];

for (int i = 0; i < 1000; ++i)

{

if (!name || !*name) return false;

char* w = Index2Heap(*name);

name++;

if (*w != toUppercaseData[*alpha]) continue;

size_t n = strlen(w);

if (strnicmp(w, alpha, n)) continue;

if (alpha[n] == 0 || alpha[n] == separator) return true;

}

return false;

}

void ClearSupplementalInput()

{

startSupplementalInput = NULL;

}

void CloseTextUtilities()

{

}

bool IsFraction(char* token)

{

if (IsDigit(token[0])) // fraction?

{

char* at = strchr(token, '/');

if (at)

{

at = token;

while (IsDigit(*++at)) { ; }

if (*at == '/' && (at[1] != '0' || at[2]))

{

while (IsDigit(*++at)) { ; }

if (!*at) return true;

}

}

}

return false;

}

bool IsAllUpper(char* ptr)

{

bool foundDigit = false;

bool foundNonDigit = false;

--ptr;

while (*++ptr)

{

if (!IsUpperCase(*ptr) && *ptr != '&' && *ptr != '-' && *ptr != '_' && !IsDigit(*ptr)) break;

if (IsDigit(*ptr)) foundDigit = true;

else foundNonDigit = true;

}

if (foundDigit && !foundNonDigit) return false; // can't be all digits

return (!*ptr);

}

void ConvertJapanText(char* output,bool pattern)

{

char* base = output--;

bool variable = false;

while (*++output)

{

int index = output - base;

unsigned char c = *output;

int kind = IsLegalNameCharacter(c); // 1==utf8 0=non letter/digit

// universally replace [] () {}

int k = -1;

JapanCharInfo* fn;

if (kind == 1) // is utf8 based

{

while ((fn = &japanControlSet[++k]) && fn->charword)

{

if (!strncmp((char*)output, fn->charword, 3))

{

memmove(output + 1, output + 3, strlen(output + 2));

*output = fn->convert;

// stuff ends variables unless its json access $x[5]

if (*output != '[' && *output != ']') variable = false;

else if (variable && *output == ']') // is it closing a variable or merely arrayref

{

char* at = output;

while (*--at != USERVAR_PREFIX)

{

if (*at == '[') break; // opening []

}

if (*at == USERVAR_PREFIX) variable = false;

}

break;

}

}

if (fn->charword) continue; // we already made a subsitution

}

// all variables need . - _ letters and digits converted from japanese to ascii

if (c == USERVAR_PREFIX) variable = true;

else if (variable && c == ']') // is it closing a variable or merely arrayref

{

char* at = output;

while (*--at != USERVAR_PREFIX)

{

if (*at == '[') break; // opening []

}

if (*at == USERVAR_PREFIX) variable = false;

}

else if (!kind ) variable = false; // not utf8 and not var legal

else if (pattern || (variable && kind == 1)) // convert to ascii only

{

k = -1;

while ((fn = &japanSet[++k]) && fn->charword)

{

if (k < 6) // ranged

{

if (pattern && !variable) continue; // dont convert ranged values except in variables

// within range?

unsigned int endval = (unsigned int)output[2];

int lower = strncmp(output, fn->charword, 3);

int upper = strncmp(output, japanSet[k + 1].charword, 3);

if (lower >= 0 && upper <= 0) // in range

{

memmove(output + 1, output + 3, strlen(output + 2) );

*output = (char)(fn->convert + (endval - (unsigned int)fn->charword[2]));

++k; // only check a range once at start

break;

}

++k; // only check a range once at start

continue;

}

else if (!strncmp((char*)output, fn->charword, 3)) // non range

{

memmove(output + 1, output + 3, strlen(output + 2));

*output = fn->convert;

break;

}

}

if (!fn->charword) variable = false; // didnt match legal vardata

}

}

}

char* RemoveEscapesWeAdded(char* at)

{

if (!at || !*at) return at;

char* startScan = at;

while ((at = strchr(at, '\\'))) // we always add ESCAPE_FLAG backslash

{

if (*(at - 1) == ESCAPE_FLAG) // alter the escape in some way

{

if (at[1] == 't') at[1] = '\t'; // legal

else if (at[1] == 'n') at[1] = '\n'; // legal

else if (at[1] == 'r') at[1] = '\r'; // legal

// other choices dont translate, eg double quote

memmove(at - 1, at + 1, strlen(at)); // remove the escape flag and the escape

}

else ++at; // we dont mark " with ESCAPE_FLAG, we are not protecting it.

}

return startScan;

}

char* CopyRemoveEscapes(char* to, char* at, int limit, bool allitems) // all includes ones we didnt add as well

{

*to = 0;

char* start = to;

if (!at || !*at) return to;

// need to remove escapes we put there

--at;

while (*++at)

{

if (*at == ESCAPE_FLAG && at[1] == '\\') // we added an escape here

{

at += 2; // move to escaped item

if (*(at - 1) == ESCAPE_FLAG) // dual escape is special marker, there is no escaped item

{

*to++ = '\r'; // legal

*to++ = '\n'; // legal

--at; // rescan the item

}

else if (*at == 't') *to++ = '\t'; // legal

else if (*at == 'n') *to++ = '\n'; // legal

else if (*at == 'r') *to++ = '\r'; // legal

else *to++ = *at; // remove our escape pair and pass the item

}

else if (*at == '\\' && allitems) // remove ALL other escapes in addition to ones we put there

{

++at; // move to escaped item

if (*at == 't') *to++ = '\t';

else if (*at == 'n') *to++ = '\n'; // legal

else if (*at == 'r') *to++ = '\r'; // legal

else { *to++ = *at; } // just pass along untranslated

}

else *to++ = *at;

if ((to - start) >= limit) // too much, kill it

{

*to = 0;

break;

}

}

*to = 0;

return start;

}

void RemoveImpure(char* buffer)

{

char* p;

while ((p = strchr(buffer, '\r'))) *p = ' '; // legal

while ((p = strchr(buffer, '\n'))) *p = ' '; // legal

while ((p = strchr(buffer, '\t'))) *p = ' '; // legal

}

char* RemoveQuotes(char* item)

{

char* q;

while ((q = strchr(item, '"'))) memmove(q, q + 1, strlen(q));

return item;

}

void ChangeSpecial(char* buffer)

{