#include "common.h"

#include "evserver.h"

int pendingUserLimit = 0;

std::atomic<int> userQueue(0);

char* version = "14.1";

char overflowMessage[400];

int overflowLen = 0;

const char* buildString = "compiled " __DATE__ ", " __TIME__ ".";

char sourceInput[200];

int cs_qsize = 0;

static int python = 0;

char repairinput[20];

unsigned int argcx;

char** argvx;

bool integrationTest = false;

char websocketclient[200];

bool nophrases = false;

char* repairat = NULL;

char baseLanguage[50];

static char* retryInput;

char* tracebuffer; // preallocated buffer for possible api tracing

int volleyFile = 0;

unsigned long startLogDelay = 0;

int forcedRandom = -1;

char* releaseBotVar = NULL;

char rootdir[MAX_WORD_SIZE];

char incomingDir[MAX_WORD_SIZE];

FILE* userInitFile;

unsigned int parseLimit = 0;

bool jabugpatch = false;

bool fastload = true;

int hadAuthCode = 0;

bool blockapitrace = false;

static bool crnl_safe = false;

int externalTagger = 0;

char defaultbot[100];

char serverlogauthcode[30];

uint64 chatstarted = 0;

char websocketparams[300];

char websocketmessage[300];

char websocketbot[300];

char jmeter[100];

int sentenceloopcount = 0;

static uint64 lastCrashTime = 0;

int autorestartdelay = 0;

int db = 0;

int gzip = 0;

bool client = false;

bool newuser = false;

bool restartfromdeath = false;

char myip[100];

uint64 preparationtime = 0;

uint64 replytime = 0;

unsigned int timeLog = 2147483647;

bool autoreload = true;

bool sentenceOverflow = false;

int sentenceLimit = SENTENCES_LIMIT;

bool loadingUser = false;

bool dieonwritefail = false;

unsigned int inputLimit = 0;

unsigned int fullInputLimit = 0;

char traceuser[500];

char* originalUserInput = NULL;

char* originalUserMessage = NULL;

bool debugcommand = false;

int traceUniversal;

int debugLevel = 0;

PRINTER printer = printf;

uint64 startNLTime;

bool retrycheat;

unsigned int idetrace = (unsigned int)-1;

int outputlevel = 0;

uint64 timedeployed = 0;

// parameters

unsigned int argc;

char** argv;

char* configFile = "cs_init.txt"; // can set config params

char* configFile2 = "cs_initmore.txt"; // can set config params

char* configLines[MAX_WORD_SIZE]; // can set config params

char livedataFolder[500]; // where is the livedata folder

char languageFolder[500]; // where is the livedata language folder

char systemFolder[500]; // where is the livedata system folder

char usersfolder[100];

char logsfolder[100];

char topicfolder[100];

static char* privateParams = NULL;

char treetaggerParams[200];

static char encryptParams[300];

static char decryptParams[300];

char buildfiles[100];

char apikey[100];

int forkcount = 1;

char newusername[100] = "";

char erasename[100] = "csuser_erase";

char buildflags[100] = "";

bool noboot = false;

static bool traceboot = false;

char tmpfolder[100];

static char* configUrl = NULL;

static char* configHeaders[20];

static int headerCount = 0;

int configLinesLength; // can set config params

bool build0Requested = false;

bool build1Requested = false;

bool servertrace = false;

static int repeatLimit = 0;

bool pendingRestart = false;

bool pendingUserReset = false;

bool rebooting = false;

int outputchoice = -1;

DEBUGAPI debugInput = NULL;

DEBUGAPI debugOutput = NULL;

DEBUGAPI debugEndTurn = NULL;

DEBUGLOOPAPI debugCall = NULL;

DEBUGVARAPI debugVar = NULL;

DEBUGVARAPI debugMark = NULL;

DEBUGAPI debugMessage = NULL;

DEBUGAPI debugAction = NULL;

#define MAX_RETRIES 20

char* revertBuffer; // copy of user input so can :revert if desired

bool postProcessing = false;

unsigned int tokenCount; // for document

bool callback = false; // when input is callback,alarm,loopback, dont want to enable :retry for that...

uint64 timerLimit = 0; // limit time per volley

int timerCheckRate = 0; // how often to check calls for time

uint64 volleyStartTime = 0;

char forkCount[10];

int timerCheckInstance = 0;

char hostname[100];

bool logline = false;

static char* tsvInputs[15];

char tsvpriorlogin[MAX_WORD_SIZE];

unsigned int tsvIndex;

unsigned int tsvMessageField;

unsigned int tsvLoginField;

unsigned int tsvSpeakerField;

unsigned int tsvCount;

unsigned int tsvSkip;

char speaker[100];

FILE* tsvFile = NULL;

bool nosuchbotrestart = false; // restart if no such bot

char* derivationSentence[MAX_SENTENCE_LENGTH];

char derivationSeparator[MAX_SENTENCE_LENGTH];

unsigned int derivationLength;

char authorizations[200]; // for allowing debug commands

char* ourMainInputBuffer = NULL; // user input buffer - ptr to primaryInputBuffer

char* ourMainOutputBuffer = NULL; // main user output buffer

char* readBuffer; // main scratch reading buffer (files, etc)

bool readingDocument = false;

bool redo = false; // retry backwards any amount

bool oobExists = false;

bool docstats = false;

unsigned int docSentenceCount = 0;

char* rawSentenceCopy; // current raw sentence

char* evsrv_arg = NULL;

unsigned short int derivationIndex[MAX_SENTENCE_LENGTH];

bool scriptOverrideAuthorization = false;

clock_t startSystem; // time chatscript started

unsigned int choiceCount = 0;

int always = 1; // just something to stop the complaint about a loop based on a constant

bool errorOnMissingElse = false;

// support for callbacks

uint64 callBackTime = 0; // one-shot pending time - forces callback with "[callback]" when output was [callback=xxx] when no user submitted yet

uint64 callBackDelay = 0; // one-shot pending time - forces callback with "[callback]" when output was [callback=xxx] when no user submitted yet

uint64 loopBackTime = 0; // ongoing callback forces callback with "[loopback]" when output was "[loopback=xxx]"

uint64 loopBackDelay = 0; // ongoing callback forces callback with "[loopback]" when output was "[loopback=xxx]"

uint64 alarmTime = 0;

uint64 alarmDelay = 0; // one-shot pending time - forces callback with "[alarm]" when output was [callback=xxx] when no user submitted yet

unsigned int outputLength = 100000; // max output before breaking.

char* extraTopicData = NULL; // supplemental topic set by :extratopic

// server data

#ifdef DISCARDSERVER

bool server = false;

#else

std::string interfaceKind = "0.0.0.0";

#ifdef WIN32

bool server = false; // default is standalone on Windows

#elif IOS

bool server = true; // default is server on LINUX

#else

bool server = true; // default is server on LINUX

#endif

#endif

unsigned int port = 1024; // server port

bool commandLineCompile = false;

PrepareMode tmpPrepareMode = NO_MODE; // controls what processing is done in preparation NO_MODE, POS_MODE, PREPARE_MODE

PrepareMode prepareMode = NO_MODE; // controls what processing is done in preparation NO_MODE, POS_MODE, PREPARE_MODE

bool noReact = false;

// :source:document data

bool documentMode = false; // read input as a document not as chat

FILE* sourceFile = NULL; // file to use for :source

bool multiuser = false;

EchoSource echoSource = NO_SOURCE_ECHO; // for :source, echo that input to nowhere, user, or log

uint64 sourceStart = 0; // beginning time of source file

unsigned int sourceTokens = 0;

unsigned int sourceLines = 0;

// status of user input

unsigned int volleyCount = 0; // which user volley is this

uint64 tokenControl = 0; // results from tokenization and prepare processing

unsigned int responseControl = ALL_RESPONSES; // std output behaviors

// general display and flow controls

bool quitting = false; // intending to exit chatbot

ResetMode buildReset = NO_RESET; // intending to reload system

bool autonumber = false; // display number of volley to user

bool showWhy = false; // show user rules generating his output

bool showTopic = false; // show resulting topic on output

bool showTopics = false; // show all relevant topics

bool showInput = false; // Log internal input additions

bool showReject = false; // Log internal repeat rejections additions

bool all = false; // generate all possible answers to input

int regression = NO_REGRESSION; // regression testing in progress

unsigned int trace = 0; // current tracing flags

char debugEntry[1000];

char bootcmd[200]; // current boot tracing flags

unsigned int timing = 0; // current timing flags

bool shortPos = false; // display pos results as you go

int inputRetryRejoinderTopic = NO_REJOINDER;

int inputRetryRejoinderRuleID = NO_REJOINDER;

int sentenceRetryRejoinderTopic = NO_REJOINDER;

int sentenceRetryRejoinderRuleID = NO_REJOINDER;

char oktest[MAX_WORD_SIZE]; // auto response test

char activeTopic[200];

char respondLevel = 0; // rejoinder level of a random choice block

static bool oobPossible = false; // 1st thing could be oob

int inputCounter = 0; // protecting ^input from cycling

int totalCounter = 0; // protecting ^input from cycling

char userPrefix[MAX_WORD_SIZE]; // label prefix for user input

char botPrefix[MAX_WORD_SIZE]; // label prefix for bot output

bool unusedRejoinder = true; // inputRejoinder has been executed, blocking further calls to ^Rejoinder

// outputs generated

RESPONSE responseData[MAX_RESPONSE_SENTENCES + 1];

unsigned char responseOrder[MAX_RESPONSE_SENTENCES + 1];

int responseIndex;

int inputSentenceCount; // which sentence of volley is this

static void HandleReBoot(WORDP boot, bool reboot);

///////////////////////////////////////////////

/// SYSTEM STARTUP AND SHUTDOWN

///////////////////////////////////////////////

void HandlePermanentBuffers(bool init)

{

if (init)

{

if (!fullInputLimit) fullInputLimit = maxBufferSize * 2;

readBuffer = mymalloc(fullInputLimit); // need to be full, to read a log file potentially

if (!readBuffer) myexit("FATAL: buffer allocation fail");

*readBuffer = 0;

currentRuleOutputBase = currentOutputBase = readBuffer; // a default needed for handle boot

tracebuffer = mymalloc(TESTPATTERN_TRACE_SIZE);

lastInputSubstitution = mymalloc(maxBufferSize);

rawSentenceCopy = mymalloc(maxBufferSize);

revertBuffer = mymalloc(maxBufferSize);

#ifndef DLL

ourMainInputBuffer = mymalloc(fullInputLimit); // precaution for overflow

ourMainOutputBuffer = mymalloc(outputsize);

#else

ourMainOutputBuffer = revertBuffer; // for boot call to function on startup potentially

#endif

currentOutputLimit = outputsize;

// need allocatable buffers for things that run ahead like servers and such.

maxBufferSize = (maxBufferSize + 63);

maxBufferSize &= 0xffffffc0; // force 64 bit align on size

unsigned int total = maxBufferLimit * maxBufferSize;

buffers = mymalloc(total); // have it around already for messages

if (!buffers)

{

(*printer)((char*)"%s", (char*)"cannot allocate buffer space");

myexit((char*)"FATAL: cannot allocate buffer space");

}

bufferIndex = 0;

baseBufferIndex = bufferIndex;

malloc_jp_token_buffers();

malloc_mssql_buffer();

}

else

{

CloseBuffers();

}

}

void InitStandalone()

{

startSystem = CPU_TICKS;

*currentFilename = 0; // no files are open (if logging bugs)

tokenControl = 0;

outputlevel = 0;

*computerID = 0; // default bot

}

static void RelocateDeadBootFacts(FACT* F)

{

// move dead/transient facts to end

FACT* tailArea = lastFactUsed + 1; // last fact

FACT* G = NULL;

while (++F <= lastFactUsed)

{

if (!(F->flags & (FACTTRANSIENT | FACTDEAD)))

{

F->flags |= FACTBUILD1;

continue;

}

else if (!F->subject) continue; // already moved this one

UnweaveFact(F);

bool dead = true;

if (!G) G = F;

while (++G < tailArea)

{

if ((G->flags & (FACTTRANSIENT | FACTDEAD))) continue;

if (!G->subject) break; // already dead

UnweaveFact(G); // cut from from dictionary referencing

memcpy(F, G, sizeof(FACT)); // replicate fact lower

WeaveFact(F); // rebind fact to dictionary

F->flags |= FACTBUILD1;

G->flags |= FACTDEAD;

dead = false;

break;

}

if (dead)

{

// kill this completely in case we cant overwrite it with valid fact

F->subject = F->verb = F->object = 0;

F->flags |= FACTDEAD;

}

}

while (!lastFactUsed->subject) --lastFactUsed; // reclaim

}

static void HandleReBoot(WORDP boot, bool volleyboot)

{

if (!boot || noboot)

{// noboot is command line paramenter to block use of boot

rebooting = false;

return;

}

// boot== ^csboot on startup (reboot==false) vs ^cs_reboot per volley (reboot==true)

if (volleyboot) rebooting = true; // global awareness we are rebooting vs system startup

else // system startup boot

{

}

// run script on startup of system. data it generates will also be layer 1 data

int oldtrace = trace;

int oldtiming = timing;

if (traceboot) trace = (unsigned int)-1;

else

{

FILE* in = FopenReadOnly("traceboot.txt"); // per external file created, enable server log on boot

if (in)

{

trace = (unsigned int)-1;

FClose(in);

}

}

int olduserlog = userLog;

if (trace) userLog = FILE_LOG | PRE_LOG;

if (*bootcmd) DoCommand(bootcmd, NULL, false); // run this command before CSBOOT is run. eg to trace boot process

if (!volleyboot) UnlockLayer(LAYER_BOOT); // unlock it to add stuff

else currentBeforeLayer = LAYER_BOOT; // be in user layer (before is boot layer)

FACT* F = lastFactUsed; // note facts allocated before running boot

Callback(boot, (char*)"()", true, true); // do before world is locked

if (ourMainOutputBuffer) *ourMainOutputBuffer = 0; // remove any boot message

myBot = 0; // restore fact owner to generic all

// cs_reboot may call boot to reload layer, this flag

// rebooting might be wrong...

if (!volleyboot)

{

RelocateDeadBootFacts(F);

CloseBotVariables();

}

else

{

ReturnToAfterLayer(LAYER_BOOT, false); // dict/fact/strings reverted and any extra topic loaded info (but CSBoot process NOT lost)

}

userLog = olduserlog;

trace = (modifiedTrace) ? modifiedTraceVal : oldtrace;

timing = (modifiedTiming) ? modifiedTimingVal : oldtiming;

stackFree = stackStart; // drop any possible stack used

rebooting = false;

}

unsigned int CountWordsInBuckets(unsigned int& unused, unsigned int* depthcount, int limit)

{

memset(depthcount, 0, sizeof(int) * limit);

unused = 0;

int words = 0;

for (unsigned long i = 0; i <= maxHashBuckets; ++i)

{

int n = 0;

if (!hashbuckets[i]) ++unused;

else

{

WORDP D = Index2Word(hashbuckets[i]);

while (D)

{

++n;

D = Index2Word(GETNEXTNODE(D));

}

}

words += n; // n is number of words in this bucket

if (n && n < limit)

depthcount[n] += n; // this many words have a bucket access of n

}

return words;

}

static void ShowMemoryUsage()

{

char data[MAX_WORD_SIZE];

int dictcount = (unsigned int)(dictionaryFree - dictionaryBase);

unsigned long dictm = dictcount * sizeof(WORDENTRY);

unsigned long dictfree = (unsigned int)(maxDictEntries - dictcount);

unsigned long dictFreeMemMB = (unsigned int)((dictfree * sizeof(WORDENTRY)) / 100000);

unsigned int factcount = lastFactUsed - factBase;

unsigned int factUsedMemMB = (unsigned int)((factcount * sizeof(FACT)) / 1000000);

unsigned int factFreeMemMB = (unsigned int)(((factEnd - lastFactUsed) * sizeof(FACT)) / 1000000);

unsigned int dictUsedMemMB = (unsigned int)(((dictionaryFree - dictionaryBase) * sizeof(WORDENTRY)) / 1000000);

// dictfree shares text space

unsigned int textUsedMemMB = (unsigned int)((heapBase - heapFree) / 1000000);

unsigned int textFreeMemMB = (unsigned int)((heapFree - heapEnd) / 1000000);

unsigned int bufferMemMB = (maxBufferLimit * maxBufferSize) / 1000000;

unsigned int cacheMB = (userTopicStoreSize + userTableSize) / 1000000;

unsigned int usedMB = factUsedMemMB + dictUsedMemMB + textUsedMemMB + bufferMemMB;

usedMB += (userTopicStoreSize + userTableSize) / 1000000;

usedMB += bufferMemMB + cacheMB;

unsigned int freeMB = factFreeMemMB + textFreeMemMB + dictFreeMemMB;

unsigned int bytes = (tagRuleCount * MAX_TAG_FIELDS * sizeof(uint64)) / 1000;

usedMB += bytes / 1000000;

char buf2[MAX_WORD_SIZE];

char buf1[MAX_WORD_SIZE];

char buf[MAX_WORD_SIZE];

strcpy(buf, StdIntOutput(factcount));

strcpy(buf2, StdIntOutput(textFreeMemMB));

unsigned int depthcount[MAX_HASH_DEPTH];

unsigned int unused;

unsigned int words = CountWordsInBuckets(unused, depthcount, MAX_HASH_DEPTH);

char depthmsg[8000];

int x = MAX_HASH_DEPTH;

const char* full = (depthcount[MAX_HASH_DEPTH - 1]) ? "+" : "";

while (!depthcount[--x]); // find last used slot

char* at = depthmsg;

*at = 0;

for (int i = 1; i <= x; ++i)

{

sprintf(at, "%u.", depthcount[i]);

at += strlen(at);

}

sprintf(at, "-%d", x);

sprintf(data, (char*)"Used %uMB: dictentries %s (%uMb) fact %s (%uMb) heap %uMb buffer %uMb usercache %uMb\r\n",

usedMB,

StdIntOutput(dictcount),

dictUsedMemMB,

buf,

factUsedMemMB,

textUsedMemMB, bufferMemMB,

cacheMB);

if (server) Log(SERVERLOG, "%s", data);

else (*printer)((char*)"%s", data);

sprintf(data, (char*)"hashbuckets: %lu (unused %u) depths %s\r\n", maxHashBuckets, unused, depthmsg);

if (server) Log(SERVERLOG, "%s", data);

else (*printer)((char*)"%s", data);

char factmb[500];

unsigned long factm = factcount * sizeof(FACT);

strcpy(factmb, StdIntOutput((int)(factm / 1000000))); // unused facts

strcpy(buf, StdIntOutput(factcount)); // unused facts

strcpy(buf1, StdIntOutput(textFreeMemMB)); // unused text

strcpy(buf2, StdIntOutput(freeMB));

sprintf(data, (char*)"Free %sMB: dict %s(%zuMb) fact %s(%sMb) stack/heap %sMB \r\n\r\n",

buf2, StdIntOutput(dictfree), dictFreeMemMB, buf, factmb, buf1);

if (server) Log(SERVERLOG, "%s", data);

else (*printer)((char*)"%s", data);

}

static void PerformBoot()

{

buildbootjid = 0;

builduserjid = 0;

UnlockLayer(LAYER_BOOT); // unlock it to add stuff

HandleReBoot(FindWord((char*)"^csboot"), false);// run script on startup of system. data it generates will also be layer 1 data

LockLayer();

currentBeforeLayer = LAYER_BOOT;

}

void CreateSystem()

{

timedeployed = ElapsedMilliseconds();

tableinput = NULL;

scriptOverrideAuthorization = false; // always start safe

loading = true;

char* os;

mystart("createsystem");

#ifdef WIN32

os = "Windows";

#elif IOS

os = "IOS";

#elif __MACH__

os = "MACH";

#elif FREEBSD

os = "FreeBSD";

#else

os = "LINUX";

#endif

if (!buffers) // restart asking for new memory allocations

{

maxBufferSize = (maxBufferSize + 63);

maxBufferSize &= 0xffffffC0; // force 64 bit align

unsigned int total = maxBufferLimit * maxBufferSize;

buffers = mymalloc(total); // have it around already for messages

if (!buffers)

{

(*printer)((char*)"%s", (char*)"cannot allocate buffer space");

myexit((char*)"FATAL: cannot allocate buffer space");

}

bufferIndex = 0;

baseBufferIndex = bufferIndex;

}

char data[MAX_WORD_SIZE];

char* kind;

int pid = 0;

#ifdef EVSERVER

kind = "EVSERVER";

pid = getpid();

#elif DEBUG

kind = "Debug";

#else

kind = "Release";

#endif

sprintf(data, (char*)"ChatScript %s %s pid:%d os:%ld-bit %s compiled: %s", kind, version, pid, (long int)(sizeof(char*) * 8), os, compileDate);

strcat(data, (char*)" rootdir:");

strcat(data, rootdir);

strcat(data, (char*)" host:");

strcat(data, hostname);

strcat(data, (char*)"\r\n");

Log(SERVERLOG, "%s%s", (server) ? "Server " : "", data);

int oldtrace = trace; // in case trace turned on by default

trace = 0;

*oktest = 0;

sprintf(data, (char*)"Params: dict:%lu fact:%lu text:%lukb hash:%lu \r\n", (unsigned long)maxDictEntries, (unsigned long)maxFacts, (unsigned long)(maxHeapBytes / 1000), (unsigned long)maxHashBuckets);

if (server) Log(SERVERLOG, "%s", data);

else (*printer)((char*)"%s", data);

sprintf(data, (char*)" buffer:%ux%ukb ucache:%ux%ukb fcache:%ux%ukb userfacts:%u outputlimit:%u loglimit:%u\r\n", (unsigned int)maxBufferLimit, (unsigned int)(maxBufferSize / 1000), (unsigned int)userCacheCount, (unsigned int)(userCacheSize / 1000), (unsigned int)fileCacheCount, (unsigned int)(fileCacheSize / 1000), (unsigned int)userFactCount,

outputsize, logsize);

if (server) Log(SERVERLOG, "%s", data);

else (*printer)((char*)"%s", data);

sprintf(data, (char*)"Libraries: ");

#ifndef DISCARDJSONOPEN

char data1[100];

sprintf(data1, (char*)"curl:%s, ", CurlVersion());

strcat(data, data1);

#endif

#ifndef DISCARD_JAPANESE

strcat(data, "Mecab-Japanese");

#endif

strcat(data, "\r\n");

if (server) Log(SERVERLOG, "%s", data);

else (*printer)((char*)"%s", data);

// in case

MakeDirectory(topicfolder);

char name[MAX_WORD_SIZE];

sprintf(name, "%s/BUILD0", topicfolder);

MakeDirectory(name);

sprintf(name, "%s/BUILD1", topicfolder);

MakeDirectory(name);

originalUserInput = NULL;

currentInput = NULL;

LoadSystem(0,argc,argv);

// leaves current layer as BOOT (2)

strcpy(dbparams, mssqlparams); // do regardless so can use dummy codes

#ifndef DISCARDMONGO

strcpy(dbparams, mongodbparams);

#endif

#ifndef DISCARDPOSTGRES

strcpy(dbparams, postgresparams);

#endif

#ifndef DISCARDMYSQL

strcpy(dbparams, mysqlparams);

#endif

trace = oldtrace; // allow boot tracing

PerformBoot();

InitSpellCheck(); // after boot vocabulary added

ShowMemoryUsage();

#ifdef DISCARDSERVER

(*printer)((char*)" Server disabled.\r\n");

#endif

#ifdef DISCARDSCRIPTCOMPILER

if (server) Log(SERVERLOG, " Script compiler disabled.\r\n");

else (*printer)((char*)" Script compiler disabled.\r\n");

#endif

#ifdef DISCARDTESTING

if (server) Log(SERVERLOG, " Testing disabled.\r\n");

else (*printer)((char*)" Testing disabled.\r\n");

#else

* callerIP = 0;

if (server && VerifyAuthorization(FopenReadOnly((char*)"authorizedIP.txt"))) // authorizedIP

{

Log(SERVERLOG, " *** Server WIDE OPEN to :command use.\r\n");

}

#endif

char route[100];

*route = 0;

#ifdef DISCARDJSONOPEN

sprintf(route, (char*)"%s", (char*)" JSONOpen access disabled.\r\n");

if (server) Log(SERVERLOG, route);

else (*printer)(route);

#endif

#ifdef TREETAGGER

if (externalPostagger == TreeTagger)

{

sprintf(route, (char*)" TreeTagger access enabled\r\n");

if (server) Log(SERVERLOG, route);

else (*printer)(route);

}

#endif

#ifndef DISCARDMICROSOFTSQL

if (gzip)

{

sprintf(route, (char*)" Text Compression enabled\r\n");

if (server) Log(SERVERLOG, route);

else (*printer)(route);

}

#endif

(*printer)((char*)"%s", (char*)"\r\n");

loading = false;

printf("System created in %u ms\r\n", (unsigned int)(ElapsedMilliseconds() - timedeployed));

}

void LoadSystem(unsigned int limit,unsigned int argc, char** argv)

{// reset the basic system

myBot = 0; // facts loaded here are universal

InitFacts(); // malloc space

InitStackHeap(); // malloc space

InitDictionary(); // malloc space

LoadDictionary(heapFree);

InitUserCache(); // malloc space

InitFunctionSystem(); // modify dictionary entries

InitScriptSystem();

InitVariableSystem();

InitSystemVariables();

InitLogs();

SetLanguage("universal");

InitUniversalTextUtilities(); // also part of basic system before a build

WalkLanguages(InitTextUtilitiesByLanguage); // also part of basic system before a build

WalkLanguages(ReadLiveData); // below layer 0

if (multidict) SetLanguage("English"); // return to std base

printf("Languages: %s\r\n", language_list);

#ifdef TREETAGGER

// We will be reserving some working space for treetagger on the heap

// so make sure it is tucked away where it cannot be touched

// NOT IN DEBUGGER WILL FORCE reloads of layers

InitTreeTagger(treetaggerParams);

#endif

#ifndef DISCARD_JAPANESE

printf("Loaded Mecab kanji tagger\r\n");

#endif

WordnetLockDictionary();

char* lang = language_list;

lang = GetNextLanguage(lang);

SetLanguage(lang);

*currentFilename = 0;

*debugEntry = 0;

InitTopicSystem(limit); // dictionary reverts to wordnet zone

InitBotVariables(argc, argv);

#ifndef DISCARD_PYTHON

if (python) Py_Initialize();

#endif

}

void Rebegin(unsigned int buildId,unsigned int argc, char** argv)

{

CloseLogs();

FreeAllUserCaches(); // user system

CloseDictionary(); // dictionary system but not its memory

CloseFacts(); // fact system

LoadSystem(buildId,argc,argv);

PerformBoot();

InitSpellCheck(); // after boot vocabulary added

}

static void ProcessArgument(char* arg)

{

#ifndef NOARGTRACE

int old = serverLog;

serverLog = FILE_LOG;

Log(SERVERLOG, " %s\r\n", arg);

serverLog = old;

#endif

if (!strnicmp(arg, (char*)"trace", 5) && stricmp(arg,(char*)"traceboot"))

{

if (arg[5] == '=') trace = atoi(arg + 6);

else if (arg[5] == 'b') {}

else trace = (unsigned int)-1;

}

else if (!strnicmp(arg, (char*)"language=", 9))

{

strcpy(current_language, arg + 9);

MakeUpperCase(current_language);

strcpy(language_list, current_language);

max_language = LANGUAGE_1; // for now

max_fact_language = FACTLANGUAGE_1; // for now

languageCount = 1;

char* x = strchr(current_language, ','); // is a multi list

if (x)

{

multidict = true;

*x = 0; // default 1st one

// how many languages are we loading

x = language_list - 1;

while ((x = strchr(++x, ','))) ++languageCount;

}

strcpy(baseLanguage, current_language);

}

else if (!strnicmp(arg, (char*)"buildflags=", 11)) CopyParam(buildflags, arg + 11, 100);

else if (!strnicmp(arg, (char*)"buffer=", 7)) // number of large buffers available 8x80000

{

maxBufferLimit = atoi(arg + 7);

char* size = strchr(arg + 7, 'x');

if (size) maxBufferSize = atoi(size + 1) * 1000;

if (maxBufferSize < OUTPUT_BUFFER_SIZE)

{

(*printer)((char*)"Buffer cannot be less than OUTPUT_BUFFER_SIZE of %d\r\n", OUTPUT_BUFFER_SIZE);

myexit((char*)"FATAL buffer size less than output buffer size");

}

}

else if (!strnicmp(arg, (char*)"timer=", 6))

{

char* x = strchr(arg + 6, 'x'); // 15000x10

if (x)

{

*x = 0;

timerCheckRate = atoi(x + 1);

}

timerLimit = atoi(arg + 6);

}

else if (!stricmp(arg, (char*)"trustpos")) trustpos = true;

else if (!strnicmp(arg, (char*)"inputlimit=", 11)) inputLimit = atoi(arg + 11);

else if (!strnicmp(arg, (char*)"fullInputlimit=", 15)) fullInputLimit = atoi(arg + 15);

else if (!strnicmp(arg, (char*)"debug=", 6)) CopyParam(debugEntry, arg + 6, 1000);

else if (!strnicmp(arg, "erasename=", 10)) CopyParam(erasename, arg + 10, 100);

else if (!strnicmp(arg, "cs_new_user=", 12)) CopyParam(newusername, arg + 12, 100);

else if (!stricmp(arg, "userencrypt")) userEncrypt = true;

else if (!stricmp(arg, "ltmencrypt")) ltmEncrypt = true;

else if (!strnicmp(arg, "defaultbot=", 11)) CopyParam(defaultbot, arg + 11, 100);

else if (!strnicmp(arg, "traceuser=", 10)) CopyParam(traceuser, arg + 10);

else if (!stricmp(arg, (char*)"noboot")) noboot = true;

else if (!stricmp(arg, (char*)"noretrybackup")) noretrybackup = true;

else if (!stricmp(arg, (char*)"traceboot")) traceboot = true;

else if (!stricmp(arg, (char*)"trace")) trace = (unsigned int)-1; // make trace work on login

else if (!stricmp(arg, (char*)"time")) timing = (unsigned int)-1 ^ TIME_ALWAYS; // make timing work on login

else if (!stricmp(arg, (char*)"recordboot")) recordBoot = true;

else if (!stricmp(arg, (char*)"servertrace")) servertrace = true;

else if (!strnicmp(arg, (char*)"repeatlimit=", 12)) repeatLimit = atoi(arg + 12);

else if (!strnicmp(arg, (char*)"apikey=", 7)) CopyParam(apikey, arg + 7);

else if (!strnicmp(arg, (char*)"logsize=", 8)) logsize = atoi(arg + 8); // bytes avail for log buffer

else if (!strnicmp(arg, (char*)"loglimit=", 9)) loglimit = atoi(arg + 9); // max mb of log file before change files

else if (!strnicmp(arg, (char*)"outputsize=", 11)) outputsize = atoi(arg + 11); // bytes avail for output buffer

else if (!stricmp(arg, (char*)"time")) timing = (unsigned int)-1 ^ TIME_ALWAYS;

else if (!strnicmp(arg, (char*)"bootcmd=", 8)) CopyParam(bootcmd, arg + 8);

#ifdef WIN32

else if (!strnicmp(arg, (char*)"windowsbuglog=", 14)) CopyParam(externalBugLog, arg + 14);

#else

else if (!strnicmp(arg, (char*)"linuxbuglog=", 12)) CopyParam(externalBugLog, arg + 12);

#endif

else if (!strnicmp(arg, (char*)"dir=", 4))

{

if (!SetDirectory(arg + 4)) (*printer)((char*)"unable to change to %s\r\n", arg + 4);

}

else if (!strnicmp(arg, (char*)"source=", 7)) CopyParam(sourceInput, arg + 7);

else if (!strnicmp(arg, (char*)"login=", 6)) CopyParam(loginID, arg + 6);

else if (!strnicmp(arg, (char*)"output=", 7)) outputLength = atoi(arg + 7);

else if (!strnicmp(arg, (char*)"save=", 5))

{

volleyLimit = atoi(arg + 5);

if (volleyLimit > 255) volleyLimit = 255; // cant store higher

}

// memory sizings

else if (!strnicmp(arg, (char*)"hash=", 5))

{

maxHashBuckets = atoi(arg + 5); // size of hash

setMaxHashBuckets = true;

}

else if (!strnicmp(arg, (char*)"dict=", 5)) maxDictEntries = atoi(arg + 5); // how many dict words allowed

else if (!strnicmp(arg, (char*)"fact=", 5)) maxFacts = atoi(arg + 5); // fact entries

else if (!strnicmp(arg, (char*)"text=", 5)) maxHeapBytes = atoi(arg + 5) * 1000; // stack and heap bytes in pages

else if (!strnicmp(arg, (char*)"debuglevel=", 11)) debugLevel = atoi(arg + 11); // log level entries

else if (!strnicmp(arg, (char*)"cache=", 6)) // value of 10x0 means never save user data

{

userCacheSize = atoi(arg + 6) * 1000;

char* number = strchr(arg + 6, 'x');

if (number) userCacheCount = atoi(number + 1);

}

else if (!strnicmp(arg, "nophrases", 9)) nophrases = true;

else if (!strnicmp(arg, "random=", 7)) forcedRandom = atoi(arg + 7);

else if (!strnicmp(arg, "legacymatch=", 12)) legacyMatch = atoi(arg+12);

else if (!strnicmp(arg, (char*)"filecache=", 10)) // read only cache of files

{

fileCacheSize = atoi(arg + 10) * 1000;

char* number = strchr(arg + 10, 'x');

if (number) fileCacheCount = atoi(number + 1);

}

else if (!strnicmp(arg, (char*)"nofastload", 10)) fastload = false; // how many user facts allowed

else if (!strnicmp(arg, (char*)"userfacts=", 10)) userFactCount = atoi(arg + 10); // how many user facts allowed

else if (!stricmp(arg, (char*)"redo")) redo = true; // enable redo

else if (!strnicmp(arg, (char*)"authorize=", 10)) CopyParam(authorizations, arg + 10); // whitelist debug commands

else if (!stricmp(arg, (char*)"nodebug")) *authorizations = '1';

else if (!strnicmp(arg, (char*)"users=", 6)) CopyParam(usersfolder, arg + 6);

else if (!strnicmp(arg, (char*)"logs=", 5)) CopyParam(logsfolder, arg + 5);

else if (!strnicmp(arg, (char*)"topic=", 6)) CopyParam(topicfolder, arg + 6);

else if (!strnicmp(arg, (char*)"tmp=", 4)) CopyParam(tmpfolder, arg + 4);

else if (!strnicmp(arg, (char*)"buildfiles=", 11)) CopyParam(buildfiles, arg + 11);

else if (!strnicmp(arg, (char*)"private=", 8)) privateParams = arg + 8;

else if (!strnicmp(arg, (char*)"treetagger", 10))

{

if (arg[10] == '=') MakeUpperCopy(treetaggerParams, arg + 11);

else CopyParam(treetaggerParams, "1");

}

else if (!strnicmp(arg, (char*)"encrypt=", 8)) CopyParam(encryptParams, arg + 8);

else if (!strnicmp(arg, (char*)"decrypt=", 8)) CopyParam(decryptParams, arg + 8);

else if (!stricmp(arg, (char*)"noautoreload")) autoreload = false;

else if (!strnicmp(arg, (char*)"livedata=", 9))

{

CopyParam(livedataFolder, arg + 9);

sprintf(systemFolder, (char*)"%s/SYSTEM", arg + 9);

sprintf(languageFolder, (char*)"%s/%s", arg + 9, current_language);

}

else if (!strnicmp(arg, (char*)"nosuchbotrestart=", 17))

{

if (!stricmp(arg + 17, "true")) nosuchbotrestart = true;

else nosuchbotrestart = false;

}

else if (!strnicmp(arg, (char*)"parseLimit=", 11)) parseLimit = atoi(arg + 11);

else if (!strnicmp(arg, (char*)"system=", 7)) CopyParam(systemFolder, arg + 7);

else if (!strnicmp(arg, (char*)"english=", 8)) CopyParam(languageFolder, arg + 8);

else if (!strnicmp(arg, (char*)"db=", 3)) db = atoi(arg + 3);

else if (!strnicmp(arg, (char*)"gzip=", 5)) gzip = atoi(arg + 5);

else if (!strnicmp(arg, (char*)"python=", 7)) python = atoi(arg + 7);

else if (!strnicmp(arg, (char*)"syslogstr=", 10)) CopyParam(syslogstr, arg + 10);

#ifndef DISCARDMYSQL

else if (!strnicmp(arg, (char*)"mysql=", 6)) CopyParam(mysqlparams, arg + 6, 300);

#endif

else if (!strnicmp(arg, (char*)"mssql=", 6)) CopyParam(mssqlparams, arg + 6, 300); // always here

#ifndef DISCARDPOSTGRES

else if (!strnicmp(arg, (char*)"pguser=", 7)) CopyParam(postgresparams, arg + 7, 300);

// Postgres Override SQL

else if (!strnicmp(arg, (char*)"pguserread=", 11))

{

CopyParam(postgresuserread, arg + 11);

pguserread = postgresuserread;

}

else if (!strnicmp(arg, (char*)"pguserinsert=", 13))

{

CopyParam(postgresuserinsert, arg + 13);

pguserinsert = postgresuserinsert;

}

else if (!strnicmp(arg, (char*)"pguserupdate=", 13))

{

CopyParam(postgresuserupdate, arg + 13);

pguserupdate = postgresuserupdate;

}

#endif

#ifndef DISCARDWEBSOCKET

else if (!strnicmp(arg, (char*)"websocket=", 10))

{

CopyParam(websocketparams, arg + 10, 300);

server = true;

}

else if (!strnicmp(arg, (char*)"websocketbot=", 13)) strcpy(websocketbot, arg + 13);

else if (!strnicmp(arg, (char*)"websocketmessage=", 17)) CopyParam(websocketmessage, arg + 17, 300);

#endif

#ifndef DISCARDMONGO

else if (!strnicmp(arg, (char*)"mongo=", 6))

{

CopyParam(mongodbparams, arg + 6, MONGO_DBPARAM_SIZE);

}

#endif

#ifndef DISCARDCLIENT

else if (!strnicmp(arg, (char*)"client=", 7)) // client=1.2.3.4:1024 or client=localhost:1024

{

server = false;

client = true;

char buffer[MAX_WORD_SIZE];

CopyParam(serverIP, arg + 7);

char* portVal = strchr(serverIP, ':');

if (portVal)

{

*portVal = 0;

port = atoi(portVal + 1);

}

if (!*loginID)

{

(*printer)((char*)"%s", (char*)"\r\nEnter client user name: ");

ReadALine(buffer, stdin);

(*printer)((char*)"%s", (char*)"\r\n");

Client(buffer);

}

else Client(loginID);

exit(0);

}

#endif

if (!strnicmp(arg, "userLimit=", 10))

pendingUserLimit = atoi(arg + 10);

if (!strnicmp(arg, "overflowMessage=",16))

{

strcpy(overflowMessage, arg + 17); // skip quote

char* end = strchr(overflowMessage, '"');

if (end) *end = 0;

overflowLen = strlen(overflowMessage);

}

else if (!stricmp(arg, (char*)"nouserlog")) userLog = 0;

else if (!strnicmp(arg, "userlogging=", 12))

{

userLog = NO_LOG;

if (strstr(arg, "none")) userLog = NO_LOG;

if (strstr(arg, "file")) userLog |= FILE_LOG;

if (strstr(arg, "stdout")) userLog |= STDOUT_LOG;

if (strstr(arg, "stderr")) userLog |= STDERR_LOG;

if (strstr(arg, "prelog")) userLog |= PRE_LOG;

}

else if (!strnicmp(arg, "userlog", 7))

{

userLog = FILE_LOG;

if (arg[7] == '=')

{

if (IsDigit(arg[8])) userLog = atoi(arg + 8);

else userLog = NO_LOG;

}

}

else if (!stricmp(arg, "nopatterndata")) nopatterndata = true;

else if (!stricmp(arg, "jabugpatch")) jabugpatch = true;

else if (!strnicmp(arg, "autorestartdelay=", 17)) autorestartdelay = atoi(arg + 17);

else if (!strnicmp(arg, "jmeter=", 7)) CopyParam(jmeter, arg + 7);

else if (!stricmp(arg, (char*)"dieonwritefail")) dieonwritefail = true;

else if (strstr(arg, (char*)"crnl_safe")) crnl_safe = true;

else if (!strnicmp(arg, (char*)"blockapitrace=", 14)) blockapitrace = atoi(arg + 14);

else if (!strnicmp(arg, (char*)"hidefromlog=", 12)) CopyParam(hide, arg + 12);

else if (!strnicmp(arg, "serverlogauthcode=", 18)) CopyParam(serverlogauthcode, arg + 18);

else if (!stricmp(arg, "noserverlog")) serverLog = NO_LOG;

else if (!stricmp(arg, "pseudoserver")) pseudoServer = true;

else if (!strnicmp(arg, "deployloggingdelay=",19)) startLogDelay = atoi(arg+19);

else if (!strnicmp(arg, "serverlogging=", 14))

{

serverLog = NO_LOG;

if (strstr(arg, "none")) serverLog = NO_LOG;