#include "common.h"

#ifdef SAFETIME // some time routines are not thread safe (not relevant with EVSERVER)

#include <mutex>

static std::mutex mtx;

#endif

bool authorize = false;

bool pseudoServer = false;

int loglimit = 0;

bool prelog = false;

FILE* userlogFile = NULL;

char* indents[100];

int ide = 0;

int inputSize = 0;

bool inputLimitHit = false;

bool convertTabs = true;

bool serverLogTemporary = false;

bool idestop = false;

bool idekey = false;

bool inputAvailable = false;

static char encryptUser[200];

static char encryptLTM[200];

char logLastCharacter = 0;

#define MAX_STRING_SPACE 100000000 // transient+heap space 100MB

size_t maxHeapBytes = MAX_STRING_SPACE;

char* heapBase = NULL; // start of heap space (runs backward)

char* heapFree = NULL; // current free string ptr

char* stackFree = NULL;

static const char* infiniteCaller = "";

char* stackStart = NULL;

char* heapEnd = NULL;

uint64 discard;

bool infiniteStack = false;

bool infiniteHeap = false;

bool userEncrypt = false;

bool ltmEncrypt = false;

unsigned long minHeapAvailable;

bool showDepth = false;

char serverLogfileName[1000]; // file to log server to

char dbTimeLogfileName[1000]; // file to log db time to

char externalBugLog[1000];

char logFilename[MAX_WORD_SIZE]; // file to user log to

bool logUpdated = false; // has logging happened

int holdUserLog;

int holdServerLog;

int userLog = FILE_LOG; // where do we log user

int serverLog = FILE_LOG; // where do we log server

int bugLog = FILE_LOG; // where do we log bugs

char hide[4000]; // dont log these json fields

unsigned int logsize = MAX_BUFFER_SIZE; // default

static char* logmainbuffer = NULL; // where we build a log line

bool serverPreLog = true; // show what server got BEFORE it works on it

unsigned int outputsize = MAX_BUFFER_SIZE; // default

bool serverctrlz = false; // close communication with \0 and ctrlz

bool echo = false; // show log output onto console as well

bool oob = false; // show oob data

bool detailpattern = false;

bool silent = false; // dont display outputs of chat

bool logged = false;

bool showmem = false;

int filesystemOverride = NORMALFILES;

bool inLog = false;

char* testOutput = NULL; // testing commands output reroute

static char encryptServer[1000];

static char decryptServer[1000];

int adjustIndent = 0;

char* lastheapfree = NULL;

// buffer information

#define MAX_BUFFER_COUNT 80

unsigned int maxReleaseStack = 0;

unsigned int maxReleaseStackGap = 0xffffffff;

unsigned int maxBufferLimit = MAX_BUFFER_COUNT; // default number of system buffers for AllocateBuffer

unsigned int maxBufferSize = MAX_BUFFER_SIZE; // default how big std system buffers from AllocateBuffer should be

unsigned int maxBufferUsed = 0; // worst case buffer use - displayed with :variables

unsigned int bufferIndex = 0; // current allocated index into buffers[]

unsigned baseBufferIndex = 0; // preallocated buffers at start

char* buffers = 0; // collection of output buffers

#define MAX_OVERFLOW_BUFFERS 20

static char* overflowBuffers[MAX_OVERFLOW_BUFFERS]; // malloced extra buffers if base allotment is gone

CALLFRAME* releaseStackDepth[MAX_GLOBAL]; // ReleaseStack at start of depth

static unsigned int overflowLimit = 0;

unsigned int overflowIndex = 0;

USERFILESYSTEM userFileSystem;

static char staticPath[MAX_WORD_SIZE]; // files that never change

static char readPath[MAX_WORD_SIZE]; // readonly files that might be overwritten from outside

static char writePath[MAX_WORD_SIZE]; // files written by app

unsigned int currentFileLine = 0; // line number in file being read

unsigned int currentLineColumn = 0; // column number in file being read

unsigned int maxFileLine = 0; // line number in file being read

unsigned int peekLine = 0;

char currentFilename[MAX_WORD_SIZE]; // name of file being read

std::map <WORDP, uint64> timeSummary; // per volley time data about functions etc

// error recover

jmp_buf scriptJump[20];

jmp_buf crashJump;

int jumpIndex = -1;

unsigned int randIndex = 0;

unsigned int oldRandIndex = 0;

char syslogstr[300] = "chatscript"; // header for syslog messages

#ifdef WIN32

#include <conio.h>

#include <direct.h>

#include <io.h>

#include <sys/types.h>

#include <sys/stat.h>

#include <Winbase.h>

#endif

#ifdef LINUX

#include <syslog.h>

#endif

void Bug()

{

if (compiling == FULL_COMPILE)

{

jumpIndex = 0; // top level of scripting abort

BADSCRIPT("INFO: Execution error (see LOGS/bugs.txt) - try again.");

}

}

void TrackTime(char* name, int elapsed)

{

WORDP D = StoreWord(name);

if (D) TrackTime(D, elapsed);

}

void TrackTime(WORDP D, int elapsed)

{

if (!D) return;

unsigned int count = 0;

unsigned int time = 0;

std::map<WORDP, uint64>::iterator it;

it = timeSummary.find(D);

if (it != timeSummary.end())

{

count = it->second >> TIMESUMMARY_COUNT_OFFSET;

time = it->second & TIMESUMMARY_TIME;

}

count += 1;

time += elapsed;

uint64 summary = count;

summary <<= TIMESUMMARY_COUNT_OFFSET;

summary |= time;

timeSummary[D] = summary;

}

/////////////////////////////////////////////////////////

/// KEYBOARD

/////////////////////////////////////////////////////////

bool KeyReady()

{

if (sourceFile && sourceFile != stdin) return true;

#ifdef WIN32

if (ide) return idekey;

return _kbhit() ? true : false;

#else

bool ready = false;

struct termios oldSettings, newSettings;

if (tcgetattr( fileno( stdin ), &oldSettings ) == -1) return false; // could not get terminal attributes

newSettings = oldSettings;

newSettings.c_lflag &= (~ICANON & ~ECHO);

tcsetattr( fileno( stdin ), TCSANOW, &newSettings );

fd_set set;

struct timeval tv;

tv.tv_sec = 0;

tv.tv_usec = 0;

FD_ZERO( &set );

FD_SET( fileno( stdin ), &set );

int res = select( fileno( stdin )+1, &set, NULL, NULL, &tv );

ready = ( res > 0 );

tcsetattr( fileno( stdin ), TCSANOW, &oldSettings );

return ready;

#endif

}

/////////////////////////////////////////////////////////

/// EXCEPTION/ERROR

/////////////////////////////////////////////////////////

void SafeLock()

{

#ifdef SAFETIME

mtx.lock();

#endif

}

void SafeUnlock()

{

#ifdef SAFETIME

mtx.unlock();

#endif

}

void JumpBack()

{

if (jumpIndex < 0) return; // not under handler control

globalDepth = 0;

longjmp(scriptJump[jumpIndex], 1);

}

void CloseDatabases(bool restart)

{

bool usedb = (db == 1 && server) || db == 2;

#ifndef DISCARDPOSTGRES

if (*postgresparams)

{

PostgresScriptShutDown(); // any script connection

PGUserFilesCloseCode(); // filesystem

}

#endif

#ifndef DISCARDMONGO

if (!restart) MongoSystemShutdown();

else MongoUserFilesClose(); // actually just closes files

#endif

#ifndef DISCARDMYSQL

if (*mysqlparams) MySQLFullCloseCode(restart); // filesystem and/or script

#endif

#ifndef DISCARDMICROSOFTSQL

if (*mssqlparams && usedb) MsSqlFullCloseCode(); // filesystem and/or script

#endif

}

void myexit(const char* msg, int code)

{

int oldcompiling = compiling;

int oldloading = loading;

compiling = NOT_COMPILING;

loading = false;

traceTestPatternBuffer = NULL;

#ifdef LINUX

const char* fatal_str = strstr(msg, "FATAL:");

if (fatal_str != NULL) {

syslog(LOG_ERR, "%s user: %s bot: %s msg: %s ",

syslogstr, loginID, computerID, msg);

}

#endif

ReportBug("myexit called with %s ", msg); // since this does not say FATAL at start, it will not loop back here

char name[MAX_WORD_SIZE];

sprintf(name, (char*)"%s/exitlog.txt", logsfolder);

FILE* out;

if (stdlogging) out = stderr;

else out = FopenUTF8WriteAppend(name);

struct tm ptm;

if (out)

{

fprintf(out, (char*)"%s %d - called myexit at %s\r\n", msg, code, GetTimeInfo(&ptm, true));

if (!stdlogging) FClose(out);

}

if (code != 0 && crashset && !crashBack) // try to recover

{

#ifdef LINUX

siglongjmp(crashJump, 1);

#else

longjmp(crashJump, 1);

#endif

}

// non recoverable

out = NULL;

if (!oldcompiling && !oldloading)

{

if (stdlogging) out = stderr;

else out = FopenUTF8WriteAppend(name);

if (out)

{

struct tm ptm;

fprintf(out, (char*)"\r\n%s: caller:%s callee:%s ", GetTimeInfo(&ptm, true), loginID, computerID);

fprintf(out, (char*)"input:%s \r\n", currentInput );

BugBacktrace(out);

}

}

if (code == 0 && out) fprintf(out, (char*)"CS exited at %s\r\n", GetTimeInfo(&ptm, true));

else if (out) fprintf(out, (char*)"CS terminated at %s\r\n", GetTimeInfo(&ptm, true));

if (!stdlogging && out) FClose(out);

if (!client) CloseSystem();

exit((code == 0) ? EXIT_SUCCESS : EXIT_FAILURE);

}

void mystart(char* msg)

{

char name[MAX_WORD_SIZE];

MakeDirectory(logsfolder);

sprintf(name, (char*)"%s/startlog.txt", logsfolder);

FILE* out;

if (stdlogging) out = stdout;

else out = FopenUTF8WriteAppend(name);

char word[MAX_WORD_SIZE];

struct tm ptm;

sprintf(word, (char*)"System startup %s %s\r\n", msg, GetTimeInfo(&ptm, true));

if (out && !stdlogging)

{

fprintf(out, (char*)"%s", word);

FClose(out);

}

if (server) Log(SERVERLOG, "%s",word);

}

/////////////////////////////////////////////////////////

/// Fatal Error/signal logging

/////////////////////////////////////////////////////////

#ifdef LINUX

#include <signal.h>

void setSignalHandlers();

void signalHandler( int signalcode )

{

char word[MAX_WORD_SIZE];

sprintf(word, (char*)"FATAL: Linux Signal code %d", signalcode);

#ifdef PRIVATE_CODE

// Check for private hook function for additional handling

static HOOKPTR fn = FindHookFunction((char*)"SignalHandler");

if (fn)

{

char* ptr = word;

((SignalHandlerHOOKFN) fn)(signalcode, ptr);

}

#endif

myexit(word,1);

}

void setSignalHandlers ()

{

char word[MAX_WORD_SIZE];

struct sigaction sa;

sa.sa_handler = &signalHandler;

sigfillset(&sa.sa_mask); // Block every signal during the handler

// Handle relevant signals

if (sigaction(SIGFPE, &sa, NULL) == -1) {

sprintf(word, (char*)"Error: cannot handle SIGFPE");

Log(BUGLOG, word);

Log(SERVERLOG, word);

}

if (sigaction(SIGSEGV, &sa, NULL) == -1) {

sprintf(word, (char*)"Error: cannot handle SIGSEGV");

Log(BUGLOG, word);

Log(SERVERLOG, word);

}

if (sigaction(SIGBUS, &sa, NULL) == -1) {

sprintf(word, (char*)"Error: cannot handle SIGBUS");

Log(BUGLOG, word);

Log(SERVERLOG, word);

}

if (sigaction(SIGILL, &sa, NULL) == -1) {

sprintf(word, (char*)"Error: cannot handle SIGILL");

Log(BUGLOG, word);

Log(SERVERLOG, word);

}

if (sigaction(SIGTRAP, &sa, NULL) == -1) {

sprintf(word, (char*)"Error: cannot handle SIGTRAP");

Log(BUGLOG, word);

Log(SERVERLOG, word);

}

if (sigaction(SIGHUP, &sa, NULL) == -1) {

sprintf(word, (char*)"Error: cannot handle SIGHUP");

Log(BUGLOG, word);

Log(SERVERLOG, word);

}

}

#endif

/////////////////////////////////////////////////////////

/// MEMORY SYSTEM

/////////////////////////////////////////////////////////

void ResetBuffers()

{

globalDepth = 0;

bufferIndex = baseBufferIndex;

memset(releaseStackDepth,0,sizeof(releaseStackDepth));

outputNest = oldOutputIndex = 0;

currentRuleOutputBase = currentOutputBase = ourMainOutputBuffer;

currentOutputLimit = outputsize;

}

void CloseBuffers()

{

while (overflowLimit > 0)

{

free(overflowBuffers[--overflowLimit]);

overflowBuffers[overflowLimit] = 0;

}

free(buffers);

buffers = 0;

}

FunctionResult AuthorizedCode(char* buffer)

{

if ((server || pseudoServer ) && !hadAuthCode && !VerifyAuthorization(FopenReadOnly((char*)"authorizedIP.txt"))) return FAILRULE_BIT; // authorizedIP

return NOPROBLEM_BIT;

}

void LoggingCheats(char* incoming)

{

// logging overrides

FILE* in = fopen("serverlogging.txt", (char*)"rb"); // per external file created, enable server log

if (in)

{

serverLog |= FILE_LOG | PRE_LOG;

fclose(in);

}

in = fopen("prelogging.txt", (char*)"rb"); // per external file created, enable prelogging

if (in)

{

serverLog |= PRE_LOG;

userLog |= PRE_LOG;

prelog = true;

fclose(in);

}

in = fopen("userlogging.txt", (char*)"rb"); // per external file created, enable user log

if (in)

{

userLog |= FILE_LOG | PRE_LOG;

fclose(in);

}

in = fopen("tracelogging.txt", (char*)"rb"); // per external file created, enable user log

if (in)

{

trace = (unsigned int)-1;

userLog |= FILE_LOG;

fclose(in);

}

char* authcode = NULL;

if (incoming && *serverlogauthcode) authcode = strstr(incoming, serverlogauthcode);

if (authcode)// dont process authcode as input from user

{

memset(authcode, ' ', strlen(serverlogauthcode)); // hide auth code entirely

if (!userLog) userLog = FILE_LOG | PRE_LOG;

if (!serverLog) serverLog = FILE_LOG | PRE_LOG;

hadAuthCode = true;

}

else hadAuthCode = false;

}

char* Myfgets(char* buffer, int size, FILE* in)

{

return fgets(buffer, size, in);

//char* answer = fgets(buffer, size, in);

//PurifyInput(buffer, true, false); // leave ending crlfs

//return answer;

}

char* AllocateBuffer(char* name)

{

if (!buffers) return (char*)""; // IDE before start

char* buffer = buffers + (maxBufferSize * bufferIndex);

if (++bufferIndex >= maxBufferLimit ) // want more than nominally allowed

{

if (bufferIndex > (maxBufferLimit+2) || overflowIndex > 20)

{

char word[MAX_WORD_SIZE];

sprintf(word,(char*)"FATAL: Corrupt bufferIndex %u or overflowIndex %u\r\n",bufferIndex,overflowIndex);

ReportBug(word);

}

--bufferIndex;

// try to acquire more space, permanently

if (overflowIndex >= overflowLimit)

{

overflowBuffers[overflowLimit] = (char*) malloc(maxBufferSize);

if (!overflowBuffers[overflowLimit]) ReportBug((char*)"FATAL: out of buffers\r\n");

overflowLimit++;

if (overflowLimit >= MAX_OVERFLOW_BUFFERS) ReportBug((char*)"FATAL: Out of overflow buffers\r\n");

Log(USERLOG,"Allocated extra buffer %d\r\n",overflowLimit);

}

buffer = overflowBuffers[overflowIndex++];

}

else if (bufferIndex > maxBufferUsed) maxBufferUsed = bufferIndex;

if (showmem) Log(USERLOG,"%d Buffer alloc %d %s %s\r\n",globalDepth,bufferIndex,name, releaseStackDepth[globalDepth]->name);

*buffer++ = 0; // prior value

*buffer++ = 0; // prior value

*buffer = 0; // empty string

return buffer;

}

void FreeBuffer(char* name)

{ // if longjump happens this may not be called. manually restore counts in setjump code

if (showmem) Log(USERLOG,"%d Buffer free %s %d %s\r\n", globalDepth,name, bufferIndex, releaseStackDepth[globalDepth]->name);

if (overflowIndex) --overflowIndex; // keep the dynamically allocated memory for now.

else if (bufferIndex) --bufferIndex;

else ReportBug((char*)"Buffer allocation underflow")

}

void ResetHeapFree(char* val)

{

lastheapfree = heapFree = val;

}

void InitStackHeap()

{

size_t size = maxHeapBytes / 64; // stack + heap size

size = (size * 64) + 64; // 64 bit align both ends

if (!heapEnd)

{

heapEnd = (char*)malloc(size); // point to end of heap (start of stack)

if (!heapEnd)

{

(*printer)((char*)"Out of memory space for text space %d\r\n", (int)size);

ReportBug((char*)"FATAL: Cannot allocate memory space for text %d\r\n", (int)size)

}

}

ResetHeapFree( heapEnd + size); // allocate backwards

heapBase = heapFree;

stackStart = stackFree = heapEnd;

minHeapAvailable = maxHeapBytes;

PrepIndent();

}

void FreeStackHeap()

{

if (heapEnd) free(heapEnd);

heapEnd = NULL;

}

char* AllocateStack(const char* word, size_t len,bool localvar,int align) // call with (0,len) to get a buffer

{

if (!stackFree)

return NULL; // shouldnt happen

if (infiniteStack)

ReportBug("FATAL: Allocating stack while InfiniteStack in progress from %s\r\n",infiniteCaller);

if (len == 0)

{

if (!word ) return NULL;

len = strlen(word);

}

if (align == 1 || align == 4) // 1 is old true value

{

stackFree += 3;

uint64 x = (uint64)stackFree;

x &= 0xfffffffffffffffc;

stackFree = (char*)x;

}

else if (align == 8)

{

stackFree += 7;

uint64 x = (uint64)stackFree;

x &= 0xfffffffffffffff8;

stackFree = (char*)x;

}

unsigned int avail = heapFree - (stackFree + len + 1);

if (avail < 5000) // dont get close

{

ReportBug((char*)"FATAL: Out of stack space stringSpace:%d ReleaseStackspace:%d \r\n",heapBase-heapFree,stackFree - stackStart);

return NULL;

}

char* answer = stackFree;

if (localvar) // give hidden data

{

*answer = '`';

answer[1] = '`';

answer += 2;

}

*answer = 0;

if (word && *word) strncpy(answer, word, len); // dont copy if word is actually an empty string allowed with a nonzero length

answer[len++] = 0;

answer[len++] = 0;

if (localvar) len += 2;

stackFree += len;

len = stackFree - stackStart; // how much stack used are we?

len = heapBase - heapFree; // how much heap used are we?

len = heapFree - stackFree; // size of gap between

if (len < maxReleaseStackGap) maxReleaseStackGap = len;

return answer;

}

void ReleaseStack(char* word)

{

stackFree = word;

}

bool AllocateStackSlot(char* variable)

{

WORDP D = StoreWord(variable,AS_IS);

unsigned int len = sizeof(char*);

if ((stackFree + len + 1) >= (heapFree - 5000)) // dont get close

{

ReportBug((char*)"Out of stack space\r\n")

return false;

}

char* answer = stackFree;

memcpy(answer,&D->w.userValue,sizeof(char*));

if (D->word[1] == LOCALVAR_PREFIX) D->w.userValue = NULL; // autoclear local var

stackFree += sizeof(char*);

len = heapFree - stackFree;

if (len < maxReleaseStackGap) maxReleaseStackGap = len;

return true;

}

char** RestoreStackSlot(char* variable,char** slot)

{

WORDP D = FindWord(variable);

if (!D) return slot; // should never happen, we allocate dict entry on save

memcpy(&D->w.userValue,slot,sizeof(char*));

#ifndef DISCARDTESTING

if (debugVar) (*debugVar)(variable, D->w.userValue);

#endif

return ++slot;

}

char* InfiniteHeap(const char* caller)

{

if (infiniteHeap) ReportBug("FATAL: Allocating InfiniteHeap from %s while one already in progress from %s\r\n", caller, infiniteHeap);

infiniteCaller = caller;

infiniteHeap = true;

uint64 base = (uint64)(heapFree);

base &= 0xFFFFFFFFFFFFFFF8ULL;

heapFree = (char*)base;

return heapFree;

}

char* InfiniteStack(char*& limit,const char* caller)

{

if (infiniteStack) ReportBug("FATAL: Allocating InfiniteStack from %s while one already in progress from %s\r\n",caller, infiniteCaller);

infiniteCaller = caller;

infiniteStack = true;

limit = heapFree - 5000; // leave safe margin of error

uint64 base = (uint64) (stackFree+7);

base &= 0xFFFFFFFFFFFFFFF8ULL;

return (char*) base; // slop may be lost when allocated finally, but it can be reclaimed later

}

void ReleaseInfiniteStack()

{

infiniteStack = false;

infiniteCaller = "";

}

HEAPREF AllocateHeapval(HEAPREF linkval, uint64 val1, uint64 val2, uint64 val3)

{

uint64* heapval = (uint64*)AllocateHeap(NULL, 5, sizeof(uint64), false);

heapval[0] = (uint64)linkval;

heapval[1] = val1;

heapval[2] = val2;

heapval[3] = val3;

return (HEAPREF)heapval;

}

HEAPREF UnpackHeapval(HEAPREF linkval, uint64 & val1, uint64 & val2, uint64 & val3)

{

uint64* data = (uint64*)linkval;

val1 = data[1];

val2 = data[2];

val3 = data[3];

return (HEAPREF)data[0];

}

STACKREF AllocateStackval(STACKREF linkval, uint64 val1, uint64 val2, uint64 val3)

{

uint64* stackval = (uint64*)AllocateStack(NULL, 5 * sizeof(uint64), false,8);

stackval[0] = (uint64)linkval;

stackval[1] = val1;

stackval[2] = val2;

stackval[3] = val3;

return (STACKREF)stackval;

}

STACKREF UnpackStackval(STACKREF linkval, uint64& val1, uint64& val2, uint64& val3)

{ // factthread requires all 5, writes on its own

uint64* data = (uint64*) linkval;

val1 = data[1];

val2 = data[2];

val3 = data[3];

return (STACKREF)data[0];

}

void CompleteBindStack64(int n,char* base)

{

stackFree = base + ((n+1) * sizeof(FACT*)); // convert infinite allocation to fixed one given element count

size_t len = heapFree - stackFree;

if (len < maxReleaseStackGap) maxReleaseStackGap = len;

infiniteStack = false;

}

void CompleteBindStack(int used)

{

if (!used) stackFree += strlen(stackFree) + 1; // convert infinite allocation to fixed one

else stackFree += used;

size_t len = heapFree - stackFree;

if (len < maxReleaseStackGap) maxReleaseStackGap = len;

infiniteStack = false;

}

HEAPREF Index2Heap(HEAPINDEX offset)

{

if (!offset) return NULL;

char* ptr = heapBase - offset;

if (ptr < heapFree)

{

ReportBug((char*)"INFO: String offset into free space\r\n")

return NULL;

}

if (ptr > heapBase)

{

ReportBug((char*)"INFO: String offset before heap space\r\n")

return NULL;

}

return ptr;

}

bool PreallocateHeap(size_t len) // do we have the space

{

if (infiniteHeap)

ReportBug("FATAL: Allocating heap while InfiniteHeap in progress from %s\r\n", infiniteCaller);

char* used = heapFree - len;

if (used <= ((char*)stackFree + 2000))

{

ReportBug("Heap preallocation fails");

return false;

}

return true;

}

bool InHeap(char* ptr)

{

return (ptr >= heapFree && ptr <= heapBase);

}

bool InStack(char* ptr)

{

return (ptr < heapFree && ptr >= stackStart);

}

void ShowMemory(char* label)

{

(*printer)("%s: HeapUsed: %d Gap: %d\r\n", label, heapBase - heapFree,heapFree - stackFree);

}

char* AllocateConstHeap(char* word, size_t len, int bytes, bool clear, bool purelocal)

{

return AllocateHeap(word, len, bytes, clear, purelocal);

}

void CompleteInfiniteHeap(char* base)

{

infiniteHeap = false;

infiniteCaller = "";

if (base >= lastheapfree) ReportBug("Heap growing backwards")

size_t len = (heapFree - base);

lastheapfree = heapFree = (char*)base;

int nominalLeft = maxHeapBytes - (heapBase - heapFree);

if ((unsigned long)nominalLeft < minHeapAvailable) minHeapAvailable = nominalLeft;

char* used = heapFree - len;

if (used <= ((char*)stackFree + 2000) || nominalLeft < 0)

ReportBug((char*)"FATAL: Out of permanent heap space\r\n")

}

char* AllocateHeap(const char* word,size_t len,int bytes,bool clear, bool purelocal) // BYTES means size of unit

{ // string allocation moves BACKWARDS from end of dictionary space (as do meanings)

/* Allocations during setup as :

2 when setting up cs using current dictionary and livedata for extensions (plurals, comparatives, tenses, canonicals)

3 preserving flags or properties when removing them or adding them while the dictionary is unlocked - only during builds

4 reading strings during dictionary setup

5 assigning meanings or glosses or posconditionsfor the dictionary

6 reading in postag information

---

Allocations happen during volley processing as

1 adding a new dictionary word - all the time on user input

2. saving plan backtrack data

3 altering concepts[] and topics[] lists as a result of a mark operation or normal behavior

4. temps information

5 spellcheck adjusted word in sentence list

6 tokenizing words and quoted stuff adjustments

7. assignment onto user variables

8. JSON reading

*/

if (infiniteHeap)

ReportBug("FATAL: Allocating stack while InfiniteHeap in progress from %s\r\n", infiniteCaller);

len *= bytes; // this many units of this size

if (len == 0)

{

if (!word ) return NULL;

len = strlen(word);

}

if (word) ++len; // null terminate string

if (purelocal) len += 2; // for `` prefix

size_t allocationSize = len;

// always allocate in word units

unsigned int allocate = ((allocationSize + 3) / 4) * 4;

char* oldHeapFree = heapFree;

heapFree -= allocate; // heap grows lower, stack grows higher til they collide

uint64 base = (uint64)heapFree;

if (bytes > 4) // force 64bit alignment alignment

{

base &= 0xFFFFFFFFFFFFFFF8ULL; // 8 byte align

}

else if (bytes == 4) // force 32bit alignment alignment

{

base &= 0xFFFFFFFFFFFFFFFCULL; // 4 byte align

}

else if (bytes == 2) // force 16bit alignment alignment

{

base &= 0xFFFFFFFFFFFFFFFEULL; // 2 byte align

}

else if (bytes != 1)

ReportBug((char*)"Allocation of bytes is not std unit %d", bytes);

if (heapFree >= lastheapfree)

{

ReportBug("Heap growing backwards")

}

lastheapfree = heapFree = (char*)base;

char* newword = heapFree;

int nominalLeft = maxHeapBytes - (heapBase - heapFree);

if ((unsigned long) nominalLeft < minHeapAvailable) minHeapAvailable = nominalLeft;

char* used = heapFree - len;

if (used <= ((char*)stackFree + 2000) || nominalLeft < 0)

ReportBug((char*)"FATAL: Out of permanent heap space\r\n")

if (word)

{

if (purelocal) // never use clear true with this

{

*newword++ = LCLVARDATA_PREFIX;

*newword++ = LCLVARDATA_PREFIX;

len -= 2;

}

memcpy(newword,word,--len);

newword[len] = 0;

}

else if (clear) memset(newword,0,len);

return newword;

}

/////////////////////////////////////////////////////////

/// FILE SYSTEM

/////////////////////////////////////////////////////////

int FClose(FILE* file)

{

if (file) fclose(file);

*currentFilename = 0;

return 0;

}

void InitUserFiles()

{

// these are dynamically stored, so CS can be a DLL.

userFileSystem.userCreate = FopenBinaryWrite;

userFileSystem.userOpen = FopenReadWritten;

userFileSystem.userClose = FClose;

userFileSystem.userRead = fread;

userFileSystem.userWrite = fwrite;

userFileSystem.userDelete = FileDelete;

userFileSystem.userDecrypt = NULL;

userFileSystem.userEncrypt = NULL;

filesystemOverride = NORMALFILES;

}

static size_t CleanupCryption(char* buffer,bool decrypt,char* filekind)

{

// clean up answer data: {"datavalues": {"USER": xxxx }}

char* answer = strstr(buffer,filekind);

if (!answer)

{

*buffer = 0;

return 0;

}

answer += strlen(filekind) + 2; // skip USER":

int realsize = jsonOpenSize - (answer-buffer) - 2; // the closing two }

memmove(buffer,answer,realsize); // drop head data

buffer[realsize] = 0; // remove trailing data

// legal json doesnt allow control characters, but we cannot change our file system ones to \r\n because

// we cant tell which are ours and which are users. So we changed to 7f7f coding.

if (decrypt)

{

char* at = buffer-1;

while (*++at)

{

if (*at == 0x7f && at[1] == 0x7f)

{

*at = '\r';

at[1] = '\n';

}

}

}

return realsize;

}

void ProtectNL(char* buffer) // save ascii \r\n in json - only comes from userdata write using them

{

char* at = buffer;

while ((at = strchr(at,'\r'))) // legal convert

{

if (at[1] == '\n' ) // legal convert

{

*at++ = 0x7f;

*at++ = 0x7f;

}

}

}

bool notcrypting = false;

static int JsonOpenCryption(char* buffer, size_t size, char* xserver, bool decrypt,char* filekind)

{

if (size == 0) return 0;

if (notcrypting) return size;

if (decrypt && size > 50) return size; // it was never encrypted, this is original material

char serverurl[1000];

char* id = loginID;

if (*id == 'b' && !id[1]) id = "u-8b02518d-c148-5d45-936b-491d39ced70c"; // cheat override to test outside of normal logins

if (decrypt) sprintf(serverurl, "%s%s/datavalues/decryptedtokens", xserver, id);

else sprintf(serverurl, "%s%s/datavalues/encryptedtokens", xserver, id);

//loginID

// for decryption, the value we are passed is a quoted string. We need to avoid doubling those

#ifdef INFORMATION

// legal json requires these characters be escaped, but we cannot change our file system ones to \r\n because

// we cant tell which are ours and which are users. So we change to 7f7f coding for /r/n.

// and we change to 0x31 for double quote.

// \b Backspace (ascii code 08) -- never seeable

// \f Form feed (ascii code 0C) -- never seable

// \n New line -- only from our user topic write

// \r Carriage return -- only from our user topic write

// \t Tab -- never seeable

// Double quote -- seeable in user data

// \\ Backslash character -- ??? not expected but possible

UserFile encoding responsible for normal JSON safe data.

Note MongoDB code also encrypts \r\n the same way. but we dont know HERE that mongo is used

and we dont know THERE that encryption was used. That is redundant but minor.

#endif

if (!decrypt) ProtectNL(buffer); // should not alter size

else // remember what we decrypt so we can overwrite it later when we save

{

if (!stricmp(filekind, "USER")) strcpy(encryptUser,buffer);

else strcpy(encryptLTM, buffer);

}

// prepare body for transfer to server to look like this for encryption:

// {"datavalues":{"user": {"data": {"userdata1":"abc"}}}

// or {"datavalues":{"ltm": {"data": {"userdata1":"abc"}}}

// or "{datavalues":{ "user": {"data": {"userdata1":"abc", "token" : "Sy4KoML_e"}}}

// FOR decryption: {"datavalues":{"USER": "HkD_r-KFl"}}

if (decrypt) sprintf(buffer + size, "}}"); // add suffix to data - no quote

else // encrypt gives a token if reusing

{

char* at = buffer + size;

char* which = NULL;

if (!stricmp(filekind, "USER")) which = encryptUser;

else which = encryptLTM;

if (which && *which)

{

sprintf(at,"\",\"token\": %s",which);

at += strlen(at);

}

else

{

*at++ = '"';

}

sprintf(at, "}}}"); // add suffix to data

}

size += strlen(buffer+size);

char url[500];

if (decrypt) sprintf(url, "{\"datavalues\":{\"%s\": ", filekind);

else sprintf(url, "{\"datavalues\":{\"%s\": {\"data\": \"", filekind);

int headerlen = strlen(url);

memmove(buffer+headerlen,buffer,size+1); // move the data over to put in the header

strncpy(buffer,url,headerlen);

// set up call to json open

char header[500];

strcpy(header,"Content-Type: application/json");

int oldArgumentIndex = callArgumentIndex;

int oldArgumentBase = callArgumentBase;

callArgumentBase = callArgumentIndex - 1;

callArgumentList[callArgumentIndex++] = (char*)"direct"; // get the text of it gives us, dont make facts out of it

callArgumentList[callArgumentIndex++] = (char*)"POST";

strcpy(url, serverurl);

callArgumentList[callArgumentIndex++] = url;

callArgumentList[callArgumentIndex++] = (char*) buffer;

callArgumentList[callArgumentIndex++] = header;