#include "common.h"

#ifdef INFORMATION

This file covers routines that createand access a "dictionary entry" (WORDP) and the "meaning" of words(MEANING).

The dictionary acts as the central hash mechanism for accessing various kinds of data.

The dictionary consists of data imported from WORDNET 3.0 (copyright notice at end of file) + augmentations + system and script pseudo - words.

A word also gets the WordNet meaning ontology(->meanings & ->meaningCount).The definition of meaning in WordNet

is words that are synonyms in some particular context.Such a collection in WordNet is called a synset.

Since words can have multiple meanings(and be multiple parts of speech), the flags of a word are a summary

of all of the properties it might haveand it has a list of entries called "meanings".Each entry is a MEANING

and points to the circular list, one of which marks the word you land at as the synset head.

This is referred to as the "master" meaning and has the gloss(definition) of the meaning.The meaning list of a master node points back to

all the real words which comprise it.

Since WordNet has an ontology, its synsets are hooked to other synsets in various relations, particular that

of parentand child.ChatScript represents these as facts.The hierarchy relation uses the verb "is" and

has the child as subject and the parent as object.Such a fact runs from the master entries, not any of the actual

word entries.So to see if "dog" is an "animal", you could walk every meaning list of the word animaland

mark the master nodes they point at.Then you would search every meaning of dog, jumping to the master nodes,

then look at facts with the master node as subjectand the verb "is" and walk up to the object.If the object is

marked, you are there.Otherwise you take that object node as subjectand continue walking up.Eventually you arrive at

a marked node or run out at the top of the tree.

Some words DO NOT have a master node.Their meaning is defined to be themselves(things like pronouns or determiners), so

their meaning value for a meaning merely points to themselves.

The meaning system is established by building the dictionaryand NEVER changes thereafter by chatbot execution.

New words can transiently enter the dictionary for various purposes, but they will not have "meanings".

A MEANING is a reference to a specific meaning of a specific word.It is an index into the dictionary

(identifying the word) and an index into that words meaning list(identifying the specific meaning).

An meaning list index of 0 refers to all meanings of the word.A meaning index of 0 can also be type restricted

so that it only refers to noun, verb, adjective, or adverb meanings of the word.

Since there are only two words in WordNet with more than 63 meanings(breakand cut) we limit all words to having no

more than 63 meanings by discarding the excess meanings.Since meanings are stored most important first,

these are no big loss.This leaves room for the 5 essential type flags used for restricting a generic meaning.

Space for dictionary words comes from a common pool.Dictionary words are

allocated linearly forward in the pool.Strings have their own pool(the heap).

All dictionary entries are indexable as a giant array.

The dictionary separately stores uppercaseand lowercase forms of the same words(since they have different meanings).

There is only one uppercase form stored, so Unitedand UnItED would be saved as one entry.The system will have

to decide which case a user intended, since they may not have bothered to capitalize a proper noun, or they

may have shouted a lowercase noun, and a noun at the start of the sentence could be either proper or not.

Dictionary words are hashed as lower case, but if the word has an upper case letter it will be stored

in the adjacent higher bucket.Words of the basic system are stored in their appropriate hash bucket.

After the basic system is read in, the dictionary is frozen.This means it remembers the spots the allocation

pointers are at for the dictionaryand heap spaceand is using mark - release memory management.

The hash buckets are themselves dictionary entries, sharing space.After the basic layers are loaded,

new dictionary entries are always allocated.Until then, if the word hashs to an empty bucket, that bucket

becomes the dictionary entry being added.

We mark sysnet entries with the word& meaning number& POS of word in the dictionary entry.The POS not used explicitly by lots of the system

but is needed when seeing the dictionary definitions(:word) and if one wants to use pos - restricted meanings in a match or in keywords.

#endif

#define DICTSEARCHLIMIT 1000

bool exportdictionary = false;

bool warnedaboutbuild0 = false;

char current_language[150] = ""; // indicate current language used

char language_list[400]; // indicate legal languages- comma separated

unsigned int language_bits = LANGUAGE_UNIVERSAL; // current language used to mark/filter WORDP items

unsigned int languageIndex; // current language used to mark facts and index data by language

unsigned int max_language = LANGUAGE_1;

unsigned int max_fact_language = FACTLANGUAGE_1;

static WORDP startOfLanguage = NULL;

static WORDP endOfLanguage = NULL;

int worstDictAvail = 1000000;

bool dictionaryBitsChanged = false;

bool multidict = false;

HEAPREF propertyRedefines = NULL; // property changes on locked dictionary entries

HEAPREF flagsRedefines = NULL; // systemflags changes on locked dictionary entries

HEAPREF ongoingDictChanges = NULL; // ability to revert dynamic changes

bool monitorDictChanges = false;

bool xbuildDictionary = false; // indicate when building a dictionary

char dictionaryTimeStamp[50]; // indicate when dictionary was built

const char* mini = ""; // what language

unsigned int* hashbuckets = 0;

unsigned int wordAccessCount = 0; // number of words looked up in this volley

unsigned int bucketDepthCount = 0; // sum of depths required for each word lookup

static unsigned char* writePtr; // used for binary dictionary writes

// memory data

unsigned long maxHashBuckets = MAX_HASH_BUCKETS;

bool setMaxHashBuckets = false;

uint64 maxDictEntries = MAX_DICTIONARY;

MEANING posMeanings[64]; // concept associated with propertyFlags of WORDs

MEANING sysMeanings[64]; // concept associated with systemFlags of WORDs

#include <map>

using namespace std;

std::map <WORDP, WORDP> irregularNouns;

std::map <WORDP, WORDP> irregularVerbs;

std::map <WORDP, WORDP> irregularAdjectives;

std::map <WORDP, WORDP> canonicalWords;

std::map <WORDP, int> wordValues; // per volley

std::map <WORDP, MEANING> backtracks; // per volley

std::map <WORDP, int> countData;

static void ReadAsciiDictionary();

HEAPREF concepts[MAX_SENTENCE_LENGTH]; // concept chains per word

HEAPREF topics[MAX_SENTENCE_LENGTH]; // topics chains per word

bool fullDictionary = true; // we have a big master dictionary, not a mini dictionary

bool primaryLookupSucceeded = false;

void LoadRawDictionary(int mini);

bool TraceHierarchyTest(int x)

{

if (!(x & TRACE_HIERARCHY)) return false;

x &= -1 ^ (TRACE_ON | TRACE_ALWAYS | TRACE_HIERARCHY | TRACE_ECHO);

return x == 0; // must exactly have been trace hierarchy

}

MEANING GetMeaning(WORDP D, int index)

{

MEANING* meanings = GetMeanings(D);

if (meanings) return meanings[index];

else return MakeMeaning(D); // switch to generic non null meaning

}

unsigned int LanguageRestrict(char* word)

{

char* restrict = strstr(word, "~l");

if (restrict && restrict[2] >= '0' && restrict[2] <= '7')

{

*restrict = 0;

unsigned int bits = (restrict[2] - '0') << LANGUAGE_SHIFT;

if (!multidict && bits > max_language)

return language_bits;

return bits;

}

return language_bits;

}

void SetTried(WORDP D, int value)

{

triedData[D] = value;

}

bool IsUniversal(char* word, uint64 properties)

{

char c = *word;

if (c == '$' || c == '^' || c == '%' || c == '~'

|| c == '+' || c == '-' || c == '.') return true;

if (c == '_' && (!word[1] || IsDigit(word[1]))) return true;

if (IsValidJSONName(word)) return true;

if (!strcmp(word,"member") || !strcmp(word, "is")) return true; // system verbs

if (properties & PUNCTUATION_BITS) return true;

if (IsPureNumber(word)) return true;

char* at = word - 1;

while (IsDigit(*++at) ) { ; }

if (*at == 0) return true; // purely a number - not allowed to write it out. not allowed to have unusual flags

return false;

}

bool IsValidLanguage(WORDP D)

{

if (!D || !multidict || !D->word) return true;

// language independent ideas (numbers, concepts, variables, functions, systemvars )

// should do international currencies but not doing yet

unsigned int bits = D->internalBits & LANGUAGE_BITS;

if (!bits) return true; // universal cs words match always

if (IsUniversal(D->word, 0)) return true;

// matches language or we match ALL!

return (bits == language_bits);

}

void RemoveConceptTopic(HEAPREF list[256], WORDP D, int index)

{ // these lists are only for script to easily access, engine doesnt use them itself

HEAPREF at = list[index];

HEAPREF prior = NULL;

while (at)

{

uint64 m;

uint64* currentEntry = (uint64*)at;

at = UnpackHeapval(at, m, discard);

if (((uint64)D) == m)

{

if (prior == NULL) list[index] = at; // list head

else ((uint64*)prior)[0] = currentEntry[0]; // splice this out

return; // assumed only on list once

}

prior = at;

}

}

void Add2ConceptTopicList(HEAPREF list[256], WORDP D, int start, int end, bool unique)

{

if (unique)

{

HEAPREF at = list[start];

while (at)

{

uint64 D1;

at = UnpackHeapval(at, D1, discard, discard);

if (D1 == (uint64)D) return; // already on list

}

}

// concepts[] and topics[]

list[start] = AllocateHeapval(list[start], (uint64)D, 0, 0); // concepts[i] and topics[i] lists of word indices

}

void ClearHeapThreads()

{

memoryMarkThreadList = NULL;

userVariableThreadList = NULL;

savedSentencesThreadList = NULL;

propertyRedefines = NULL; // property changes on locked dictionary entries

flagsRedefines = NULL; // system flag changes on locked dictionary entries

factThread = NULL;

rulematches = NULL;

patternwordthread = NULL;

variableChangedThreadlist = NULL;

memoryVariableThreadList = NULL; // list of variable changes for each mark

memoryVariableChangesThreadList = NULL; // list of original values in this mark

matchedWordsList = NULL;

for (int i = 1; i < MAX_SENTENCE_LENGTH; ++i) concepts[i] = topics[i] = NULL;

// kernelVariableThreadList

// botVariableThreadList;

}

void ClearVolleyWordMaps()

{

wordValues.clear();

backtracks.clear();

ClearWhereInSentence();

}

void ClearWordMaps() // both static for whole dictionary and dynamic per volley

{

irregularNouns.clear();

irregularVerbs.clear();

irregularAdjectives.clear();

canonicalWords.clear();

triedData.clear(); // prevent document reuse

ClearVolleyWordMaps();

}

void ClearWhereAt(int where) // remove all concepts and markings at this slot in sentence

{

WORDP D;

if (trace & TRACE_HIERARCHY)

{

HEAPREF list = concepts[where];

while (list)

{

uint64 val;

list = UnpackHeapval(list, val, discard, discard);

WORDP X = (WORDP)val;

Log(USERLOG, " %s ", X->word);

}

list = topics[where];

while (list)

{

uint64 val;

list = UnpackHeapval(list, val, discard, discard);

WORDP X = (WORDP)val;

Log(USERLOG, " %s ", X->word);

}

}

concepts[where] = NULL; // drop memory list at this slot

topics[where] = NULL; // drop memory list

WORDP holdlist[10000];

HEAPINDEX holdint[10000];

int mark[10000];

memset(mark, 0, sizeof(mark));

unsigned int index = 0;

bool changed = false;

// copy because we will rewrite triedData.

map<WORDP, HEAPINDEX>::iterator it1;

for (it1 = triedData.begin(); it1 != triedData.end(); ++it1)

{

D = it1->first;

unsigned int x = it1->second;

holdint[index] = x;

holdlist[index++] = D;

}

for (unsigned int item = 0; item < index; ++item)

{

HEAPINDEX access = holdint[item]; // heap access

char* data = Index2Heap(access);

if (data == 0) continue;

unsigned char* lcltriedData = (unsigned char*)(data + 8); // skip over 64bit tried by meaning field

int i;

int counter = 0;

for (i = 0; i < MAXREFSENTENCE_BYTES; i += REF_ELEMENT_SIZE) // 6 bytes per entry x 300 entries

{

unsigned char start = lcltriedData[i];

if (start == where) // modify it

{

if (!mark[item]) // change over to a copy (dont harm original)

{

int newaccess = CopyWhereInSentence(access);

holdint[item] = newaccess;

mark[item] = 1;

changed = true;

data = Index2Heap(newaccess);

lcltriedData = (unsigned char*)(data + 8); // skip over 64bit tried by meaning field

}

int remain = MAXREFSENTENCE_BYTES - i - REF_ELEMENT_SIZE;

if (remain == 0) // just chop it off

{

lcltriedData[i] = 0xff;

lcltriedData[i + 1] = 0xff;

break;

}

memmove(lcltriedData + i, lcltriedData + i + REF_ELEMENT_SIZE, remain); // remove element

// insert end marker at end

lcltriedData[MAXREFSENTENCE_BYTES - REF_ELEMENT_SIZE] = 0xff;

lcltriedData[MAXREFSENTENCE_BYTES - REF_ELEMENT_SIZE + 1] = 0xff;

if (++counter > 100) // precaution

break;

i -= REF_ELEMENT_SIZE;

continue; // check next one

}

else if (start > where) break;

}

}

if (changed)

{

for (unsigned int item = 0; item < index; ++item)

{

if (!mark[item]) continue;

D = holdlist[item];

HEAPINDEX access = holdint[item]; // heap access

triedData[D] = access; // replace now

}

}

}

void ClearWordWhere(WORDP D, int at)

{

triedData.erase(D);

// also remove from concepts/topics lists

if (at == -1)

{

for (unsigned int i = 1; i <= (unsigned int)wordCount; ++i)

{

RemoveConceptTopic(concepts, D, i);

RemoveConceptTopic(topics, D, i);

}

}

}

void SetFactBack(WORDP D, MEANING M)

{

if (GetFactBack(D) == 0)

{

backtracks[D] = M;

}

}

MEANING GetFactBack(WORDP D)

{

std::map<WORDP, MEANING>::iterator it;

it = backtracks.find(D);

return (it != backtracks.end()) ? it->second : 0;

}

void ClearBacktracks()

{

backtracks.clear();

}

void SetPlural(WORDP D, MEANING M)

{

irregularNouns[D] = dictionaryBase + M; // must directly assign since word on load may not exist

}

void SetComparison(WORDP D, MEANING M)

{

irregularAdjectives[D] = dictionaryBase + M; // must directly assign since word on load may not exist

}

void SetTense(WORDP D, MEANING M)

{

irregularVerbs[D] = dictionaryBase + M; // must directly assign since word on load may not exist

}

void SetCanonical(WORDP D, MEANING lemma)

{

canonicalWords[D] = dictionaryBase + lemma; // must directly assign since word on load may not exist

}

WORDP RawCanonical(WORDP D)

{

std::map<WORDP, WORDP>::iterator it;

it = canonicalWords.find(D);

if (it == canonicalWords.end()) return NULL;

return it->second;

}

WORDP GetCanonical(WORDP D, uint64 kind)

{

std::map<WORDP, WORDP>::iterator it;

it = canonicalWords.find(D);

if (it == canonicalWords.end()) return NULL;

WORDP E = it->second;

if (*E->word != '`' || kind == (uint64) -1) return E; // normal english canonical or simple canonical of foreign (not multiple choice). 0 kind means want all

// foreign canonicals have multiple choices. This code only picks first one if no type given. Other code may decide more complex, but generally treetagger will do that

char word[MAX_WORD_SIZE];

char type[MAX_WORD_SIZE];

strcpy(word, E->word + 1);

char* lemma = word;

char* tags = strrchr(word, '`');

if (tags) *tags++ = 0; // type decriptors

WORDP lemmachoice = NULL;

int lemmacount = 0;

// walk list of pos tags and simultaneously walk the list of lemmas

while (tags && *tags && (tags = ReadCompiledWord(tags, type)))

{

char* split = strchr(lemma, '`');

if (split) *split = 0;

// if we have multiple meanings unresolved, we cannot pick a lemma based on type

if (*type == 'V' && kind & VERB)

{

lemmachoice = StoreWord(lemma, AS_IS|VERB);

++lemmacount;

}

else if (*type == 'N' && kind & NOUN)

{

lemmachoice = StoreWord(lemma, AS_IS|NOUN);

++lemmacount;

}

else if (*type == 'A' && type[2] == 'J' && kind & ADJECTIVE)

{

lemmachoice = StoreWord(lemma, AS_IS| NOUN);

++lemmacount;

}

if (split)

{

*split = '`';

lemma = split + 1;

}

tags = SkipWhitespace(tags);

}

if (lemmacount == 1) return lemmachoice;

// if we dont know the type, we dont know which lemma to pick

// So pick one different from original, so we get broader mark coverage

char* next = word;

while (*next)

{

char* end = strchr(next, '`'); // multiple choice?

if (end)

{

WORDP X = FindWord(next, end - next,PRIMARY_CASE_ALLOWED);

if (X && stricmp(X->word, D->word)) return X; // prefer a different one

next = end + 1;

}

else

{

WORDP X = FindWord(next);

if (X) return X;

break;

}

}

return StoreWord(word, AS_IS);

}

WORDP GetTense(WORDP D)

{

std::map<WORDP, WORDP>::iterator it;

it = irregularVerbs.find(D);

return (it != irregularVerbs.end()) ? it->second : NULL;

}

WORDP GetPlural(WORDP D)

{// given singular returns plural and visa versa

std::map<WORDP, WORDP>::iterator it;

it = irregularNouns.find(D);

return (it != irregularNouns.end()) ? it->second : NULL;

}

WORDP GetComparison(WORDP D)

{

std::map<WORDP, WORDP>::iterator it;

it = irregularAdjectives.find(D);

return (it != irregularAdjectives.end()) ? it->second : NULL;

}

void SetWordValue(WORDP D, int x)

{

wordValues[D] = x;

}

int GetWordValue(WORDP D)

{

std::map<WORDP, int>::iterator it;

it = wordValues.find(D);

return (it != wordValues.end()) ? it->second : 0;

}

// start and ends of space allocations

WORDP dictionaryBase = NULL; // base of allocated space that encompasses dictionary, heap space, and meanings

WORDP dictionaryFree = NULL; // current next dict space available going forward (not a valid entry)

// return-to values for layers

WORDP dictionaryPreBuild[NUMBER_OF_LAYERS + 1];

char* heapPreBuild[NUMBER_OF_LAYERS + 1];

// build0Facts

// return-to values after build1 loaded, before user is loaded

WORDP dictionaryLocked = NULL;

FACT* factLocked = 0;

char* stringLocked;

// format of word looked up

uint64 verbFormat;

uint64 nounFormat;

uint64 adjectiveFormat;

uint64 adverbFormat;

// dictionary ptrs for these words

WORDP Dplacenumber;

WORDP Dpropername;

MEANING Mphrase;

MEANING MabsolutePhrase;

MEANING MtimePhrase;

WORDP Dclause;

WORDP Dverbal;

WORDP Dmalename, Dfemalename, Dhumanname;

WORDP Dtime;

WORDP Dunknown;

WORDP Dchild, Dadult;

WORDP Dtopic;

MEANING Mchatoutput;

MEANING Mburst;

MEANING Mpending;

MEANING Mkeywordtopics;

MEANING Mconceptlist;

MEANING Mmoney;

MEANING Mintersect;

MEANING MgambitTopics;

MEANING MadjectiveNoun;

MEANING Mnumber;

WORDP Dpronoun;

WORDP Dadjective;

WORDP Dauxverb;

WORDP DunknownWord;

static char* predefinedSets[] = // some internally mapped concepts not including emotions from LIVEDATA/interjections

{

(char*)"~repeatme",(char*)"~repeatinput1",(char*)"~repeatinput2",(char*)"~repeatinput3",(char*)"~repeatinput4",(char*)"~repeatinput5",(char*)"~repeatinput6",(char*)"~uppercase",(char*)"~utf8",(char*)"~sentenceend",

(char*)"~pos",(char*)"~sys",(char*)"~grammar_role",(char*)"~daynumber",(char*)"~yearnumber",(char*)"~dateinfo",(char*)"~formatteddate",(char*)"~email_url",(char*)"~fahrenheit",(char*)"~celsius",(char*)"~kelvin",

(char*)"~kindergarten",(char*)"~grade1_2",(char*)"~grade3_4",(char*)"~grade5_6",(char*)"~twitter_name",(char*)"~hashtag_label",(char*)"~filename",

(char*)"~shout",(char*)"~distance_noun_modify_adverb",

(char*)"~distance_noun_modify_adjective",(char*)"~modelnumber",

(char*)"~passive_verb",

(char*)"~time_noun_modify_adverb",

(char*)"~time_noun_modify_adjective",

(char*)"~capacronym",(char*)"~emoji",

NULL

};

void RestorePropAndSystem(char* stringUsed)

{

uint64 D;

while (propertyRedefines) // must release these

{

uint64 properties;

if (stringUsed && propertyRedefines >= stringUsed) break; // not part of this freeing

propertyRedefines = UnpackHeapval(propertyRedefines, D, properties, discard);

((WORDP)D)->properties = properties;

}

while (flagsRedefines) // must release these

{

uint64 properties;

if (stringUsed && flagsRedefines >= stringUsed) break; // not part of this freeing

flagsRedefines = UnpackHeapval(flagsRedefines, D, properties, discard);

((WORDP)D)->systemFlags = properties;

}

}

void DictionaryRelease(WORDP until, char* stringUsed)

{

RestorePropAndSystem(stringUsed);

ongoingDictChanges = NULL; // discard user tracked changes since we have our own essential tracking

if (until) while (dictionaryFree > until) DeleteDictionaryEntry(--dictionaryFree); // remove entry from buckets

lastheapfree = heapFree = stringUsed;

}

char* UseDictionaryFile(const char* name,const char* list)

{

static char junk[100];

if (*mini) sprintf(junk, (char*)"DICT/%s", mini);

else if (!*current_language) sprintf(junk, (char*)"%s", (char*)"DICT");

else if (!name) sprintf(junk, (char*)"DICT/%s", current_language);

else if (list && strchr(list, ',')) sprintf(junk, (char*)"DICT"); // use global conglomerate

else sprintf(junk, (char*)"DICT/%s", current_language);

MakeDirectory(junk); // if it doesnt exist

if (name && *name)

{

strcat(junk, (char*)"/");

strcat(junk, name);

}

return junk;

}

MEANING FindChild(MEANING who, int n)

{ // GIVEN SYNSET

FACT* F = GetObjectNondeadHead(who);

unsigned int index = Meaning2Index(who);

while (F)

{

FACT* at = F;

F = GetObjectNondeadNext(F);

if (at->verb != Mis) continue;

if (index && at->object != who) continue; // not us

if (--n == 0) return at->subject;

}

return 0;

}

bool ReadForeignPosTags(const char* fname)

{

FILE* in = FopenReadOnly(fname);

if (!in) return false;

char word[MAX_WORD_SIZE];

char wordx[MAX_WORD_SIZE];

*word = '_'; // marker to keep any collision away from foreign pos

*wordx = '~'; // corresponding concept

while (ReadALine(readBuffer, in) >= 0) // foreign name followed by bits of english pos

{

char* ptr = ReadCompiledWord(readBuffer, word + 1);

if (!word[1] || word[1] == '#')

continue;

uint64 flags = 0;

char flag[MAX_WORD_SIZE];

while (*ptr) // get english pos values of this tag

{

ptr = ReadCompiledWord(ptr, flag);

if (!*flag || *flag == '#') break;

uint64 val = FindPropertyValueByName(flag);

if (!val)

{

(*printer)("Unable to find flag %s\r\n", flag);

}

flags |= val;

}

StoreWord(word, flags); // _foreignpos gets bits

MakeLowerCopy(wordx + 1, word + 1); // force the name to lowercase because the name might not be valid, e.g. DET:ART

BUILDCONCEPT(wordx);

}

fclose(in);

// Also store the unknown tag, just in case the pos tagger fails to attach anything

strcpy(word + 1, (char*)"unknown-tag");

StoreWord(word, 0);

*word = '~'; // corresponding concept

BUILDCONCEPT(word);

return true;

}

unsigned char BitCount(uint64 n)

{

unsigned char count = 0;

while (n)

{

count++;

n &= (n - 1);

}

return count;

}

WORDP GetSubstitute(WORDP D)

{

if (D && D->systemFlags & HAS_SUBSTITUTE)

{

if (D->w.substitutes) return D->w.substitutes;

}

return NULL;

}

void BuildShortDictionaryBase();

static void EraseFile(const char* file,bool tmp)

{

char word[MAX_WORD_SIZE];

FILE* out;

if (tmp)

{

sprintf(word, "tmp/%s", file);

out = FopenUTF8Write(word);

}

else out = FopenUTF8Write(UseDictionaryFile(file));

FClose(out);

}

void ClearDictionaryFiles(bool tmp)

{

char buffer[MAX_WORD_SIZE];

remove(UseDictionaryFile((char*)"dict.bin"));

remove(UseDictionaryFile((char*)"facts.bin"));

EraseFile((char*)"dict.bin",tmp); // create but empty file

EraseFile((char*)"facts.bin",tmp); // create but empty file

unsigned int i;

for (i = 'a'; i <= 'z'; ++i)

{

sprintf(buffer, (char*)"%c.txt", i);

EraseFile(buffer,tmp); // create but empty file

}

for (i = '0'; i <= '9'; ++i)

{

sprintf(buffer, (char*)"%c.txt", i);

EraseFile(buffer,tmp); // create but empty file

}

EraseFile("other.txt", tmp); // create but empty file

}

static const char* GetLanguage(WORDP D)

{

if (!D) return current_language;

char* comma = strchr(language_list, ',');

int n = (D->internalBits & LANGUAGE_BITS) >> LANGUAGE_SHIFT;

if (!n || !comma)

{

return (comma) ? "UNIVERSAL" : current_language;

}

char list[MAX_WORD_SIZE];

strcpy(list, language_list);

static char languages[20][100];

memset(languages, 0, sizeof(languages));

int x = 1;

char* ptr = list;

while ((comma = strchr(ptr,',')))

{

*comma = 0;

strcpy(languages[x++], ptr);

ptr = comma + 1;

}

strcpy(languages[x++], ptr);

int limit = LANGUAGE_BITS >> LANGUAGE_SHIFT;

if (n >= 1 && n <= limit) return languages[n];

else return "unknown language";

}

bool SetLanguage(char* arg1)

{

if (!arg1 || !*arg1) return false;

char lang[100];

arg1 = TrimSpaces(arg1);

MakeUpperCopy(lang, arg1);

if (!stricmp(lang, "UNIVERSAL"))

{

languageIndex = language_bits = LANGUAGE_UNIVERSAL;

strcpy(current_language, "UNIVERSAL");

return true;

}

char copy[200];

strcpy(copy, language_list);

char* which = strstr(copy, lang);

if (!which) // is it a legal choice?

{

if (compiling) BADSCRIPT("Illegal set language %s",arg1)

return false;

}

// eg language=english,spanish,german,japanese, chinese

strcpy(current_language, lang);

if (!stricmp(lang, "GERMAN") || !stricmp(lang, "SPANISH") || !stricmp(lang, "FRENCH"))

{

numberStyle = FRENCH_NUMBERS;

numberComma = '.';

numberPeriod = ',';

}

else

{

numberStyle = AMERICAN_NUMBERS; // ignores indian

numberComma = ',';

numberPeriod = '.';

}

if (!multidict) {}

else

{

languageIndex = 1;

*which = 0; // hide all language at and after

which = copy - 1;

// language=english,spanish,german,japanese

while ((which = strchr(++which, ',')))

++languageIndex; // how many commas before

language_bits = languageIndex << LANGUAGE_SHIFT; // could have up to 7 languages

}

return true;

}

static void WriteDictionaryReference(char* label, WORDP D, FILE* out)

{

if (!D) return;

if (D->internalBits & DELETED_MARK) return; // ignore bad links

fprintf(out, (char*)"%s=%s ", label, D->word);

}

void WriteTextEntry(WORDP D,bool tmp)

{

char word[MAX_WORD_SIZE];

MakeLowerCopy(word, D->word);

// choose appropriate subfile

char c = *word;

char name[40];

if (IsDigit(c)) sprintf(name, (char*)"%c.txt", c); // main real dictionary

else if (!IsLowerCase(c) || (unsigned char)c > 127)

sprintf(name, (char*)"%s", (char*)"other.txt"); // main real dictionary

else sprintf(name, (char*)"%c.txt", c);// main real dictionary

FILE* out;

if (!tmp) out = FopenUTF8WriteAppend(UseDictionaryFile(name));

else

{

char hold[MAX_WORD_SIZE];

sprintf(hold, "tmp/%s", name);

out = FopenUTF8WriteAppend(hold);

}

if (!out) myexit((char*)"Dict write failed");

if (!stricmp(current_language,"spanish") && RawCanonical(D)) // we were given a lemma for it

{

WORDP entry, canonical;

uint64 sysflags;

if (ComputeSpanish(2, D->word, entry, canonical, sysflags))

{

// if canonical and we are different and canonical matches what we expected

if (!stricmp(canonical->word, RawCanonical(D)->word) && stricmp(canonical->word,D->word))

{

bool discard = false;

char* type = "";

// no other meaning to worry about with this conjugation and we already know the lemma for it and can detect it

if (D->properties & (VERB_PRESENT | VERB_PRESENT_3PS | VERB_PAST | VERB_PAST_PARTICIPLE | VERB_PRESENT_PARTICIPLE)

&& !(D->properties & (NOUN | ADVERB | ADJECTIVE | PREPOSITION | CONJUNCTION))) {

discard = true;

type = "verb";

}

else if (D->properties & NOUN_PLURAL && !(D->properties & (VERB | ADVERB | ADJECTIVE | PREPOSITION | CONJUNCTION)))

{

discard = true;

type = "noun";

}

if (discard)

{

FClose(out);

printf("discard %s %s\r\n", D->word,type);

return; // already handled

}

}

}

}

// write out the basics name ( meaningcount idiomcount)

fprintf(out, (char*)" %s ( ", D->word); // marker for termination of word and some other things

int count = GetMeaningCount(D);

if (count) fprintf(out, (char*)"meanings=%d ", count);

// check valid glosses (sometimes we have troubles that SHOULD have been found earlier)

int ngloss = 0;

if (!(D->internalBits & CONDITIONAL_IDIOM)) // glosses lose out to conditional idiom data

{

for (int i = 1; i <= count; ++i)

{

if (GetGloss(D, i)) ++ngloss;

}

if (ngloss) fprintf(out, (char*)"glosses=%d ", ngloss);

if (ngloss != GetGlossCount(D))

{

ReportBug((char*)"Bad gloss count for %s\r\n", D->word);

}

}

// now do the dictionary bits into english

char flags[MAX_WORD_SIZE];

WriteDictionaryFlags(D, flags);

if (*flags) fprintf(out, "%s", flags);

if (D->internalBits & CONDITIONAL_IDIOM)

fprintf(out, (char*)" CONDITIONAL_IDIOM poscondition=%s ", D->w.conditionalIdiom->word);

fprintf(out, (char*)"%s", (char*)") ");

// these must have valuable ->properties on them

WriteDictionaryReference((char*)"conjugate", GetTense(D), out);

WriteDictionaryReference((char*)"plural", GetPlural(D), out);

WriteDictionaryReference((char*)"comparative", GetComparison(D), out);

if (RawCanonical(D)) WriteDictionaryReference((char*)"lemma", RawCanonical(D), out);

// show the meanings, with illustrative gloss

fprintf(out, (char*)"\r\n");

// now dump the meanings and their glosses

for (int i = 1; i <= count; ++i)

{

MEANING M = GetMeaning(D, i);

fprintf(out, (char*)" %s ", WriteMeaning(M, true)); // words are small

if (M & SYNSET_MARKER) // facts for this will be OUR facts

{

M = MakeMeaning(D, i); // our meaning id

FACT* F = GetSubjectNondeadHead(D);

while (F)

{

if (M == F->subject && F->verb == Mis) // show up path as information only

{

fprintf(out, (char*)"(^%s) ", WriteMeaning(F->object)); // small word. shows an object that we link to up (parent)

break;

}

F = GetSubjectNondeadNext(F);

}

}

if (D->internalBits & CONDITIONAL_IDIOM) fprintf(out, (char*)"\r\n");

else

{

char* gloss = GetGloss(D, i);

if (gloss == NULL) gloss = "";

fprintf(out, (char*)"%s\r\n", gloss);

}

}

FClose(out);

}

void ExportCurrentDictionary()

{

if (!startOfLanguage)

{

printf("erase dict.bin first, then try again. and clear TOPIC/");

return;

}

ClearDictionaryFiles(true);

exportdictionary = true;

WORDP D = startOfLanguage - 1;

while (++D < endOfLanguage) WriteTextEntry(D, true);

exportdictionary = false;

}

void WriteDictionary(WORDP D,uint64 junk)

{

if (D->internalBits & DELETED_MARK) return;

if (*D->word == USERVAR_PREFIX && D->word[1]) return; // var never and money never, but let $ punctuation through

unsigned int internal = D->internalBits;

RemoveInternalFlag(D, UTF8 | UPPERCASE_HASH | DEFINES); // remove these

WriteTextEntry(D);

D->internalBits = internal;

}

void BuildDictionary(char* label)

{

xbuildDictionary = true;

int miniDict = 0;

char word[MAX_WORD_SIZE];

mini = current_language;

char* ptr = ReadCompiledWord(label, word);

bool makeBaseList = false;

if (!stricmp(word, (char*)"wordnet")) // the FULL wordnet dictionary w/o synset removal

{

miniDict = -1;

ReadCompiledWord(ptr, word);

mini = "WORDNET";

}

else if (!stricmp(word, (char*)"basic"))

{

miniDict = 1;

makeBaseList = true;

FClose(FopenUTF8Write((char*)"RAWDICT/basicwordlist.txt"));

maxHashBuckets = 10000;

setMaxHashBuckets = true;

mini = "BASIC";

}

else if (!stricmp(word, (char*)"layer0") || !stricmp(word, (char*)"layer1")) // a mini dictionary

{

miniDict = !stricmp(word, (char*)"layer0") ? 2 : 3;

mini = (miniDict == 2) ? (char*)"LAYER0" : (char*)"LAYER1";

maxHashBuckets = 10000;

setMaxHashBuckets = true;

}

else if (stricmp(current_language, (char*)"english")) // a foreign dictionary

{

miniDict = 6;

maxHashBuckets = 10000;

setMaxHashBuckets = true;

}

UseDictionaryFile(NULL);