// markSystem.cpp - annotates the dictionary with what words/concepts are active in the current sentence

#include "common.h"

#ifdef INFORMATION

For every word in a sentence, the word knows it can be found somewhere in the sentence, and there is a 64-bit field of where it can be found in that sentence.

The field is in a hashmap and NOT in the dictionary word, where it would take up excessive memory.

Adjectives occur before nouns EXCEPT:

1. object complement (with some special verbs)

2. adjective participle (sometimes before and sometimes after)

In a pattern, an author can request:

1. a simple word like bottle

2. a form of a simple word non - canonicalized like bottled or apostrophe bottle

3. a WordNet concept like bottle~1

4. a set like ~dead or : dead

For #1 "bottle", the system should chase all upward all sets of the word itself, and all

WordNet parents of the synset it belongs to and all sets those are in.

Marking should be done for the original and canonical forms of the word.

For #2 "bottled", the system should only chase the original form.

For #3 "bottle~1", this means all words BELOW this in the wordnet hierarchy not including the word

"bottle" itself.This, in turn, means all words below the particular synset head it corresponds to

and so instead becomes a reference to the synset head : (char*)"0173335n" or some such.

For #4 "~dead", this means all words encompassed by the set ~dead, not including the word ~dead.

So each word in an input sentence is scanned for marking.

the actual word gets to see what sets it is in directly.

Thereafter the system chases up the synset hierarchy fanning out to sets marked from synset nodes.

#endif

#define END_OF_REFERENCES 0xff

#define GENERIC_MEANING 0 // not a specific meaning of the word

int verbwordx = -1;

int uppercaseFind = -1; // unknown

static bool failFired = false;

bool trustpos = false;

int marklimit = 0;

static int freeTriedList = 0;

std::map <WORDP, int> triedData; // per volley index into heap space

static STACKREF wordlist = NULL;

static HEAPREF pendingConceptList = NULL;

static int MarkSetPath(int depth, int exactWord, MEANING M, int start, int end, unsigned int level, int kind); // walks set hierarchy

// mark debug tracing

bool showMark = false;

static unsigned int markLength = 0; // prevent long lines in mark listing trace

#define MARK_LINE_LIMIT 80

int upperCount, lowerCount;

ExternalTaggerFunction externalPostagger = NULL;

char unmarked[MAX_SENTENCE_LENGTH]; // can completely disable a word from mark recognition

/**************************************/

/* Word Reference in Sentence system */

/**************************************/

// Wheredata is a 64bit tried by meaning field (aligned)

// + Sentence references for the word.

// Each dictionary word can have up thru 63 different addressable meanings

// as well as the generic word as a whole. These are the "tried" bits, used

// when marking meanings to avoid redundant sweeps.

// Sentence references are where the word/concept is found in the sentence and

// is used by pattern matching.

// Each reference is 4 bytes

// byte0: start index into sentence

// byte1: end index into sentence

// byte2: if fundamental meaning this is where start of subject in sentence is (fundamental start/end is on the verb)

// byte3: which exact dictionary index (capitalization) matched if exact match is involved

// bytes4-7: int index of dictionary word

bool RemoveMatchValue(WORDP D, int position)

{

int access = GetAccess(D);

if (!access) return false;

bool changed = false;

unsigned char* data = (unsigned char*) (Index2Heap(access) + 8); // skip over 64bit tried by meaning field

unsigned char* tried = NULL;

bool didmod = false;

for (int i = 0; i < MAXREFSENTENCE_BYTES; i += REF_ELEMENT_SIZE)

{

if (data[i] == position)

{

didmod = true;

if (!changed)// protect by moving data to new area so restoresentence is safe

{

int newaccess = CopyWhereInSentence(access);

tried = (unsigned char*)(Index2Heap(newaccess) + 8);

changed = true;

SetTried(D, newaccess);

}

memmove(tried +i, tried +i+ REF_ELEMENT_SIZE,(MAXREFSENTENCE_BYTES - i - REF_ELEMENT_SIZE));

break;

}

}

return didmod;

}

static int WhereWordHitWithData(WORDP D, int start,unsigned char* data)

{

if (data) for (int i = 0; i < MAXREFSENTENCE_BYTES; i += REF_ELEMENT_SIZE)

{

if (data[i] >= start)

{

if (data[i] == start) return data[i + 1]; // return end of it

else break; // cannot match later - end of data will be END_OF_REFERENCES (0xff) for all

}

}

return 0;

}

static int WhereWordHit(WORDP D, int start)

{

unsigned char* data = GetWhereInSentence(D);

if (data) for (int i = 0; i < MAXREFSENTENCE_BYTES; i += REF_ELEMENT_SIZE)

{

if (data[i] >= start)

{

if (data[i] == start) return data[i + 1]; // return end of it

else break; // cannot match later

}

}

return 0;

}

bool HasMarks(int start)

{

WORDP D = FindWord(wordStarts[start]); // nominal word there

if (!D) return false;

unsigned char* data = GetWhereInSentence(D); // has 2 hidden int fields before this point

if (!data) return false;

int whereHitEnd = WhereWordHitWithData(D, start, data); // word in sentence index

return whereHitEnd != 0;

}

bool MarkWordHit(int depth, MEANING exactWord, WORDP D, int meaningIndex, int start, int end)

{ // keep closest to start at bottom, when run out, drop later ones

if (!D || !D->word) return false;

if (start > wordCount)

{

ReportBug((char*)"INFO: save position is too big")

return false;

}

if (end > wordCount) end = wordCount;

if (start == verbwordx && !stricmp(wordStarts[start], "verify"))

return false;

// been here before?

unsigned char* data = GetWhereInSentence(D); // has 2 hidden int fields before this point

if (!data) data = (unsigned char*)AllocateWhereInSentence(D);

if (!data) return false; // allocate failure

int whereHitEnd = WhereWordHitWithData(D, start,data); // word in sentence index

if (*D->word != '~') // real word, not a concept

{

uint64 meaningBit = 1ull << meaningIndex; // convert index into bit

if (whereHitEnd < end) SetTriedMeaningWithData(GENERIC_MEANING,(unsigned int*)data);

uint64 triedBits = GetTriedMeaning(D);

if (meaningBit & triedBits) return false; // did this meaning already

SetTriedMeaningWithData(triedBits | meaningBit, (unsigned int*)data); // update more

}

else if (whereHitEnd >= end) return false; // no need since already covering concept in this area

if (++marklimit > 5000)

{

if (!failFired) ReportBug("INFO: Mark limit hit")

failFired = true;

return false;

}

// diff < 0 means peering INSIDE a multiword token before last word

// we label END as the word before it (so we can still see next word) and START as the actual multiword token

bool added = false;

for (int i = 0; i < MAXREFSENTENCE_BYTES; i += REF_ELEMENT_SIZE)

{

unsigned char begin = data[i];

if (begin < start) continue; // skip over it

else if (begin == start) // we have already marked this somewhat

{

if (end > data[i+1]) added = true; // prefer the longer match

}

else // insert here

{

memmove(data+i+ REF_ELEMENT_SIZE,data+i,MAXREFSENTENCE_BYTES - i - REF_ELEMENT_SIZE); // create a hole for entry

data[i] = (unsigned char)start;

added = true;

}

if (added)

{

data[i + 1] = (unsigned char)(end & 0x00ff);

data[i + 2] = (unsigned char) (end >> 8); // for fundamental meanings this is the 3rd field reference

data[i + 3] = 0;

MEANING* exact = (MEANING*)(data + i + 4);

*exact = exactWord;

}

break; // have location

}

if (added)

{

if (*D->word == '~')// track the actual sets done matching start word location (good for verbs, not so good for nouns)

{

if (!(D->internalBits & TOPIC)) Add2ConceptTopicList(concepts, D, start, end, false); // DOESNT need to be be marked as concept

else Add2ConceptTopicList(topics, D, start, end, false);

}

if ((trace & (TRACE_PREPARE | TRACE_HIERARCHY) || prepareMode == PREPARE_MODE || showMark) && (D->word[0] != '~' || !IsDigit(D->word[1])))

{

markLength += D->length;

if (markLength > MARK_LINE_LIMIT)

{

markLength = 0;

Log(USERLOG,"\r\n");

Log(USERLOG,"");

}

while (depth-- >= 0) Log((showMark) ? ECHOUSERLOG : USERLOG, " ");

char which[20];

*which = 0;

which[1] = 0;

if (exactWord && D->internalBits & UPPERCASE_HASH) which[0] = '^';

Log((showMark) ? ECHOUSERLOG : USERLOG, (D->internalBits & TOPIC) ? "+T%s%s " : (char*)" +%s%s", D->word, which);

Log((showMark) ? ECHOUSERLOG : USERLOG," (%d-%d)\r\n", start, end);

markLength = 0;

}

}

return added;

}

unsigned int GetAccess(WORDP D)

{

std::map<WORDP, int>::iterator it;

it = triedData.find(D);

if (it == triedData.end()) return 0;

int access = it->second; // heap index

return access;

}

unsigned char* GetWhereInSentence(WORDP D) // [0] is the meanings bits, the rest are start/end/case bytes for 8 locations

{

if (!D) return NULL;

int access = GetAccess(D);

return (!access) ? NULL : (unsigned char*)Index2Heap(access) + 8; // skip over 64bit tried by meaning field

}

int CopyWhereInSentence(int oldindex)

{

unsigned int* olddata = (unsigned int*)Index2Heap(oldindex); // original location

if (!olddata) return 0;

size_t len = (sizeof(uint64) + MAXREFSENTENCE_BYTES + 3) / 4;

// 64bit tried by meaning field (aligned) + sentencerefs (2 bytes each + a byte for uppercase index)

unsigned int* data = (unsigned int*)AllocateHeap(NULL, len, 4, false); // 64 bits (2 words) + 48 bytes (12 words) = 14 words

if (data) memcpy((char*)data, olddata, len * 4);

return Heap2Index((char*)data);

}

void ClearWhereInSentence() // erases the WHEREINSENTENCE and the TRIEDBITS

{

memset(concepts, 0, sizeof(unsigned int) * MAX_SENTENCE_LENGTH);

memset(topics, 0, sizeof(unsigned int) * MAX_SENTENCE_LENGTH);

// be able to reuse memory

if (documentMode) for (std::map<WORDP, int>::iterator it = triedData.begin(); it != triedData.end(); ++it)

{

MEANING* data = (MEANING*)Index2Heap(it->second);

*data = freeTriedList;

freeTriedList = it->second;

}

triedData.clear();

memset(unmarked, 0, MAX_SENTENCE_LENGTH);

}

void ClearTriedData() // erases the WHEREINSENTENCE and the TRIEDBITS

{

triedData.clear();

freeTriedList = 0;

}

unsigned int* AllocateWhereInSentence(WORDP D)

{

if (documentMode && freeTriedList) // reuse memory

{

MEANING* d = (MEANING*)Index2Heap(freeTriedList);

freeTriedList = *d;

}

size_t len = (sizeof(uint64) + MAXREFSENTENCE_BYTES + 3) / 4;

// 64bit tried by meaning field (aligned) + sentencerefs (3 bytes each + a byte for uppercase index)

unsigned int* data = (unsigned int*)AllocateHeap(NULL, len, 4, false); // 64 bits (2 words) + 48 bytes (12 words) = 14 words

if (!data) return NULL;

memset((char*)data, END_OF_REFERENCES, len * 4); // clears sentence xref start/end bits and casing byte

data[0] = 0; // clears the tried meanings list

data[1] = 0;

// store where in the temps data

int index = Heap2Index((char*)data); // original index!

triedData[D] = index;

return data + 2; // analogous to GetWhereInSentence

}

void SetTriedMeaningWithData(uint64 bits, unsigned int* data)

{

*(data - 2) = (unsigned int)(bits >> 32);

*(data - 1) = (unsigned int)(bits & 0xffffffff); // back up to the tried meaning area

}

void SetTriedMeaning(WORDP D, uint64 bits)

{

unsigned int* data = (unsigned int*)GetWhereInSentence(D);

if (!data)

{

data = AllocateWhereInSentence(D); // returns past the tried bits of chunk

if (!data) return; // failed to allocate

}

*(data - 2) = (unsigned int)(bits >> 32);

*(data - 1) = (unsigned int)(bits & 0xffffffff); // back up to the tried meaning area

}

uint64 GetTriedMeaning(WORDP D) // which meanings have been used (up to 64)

{

std::map<WORDP, int>::iterator it;

it = triedData.find(D);

if (it == triedData.end()) return 0;

unsigned int* data = (unsigned int*)Index2Heap(it->second); // original location

if (!data) return 0;

uint64 value = ((uint64)(data[0])) << 32;

value |= (uint64)data[1];

return value; // back up to the correct meaning zone

}

unsigned int GetIthSpot(WORDP D, int i, int& start, int& end)

{

if (!D) return 0; // not in sentence

unsigned char* data = GetWhereInSentence(D);

if (!data) return 0;

i *= REF_ELEMENT_SIZE;

if (i >= MAXREFSENTENCE_BYTES) return 0; // at end

start = (unsigned char) data[i];

if (start == END_OF_REFERENCES) return 0;

end = (unsigned char) data[i + 1];

if (end > wordCount)

{

static bool did = false;

if (!did) ReportBug((char*)"INFO: Getith out of range %s at %d\r\n", D->word, volleyCount);

did = true;

}

if (data[i + 2]) end |= data[i + 2]; // fundamental Meaning extra value

return start;

}

static unsigned char* DataIntersect(WORDP D)

{

char* ptr = D->word;

char* at = strchr(ptr + 1, '~'); // joiner of disparate concepts, like ~pet~tasty

unsigned char* data = NULL;

// dont do word~1~concept or word~n~concept

if (at && at[2]) // word with ~casemarking data added, has no data on its own, not trial~n or trial~1

{

WORDP first = FindWord(ptr, (at - ptr)); // the first piece

data = GetWhereInSentence(first);

if (!data) return 0;

size_t len = strlen(at);

char* at1 = strchr(at + 1, '~'); // and a 3rd piece

if (at1) len = at1 - at;

WORDP second = FindWord(at, len); // the 2nd piece

unsigned char* seconddata = GetWhereInSentence(second);

if (!seconddata) return 0; // word not found so conjoin cant either

unsigned char* thirddata = NULL;

if (at1)

{

WORDP third = FindWord(at1);

thirddata = GetWhereInSentence(third);

if (!thirddata) return 0; // not there

}

unsigned char* commonData = (unsigned char*)AllocateWhereInSentence(D);

if (!commonData) return 0; // allocate failure

memcpy(commonData, data, (sizeof(uint64) + MAXREFSENTENCE_BYTES + 3) / 4); // starts with the base

// keep common positions of this second word (and optionally third) with existing first

for (int i = 0; i < MAXREFSENTENCE_BYTES; i += REF_ELEMENT_SIZE) // walk commondata

{

if (commonData[i] == END_OF_REFERENCES) break; // no more data in base

bool found1 = false;

bool found2 = false;

for (int j = 0; j < MAXREFSENTENCE_BYTES; j += REF_ELEMENT_SIZE)

{

if (seconddata[j] == END_OF_REFERENCES) break; // end of this piece

if (seconddata[j] == commonData[i]) // found here

{

found1 = true;

break;

}

}

if (thirddata) for (int j = 0; j < MAXREFSENTENCE_BYTES; j += REF_ELEMENT_SIZE)

{

if (thirddata[j] == END_OF_REFERENCES) break; // end of this piece

if (thirddata[j] == commonData[i]) // found here

{

found2 = true;

break;

}

}

else found2 = true; // dont need a third

if (!found1 || !found2) // our common data is not common to all

{

memmove(commonData + i, commonData + i + REF_ELEMENT_SIZE, (MAXREFSENTENCE_BYTES - i - REF_ELEMENT_SIZE));

i -= REF_ELEMENT_SIZE;

}

}

if (commonData[0] == END_OF_REFERENCES) return 0; // nothing in common

data = commonData; // the common data of word and concept

}

return data;

}

unsigned int GetNextSpot(WORDP D, int start, int &startPosition, int& endPosition, bool reverse, int legalgap)

{// spot can be 1-31, range can be 0-7 -- 7 means its a string, set last marker back before start so can rescan

// BUG - we should note if match is literal or canonical, so can handle that easily during match eg

// '~shapes matches square but not squares (whereas currently literal fails because it is not ~shapes)

if (!D) return 0; // not in sentence

unsigned char* data = GetWhereInSentence(D);

char* ptr = D->word;

if (!data && D->internalBits & HAS_CASEMARKING) data = DataIntersect(D); // word with ~casemarking data added, has no data on its own, not trial~n or trial~1 - or concept intersect: ~pet~tasty

if (!data) return 0;

// now perform the real analysis

uppercaseFind = -1;

int i;

startPosition = 0;

for (i = 0; i < MAXREFSENTENCE_BYTES; i += REF_ELEMENT_SIZE) // each 8 byte ref is start,end,extra,unused, 4byte exact,

{

unsigned char at = data[i];

if (at == END_OF_REFERENCES) // end of unit marker

{

if (!reverse) return 0;

else break; // cannot back further

}

unsigned char end = data[i + 1];

unsigned char fundamentalExtra = data[i + 2]; // usually 0, except for fundamental meaning matches

MEANING* exact = (MEANING*)(data + i + 4);

if (unmarked[at]) { ; }

else if (reverse)

{

if (at < start) // valid. but starts far from where we are

{

startPosition = at; // bug fix backward gaps as well

endPosition = end | (fundamentalExtra << 8); // hidden subject data for fundamental meanings

uppercaseFind = (int)*exact;

continue; // find the CLOSEST without going over

}

else if (at >= start) break; // getting worse

}

else if (at > start) // scanning forward

{

if (legalgap && (at - start) > legalgap) startPosition = 0; // too far away and optional

else

{

startPosition = at;

endPosition = end | (fundamentalExtra << 8); // hidden subject data for fundamental meanings

uppercaseFind = (int)*exact;

}

break;

}

}

return startPosition; // we have a closest or we dont

}

/**************************************/

/* End of Word Reference in Sentence system */

/**************************************/

static void TraceHierarchy(FACT* F,char* msg)

{

if (TraceHierarchyTest(trace))

{

char word[MAX_WORD_SIZE];

char* fact = WriteFact(F, false, word); // just so we can see it

unsigned int hold = globalDepth;

globalDepth = 4 + 1;

if (!msg) Log(USERLOG,"%s\r\n", fact); // \r\n

else Log(USERLOG,"%s (%s)\r\n", fact,msg); // \r\n

globalDepth = hold;

}

}

static void AddPendingConcept(FACT* F, unsigned int start, unsigned int end)

{

pendingConceptList = AllocateHeapval(pendingConceptList, (uint64)F, start, end);

TraceHierarchy(F,"delayed");

}

static bool ProcessPendingFact(FACT* F, int start, int end)

{

WORDP O = Meaning2Word(F->object);

if (WhereWordHit(O, start) >= (int)end) return true; // already marked this set

// FACT wanted to map subject to concept object, but it had an exclude on a set that needed completion

int depth = 4;

WORDP S = Meaning2Word(F->subject);

int startPosition, endPosition;

if (GetNextSpot(S, start - 1, startPosition, endPosition) && startPosition == start && endPosition == end)

{

TraceHierarchy(F,"");

return true; // not allowed to proceed

}

return false; // unmarked. we dont know

}

static void ProcessPendingConcepts()

{

// (~setmember exclude ~set)

// has subject been marked at this position, if so, we cannot trigger concept for this position

if (!pendingConceptList) return;

HEAPREF startList = pendingConceptList; // F start|end

HEAPREF begin = startList;

bool changed = false;

while (1)

{

if (!startList) // we will be cycling list trying to get changes until no list or no changes

{

if (!changed) break; // no improvement

startList = begin;

changed = false;

}

uint64 start;

uint64 end;

uint64* currentEntry = (uint64*) startList;

uint64 Fx;

startList = UnpackHeapval(startList, Fx, start,end);

FACT* F = (FACT*)Fx;

WORDP concept = NULL; // set we want to trigger

while (F) // will be NULL if we have already finished with it

{

// expect exit here because writing a concept with NO members is nuts

if (F->verb != Mexclude) break; // ran out of set restrictions

// before we can trigger this set membership

concept = Meaning2Word(F->object);

if (ProcessPendingFact(F, (int)start, (int)end)) // failed

{

currentEntry[1] = 0; // kill fact use

changed = true;

}

F = GetObjectNondeadNext(F);

}

// now flow path of this set upwards since all excludes have been considered

if (F && F->verb != Mexclude)

{

TraceHierarchy(F,"resume");

if (MarkWordHit(4, EXACTNOTSET, concept, 0, (int)start, (int)end)) // new ref added

{

if (MarkSetPath(4 + 1, EXACTNOTSET, F->object, (int)start, (int)end, 4 + 1, FIXED) != -1) changed = true; // someone marked

}

}

}

// activate anything not already deactivated now

while (pendingConceptList) // one entry per pending set that had excludes

{

bool exact = false;

bool canonical = false;

uint64 Fx;

uint64 start;

uint64 end;

pendingConceptList = UnpackHeapval(pendingConceptList, Fx,start,end);

if (!Fx) continue;

FACT* F = (FACT*)Fx;

WORDP E = (F) ? Meaning2Word(F->object) : NULL;

// mark all members of the link

TraceHierarchy(F,"defaulting");

if (MarkWordHit(4, EXACTNOTSET, E, 0, (int)start, (int)end)) // new ref added

{

MarkSetPath(4 + 1, EXACTNOTSET, F->object, (int)start, (int)end, 4 + 1, FIXED);

}

}

}

static bool IsValidStart(int start)

{

if (start == 1) return true;

if (!(tokenControl & DO_INTERJECTION_SPLITTING))

{

// if not splitting interjections into their own sentence, then could be a valid start

// if all previous words are an interjection

WORDP D = FindWord("~interjections");

if (!D) return false;

int i = 0;

while (++i<start)

{

if (*wordStarts[i] == ',') { ; }

else if (*wordStarts[i] == '-') { ; }

else

{

int end = WhereWordHit(D,i);

if (end == 0 || end >= start) return false;

i = end;

}

}

return true;

}

return false;

}

MEANING EncodeConceptMember(char* word,int& flags)

{

char hold[MAX_WORD_SIZE];

if (*word == '~') MakeLowerCopy(hold, word); // require concept name be lower case

else strcpy(hold, JoinWords(BurstWord(word, CONTRACTIONS)));

char* at = hold;

while ((at = strchr(at, '_'))) *at = ' '; // change _ to spaces

at = hold;

if (*word == '\'' && word[1] == '\'') // 2 quotes mean case sensitive user typing

{

flags |= RAWCASE_ONLY;

at += 2;

}

else if (*word == '\'') // 1 quote means not canonical (allowing FIXED or RAW)

{

flags |= ORIGINAL_ONLY;

++at;

}

return MakeMeaning(StoreWord(at,AS_IS));

}

static int MarkSetPath(int depth,int exactWord,MEANING M, int start, int end, unsigned int level, int kind) // walks set hierarchy

{// travels up concept/class sets only, though might start out on a synset node or a regular word

unsigned int flags = GETTYPERESTRICTION(M);

if (!flags) flags = BASIC_POS; // what POS we allow from Meaning

WORDP D = Meaning2Word(M);

int index = Meaning2Index(M); // always 0 for a synset or set

// check for any repeated accesses of this synset or set or word

uint64 offset = 1ull << index;

int result = NOPROBLEM_BIT;

FACT* H = GetSubjectNondeadHead(D); // thisword/concept member y

while (H)

{

FACT* F = H;

H = GetSubjectNondeadNext(H);

if (F->verb != Mmember) continue;

// ~concept members and word equivalent

if (trace & TRACE_HIERARCHY) TraceHierarchy(F,"");

WORDP concept = Meaning2Word(F->object);

if (concept->internalBits & OVERRIDE_CONCEPT) // override by ^testpattern, is this legal fact?

{

if (!(F->flags & OVERRIDE_MEMBER_FACT)) continue; // pretend he doesnt exist

}

// if subject has type restriction, it must pass

unsigned int restrict = GETTYPERESTRICTION(F->subject );

if (!restrict && index) restrict = GETTYPERESTRICTION(GetMeaning(D,index)); // new (may be unneeded)

// reasons we cant use this fact

// for true interjection, END_ONLY can mean wordCount or next word is , or -

bool block = false;

if (kind != RAWCASE && F->flags & RAWCASE_ONLY) { block = true; } // incoming is not raw correctly cased words and must be

else if (kind == CANONICAL && F->flags & ORIGINAL_ONLY) { block = true; } // incoming is not original words and must be

else if (restrict && !(restrict & flags)) { block = true; } // type restriction in effect for this concept member

else if (F->flags & (START_ONLY | END_ONLY))

{

if (F->flags & START_ONLY && !IsValidStart(start)) { block = true; } // must begin the sentence

else if (F->flags & END_ONLY && end != wordCount && !(F->flags & START_ONLY)) { block = true; } // must begin the sentence

else if ((F->flags & (START_ONLY | END_ONLY)) == (START_ONLY | END_ONLY) && IsValidStart(start))

{

if (end == wordCount) { ; }

else if (*wordStarts[end + 1] == ',') { ; }

else if (*wordStarts[end + 1] == '-') { ; }

else block = true;

}

}

int mindex = Meaning2Index(F->subject);

// index meaning restriction (0 means all)

if (!block && index == mindex) // match generic or exact subject

{

bool mark = true;

// test for word not included in set

if (index)

{

unsigned int pos = GETTYPERESTRICTION(GetMeaning(Meaning2Word(F->subject), index));

if (!(flags & pos))

mark = false; // we cannot be that meaning because type is wrong

}

if (!mark)

{

if (trace & TRACE_HIERARCHY) TraceHierarchy(F, "");

}

else if (*concept->word == '~' && WhereWordHit(concept, start) >= end) mark = false; // already marked this set

else //does set has some members it does not want

{

FACT* G = GetObjectNondeadHead(concept);

while (G)

{

// all simple excludes will be first

// all set excludes will be second

// actual values of set will be third

// User can defeat that by runtime addition of set members. Too bad for now.

// so technically we could free up the HAS_EXCLUDE bit

if (G->verb == Mexclude) // see if this is marked for this position, if so, DONT trigger topic

{

WORDP S = Meaning2Word(G->subject); // what to exclude

int startPosition,endPosition;

// need to test for original only as well - BUG

if (GetNextSpot(S,start-1,startPosition,endPosition) && startPosition == start && endPosition == end)

{

if (trace & TRACE_HIERARCHY) TraceHierarchy(F,"");

mark = false;

break;

}

// if we see a concept exclude, regular ones already passed

if (*S->word == '~') // concept exclusion - status unknown

{

AddPendingConcept(G, start, end); // must review later

mark = false; // dont mark now

break; // revisit all exclusions later

}

}

else if (G->verb == Mmember) // ran out of excludes

{

mark = true;

break;

}

G = GetObjectNondeadNext(G);

}

}

if (mark)

{

if (MarkWordHit(depth, exactWord, concept, index,start, end)) // new ref added

{

if (MarkSetPath(depth+1, exactWord, F->object, start, end, level + 1, kind) != -1) result = 1; // someone marked

}

}

}

else if (!index && mindex) // we are all meanings (limited by pos use) and he is a specific meaning

{

WORDP J = Meaning2Word(F->subject);

MEANING M1 = GetMeaning(J, mindex);

unsigned int pos = GETTYPERESTRICTION(M1);

if (flags & pos) // && start == end wont work if spanning multiple words revised due to "to fish" noun infinitive

{

if (MarkWordHit(depth, exactWord, Meaning2Word(F->object), Meaning2Index(F->object),start, end)) // new ref added

{

if (MarkSetPath(depth+1, exactWord, F->object, start, end, level + 1, kind) != -1) result = 1; // someone marked

}

}

}

}

return result;

}

static void RiseUp(int depth, int exactWord,MEANING M,unsigned int start, unsigned int end,unsigned int level,int kind) // walk wordnet hierarchy above a synset node

{ // M is always a synset head

M &= -1 ^ SYNSET_MARKER;

unsigned int index = Meaning2Index(M);

WORDP D = Meaning2Word(M);

char word[MAX_WORD_SIZE];

sprintf(word,(char*)"%s~%u",D->word,index); // some meaning is directly referenced?

MarkWordHit(depth, exactWord, FindWord(word),0,start,end); // direct reference in a pattern

// now spread and rise up

if (MarkSetPath(depth, exactWord,M,start,end,level,kind) == -1) return; // did the path already

FACT* F = GetSubjectNondeadHead(D);

while (F)

{

if (F->verb == Mis && (index == 0 || F->subject == M)) RiseUp(depth+1,exactWord,F->object,start,end,level+1,kind); // allowed up

F = GetSubjectNondeadNext(F);

}

}

static void MarkAllMeaningAndImplications(int depth, MEANING M, int start, int end, int kind, bool sequence,bool once)

{ // M is always a word or sequence from a sentence

// but if uppercase, there may be multiple forms, so handle all

if (!M) return;

WORDP D = Meaning2Word(M);

unsigned int len = D->length;

unsigned int hash = (D->hash % maxHashBuckets); // mod by the size of the table

int uindex = 0; // lowercase bucket

WORDP X = dictionaryBase + hashbuckets[hash + 1]; // look in uppercase bucket for this word

while (X != dictionaryBase) // all entries matching in upper case bucket

{

if (D->hash == X->hash && X->length == len && !StricmpUTF(D->word, X->word, len))

{

MEANING M1 = M;

MEANING windex = MakeMeaning(X);

if ((M & MEANING_BASE) != windex) // alternate spelling to what was passed in

{

M1 = windex | (M & TYPE_RESTRICTION); // no idea what meaning, go generic

}

MarkMeaningAndImplications(depth, windex, M1, start, end, kind, sequence, once);

}

X = dictionaryBase + GETNEXTNODE(X);

}

}

void MarkMeaningAndImplications(int depth, MEANING exactWord,MEANING M,int start, int end,int kind,bool sequence,bool once)

{ // M is always a word or sequence from a sentence

if (!M) return;

WORDP D = Meaning2Word(M);

if (D->properties & NOUN_TITLE_OF_WORK && kind == CANONICAL) return; // accidental canonical match of a title. not intended

// We want to avoid wandering fact relationships for meanings we have already scanned.

if (!exactWord)

{

if (D->internalBits & UPPERCASE_HASH) MarkAllMeaningAndImplications(depth, M, start, end,kind,sequence, once);

else if (kind == CANONICAL) exactWord = (MEANING)DONTUSEEXACT; // dont use concept word or lowercase word as the match

}

// We mark words/phrases and concepts and words/concepts implied by them.

// We mark words by meaning (63) + generic. They always have a fixed size match.

// We mark concepts by size match at a start position. You might match 1 word or several in a row.

// For match variable retrieval we want the longest match at a position.

// Because we can come in here with a general word with and without a type restriction,

// we have scan out from the word because we cant mark the different ways we scanned before,

int index = Meaning2Index(M);

int whereHit = WhereWordHit(D, start);

unsigned int restrict = GETTYPERESTRICTION(M);

unsigned int size = GetMeaningCount(D);

if (size == 0)

{

M = MakeMeaning(D); // remove restriction

restrict = 0;

}

if (*D->word == '~')

{

if (whereHit >= end) return; // already have best concept storage

}

else

{

if (whereHit < end) SetTriedMeaning(D, 0); // found nothing at this index, insure nothing to start

}

// we mark word hit before using MarkSetPath, so that exclude is supported

// words we dont know we dont bother marking

if (!once || D->properties & (PART_OF_SPEECH | NOUN_TITLE_OF_WORK | NOUN_HUMAN) || D->systemFlags & PATTERN_WORD || D->internalBits & CONCEPT)

{

MarkWordHit(depth, exactWord, D, 0, start, end);

MarkSetPath(depth + 2, exactWord, M, start, end, 0, kind); // generic membership of this word all the way to top

}

// we dont mark random junk discovered, only significant sequences

else if (sequence) // phrase is not a pattern word, maybe it goes to some concepts

{

FACT* F = GetSubjectNondeadHead(D);

while (F)

{

// mark sequence if someone cared about it

if (ValidMemberFact(F)) // ~concept members and word equivalent

{

MarkWordHit(depth, exactWord, D, 0, start, end); // we found something to relate to, so mark us

MarkSetPath(depth + 2, exactWord, M, start, end, 0, kind); // generic membership of this word all the way to top

break;

}

F = GetSubjectNext(F);

}

}

// check for POS restricted forms of this word

char word[MAX_WORD_SIZE];

if (*D->word != '~' && !once) // words, not concepts

{

if (restrict & NOUN && !(posValues[start] & NOUN_INFINITIVE)) // BUG- this wont work up the ontology, only at the root of what the script requests - doesnt accept "I like to *fish" as a noun, so wont refer to the animal

{

sprintf(word, (char*)"%s~n", D->word);

MarkWordHit(depth, exactWord, FindWord(word, 0, PRIMARY_CASE_ALLOWED), 0, start, end); // direct reference in a pattern

}

if ((restrict & VERB) || posValues[start] & NOUN_INFINITIVE)// accepts "I like to *swim as not a verb meaning"

{

sprintf(word, (char*)"%s~v", D->word);

MarkWordHit(depth, exactWord, FindWord(word, 0, PRIMARY_CASE_ALLOWED), 0, start, end); // direct reference in a pattern

}

if (restrict & ADJECTIVE) // and adverb

{

sprintf(word, (char*)"%s~a", D->word);

MarkWordHit(depth, exactWord, FindWord(word, 0, PRIMARY_CASE_ALLOWED), 0, start, end); // direct reference in a pattern

}

}

// now follow out the allowed synset hierarchies

if (!restrict) restrict = ESSENTIAL_FLAGS & finalPosValues[end]; // unmarked ptrs can rise all branches compatible with final values - end of a multiword (idiom or to-infintiive) is always the posvalued one

if (!once && *D->word != '~') for (unsigned int k = 1; k <= size; ++k)

{

M = GetMeaning(D,k); // it is a flagged meaning unless it self points

if (!(GETTYPERESTRICTION(M) & restrict)) continue; // cannot match type

// walk the synset words and see if any want vague concept matching like dog~~

MEANING T = M; // starts with basic meaning

unsigned int n = (index && (int)k != index) ? (unsigned int)80 : (unsigned int) 0; // only on this meaning or all synset meanings

while (n < 50) // insure not infinite loop

{

WORDP X = Meaning2Word(T);

unsigned int ind = Meaning2Index(T);

sprintf(word,(char*)"%s~~",X->word);

MarkWordHit(depth, exactWord, FindWord(word, 0, PRIMARY_CASE_ALLOWED),0,start,end); // direct reference in a pattern

if (!ind) break; // has no meaning index

MEANING* meanings = GetMeanings(X);

if (!meanings) break;

T = meanings[ind];

if (!T) break;

if ((T & MEANING_BASE) == (M & MEANING_BASE)) break; // end of loop

++n;

}

M = (M & SYNSET_MARKER) ? MakeMeaning(D,k) : GetMaster(M); // we are the master itself or we go get the master

RiseUp(depth+1, exactWord,M,start,end,0,kind); // allowed meaning pos (self ptrs need not rise up)

}

}

static void HuntMatch(int kind, char* word,bool strict,int start, int end, unsigned int& usetrace)

{

WORDP set[20];

WORDP D;

int oldtrace = trace;

// if user typed upper case specifically, trust him

if (start == end && start != 1 && IsUpperCase(word[0]))

{

if (!IsUpperCase(word[1])) strict = true;

}

int i = GetWords(word,set,strict); // words in any case and with mixed underscore and spaces

while (i)

{

D = set[--i];

// dont redo effort

if (D->internalBits & BEEN_HERE) continue; // huntmatch already covered this

D->internalBits |= BEEN_HERE;

wordlist = AllocateStackval(wordlist, (uint64) D,0);

// markallimplied matches words with part of speech. This doesnt

if (!(D->systemFlags & PATTERN_WORD) && !(D->properties & PART_OF_SPEECH) && !(D->internalBits & UPPERCASE_HASH)) // given no flag reason to use, see if concept member

{

FACT* F = GetSubjectHead(D); // is it a part of some concept? Or a direct wor

while (F)

{

if (ValidMemberFact(F)) break; // legal fact for us

F = GetSubjectNext(F);

}

if (!F) continue;

}

// not allowed to detect uppercase when user input

// is form is conjugated (fitted != Fit) except

// for Plural noun OR monitoring => Monitor as canonical

if (start == end && IsUpperCase(*D->word) && kind == FIXED){ ; }

else if (start == end && IsUpperCase(*D->word) && kind == CANONICAL &&

stricmp(D->word, wordStarts[start])) // user didnt type this, upper case should be noun singular -- bug for us if he did proper noun plural

{

continue; // avoiding canonical Monitor of monitoring

}

trace = (D->subjectHead || D->systemFlags & PATTERN_WORD || D->properties & PART_OF_SPEECH) ? usetrace : 0; // being a subject head means belongs to some set. being a marked word means used as a keyword

if ((*D->word == 'I' || *D->word == 'i') && !D->word[1]) continue; // dont follow out this I or i word

int refined = kind;

if (kind == RAW) // promote to rawcase?

{

size_t len = strlen(D->word);

if (!strncmp(D->word, word, len - 1)) // except for 1st letter is rest of word what user types

{

if (start != 1) refined = RAWCASE; // only first word of sentence may be provoked

else if (!IsUpperCase(*word)) refined = RAWCASE; // lower case by user not impacted by grammar stricture

else if (IsUpperCase(word[1])) refined = RAWCASE; // 2 uppercase in a row is intentional (or caps lock so we cant tell)

}

}

MarkMeaningAndImplications(0, 0,MakeMeaning(D),start,end, refined,true);

}

trace = (modifiedTrace) ? modifiedTraceVal : oldtrace;

}

static void SetSequenceStamp() // mark words in sequence, original and canonical (but not mixed) - detects proper name potential up to 5 words - and does discontiguous phrasal verbs

{// words are always fully generic, never restricted by meaning or postag

// these use underscores

char* rawbuffer = AllocateStack(NULL, maxBufferSize); // includes typos

char* fixedbuffer = AllocateStack(NULL, maxBufferSize);

char* canonbuffer = AllocateStack(NULL, maxBufferSize);

wordlist = NULL;

char* limit = GetUserVariable("$cs_sequence", false, true);

int sequenceLimit = (*limit) ? atoi(limit) : SEQUENCE_LIMIT;

if (parseLimited && sequenceLimit > 2) sequenceLimit = 2;

unsigned int oldtrace = trace;

unsigned int usetrace = trace;

if (trace & (TRACE_HIERARCHY | TRACE_PREPARE) || prepareMode == PREPARE_MODE)

{

Log(USERLOG,"\r\nSequences:\r\n");

usetrace = (unsigned int) -1;

if (oldtrace && !(oldtrace & TRACE_ECHO)) usetrace ^= TRACE_ECHO;

}

// consider all sets of up to 5-in-a-row

if (endSentence > wordCount)

endSentence = wordCount;

for (int i = startSentence; i <= (int)endSentence; ++i)