#include "common.h"

#ifdef INFORMATION

Json units(JSON arraysand objects) consist of a dictionary entry labelled ja - ... or jo - ...

Facts are stored on these.When initially created as a permanent unit, there is a special

fact to represent the empty value so the dictionary entry will not disappear across volleys.

Otherwise it would be destroyedand a new unit arbitrarily created later that has no connection

to the one the user just created.

When you add an item to an array via JsonArrayInsert, this empty fact is destroyed.And when the

last array value is destroyed via JsonArrayDelete, the marker fact is recreated.

When you add an item to an object via JsonObjectInsert or by direct assignment, the marker fact

is destroyed.When you use Assign of null to a json object, if that was the last field, then the

marker fact is recreated.

The marker fact is normally bypassed by all accesses to the items of a json object(so it does not

query or count or whatever) except when specially accessed by the above routines.

#endif

// GENERAL JSON SUPPORT

#include "jsmn.h"

static bool curl_done_init = false;

static int jsonCreateFlags = 0;

#define MAX_JSON_LABEL 50

static char jsonLabel[MAX_JSON_LABEL + 1];

bool safeJsonParse = false;

int jsonIdIncrement = 1;

int jsonDefaults = 0;

int json_open_counter = 0;

uint64 json_open_time = 0;

int jsonStore = 0; // where to put json fact refs

int jsonIndex;

int jsonOpenSize = 0;

static unsigned int jsonPermanent = FACTTRANSIENT;

bool jsonNoArrayduplicate = false;

bool jsonObjectDuplicate = false;

bool jsonDontKill = false;

bool directJsonText = false;

static char* curlBufferBase = NULL;

static int objectcnt = 0;

static FunctionResult JSONpath(char* buffer, char* path, char* jsonstructure, bool raw, bool nofail);

static MEANING jcopy(MEANING M);

static MEANING MergeArray(MEANING ar1, MEANING ar2);

typedef enum {

URL_SCHEME = 0,

URL_AUTHORITY,

URL_PATH,

URL_QUERY,

URL_FRAGMENT

} urlSegment;

void InitJson()

{

json_open_counter = 0;

json_open_time = 0;

jsonDefaults = 0;

}

static int JSONArgs()

{

int index = 1;

directJsonText = false;

bool used = false;

jsonCreateFlags = 0;

jsonPermanent = FACTTRANSIENT; // default

jsonNoArrayduplicate = false;

jsonObjectDuplicate = false;

if (jsonDefaults & JSON_ARRAY_UNIQUE) jsonNoArrayduplicate = true;

if (jsonDefaults & JSON_OBJECT_DUPLICATE) jsonObjectDuplicate = true;

char* arg1 = ARGUMENT(1);

if (*arg1 == '"') // remove quotes

{

++arg1;

size_t len = strlen(arg1);

if (arg1[len - 1] == '"') arg1[len - 1] = 0;

}

char word[MAX_WORD_SIZE];

while (*arg1)

{

arg1 = ReadCompiledWord(arg1, word);

if (!stricmp(word, (char*)"permanent"))

{

jsonPermanent = 0;

used = true;

}

else if (!stricmp(word, (char*)"boot")) // build to migrate to system boot layer

{

jsonPermanent = FACTBOOT;

used = true;

}

else if (!stricmp(word, (char*)"USER_FLAG3"))

{

jsonCreateFlags |= USER_FLAG3;

used = true;

}

else if (!stricmp(word, (char*)"USER_FLAG2"))

{

jsonCreateFlags |= USER_FLAG2;

used = true;

}

else if (!stricmp(word, (char*)"USER_FLAG1"))

{

jsonCreateFlags |= USER_FLAG1;

used = true;

}

else if (!stricmp(word, (char*)"autodelete"))

{

jsonCreateFlags |= FACTAUTODELETE;

used = true;

}

else if (!stricmp(word, (char*)"unique"))

{

jsonNoArrayduplicate = true;

used = true;

}

else if (!stricmp(word, (char*)"duplicate"))

{

jsonObjectDuplicate = true;

used = true;

}

else if (!stricmp(word, (char*)"transient")) used = true;

else if (!stricmp(word, (char*)"direct")) used = directJsonText = true; // used by jsonopen

else if (!stricmp(word, (char*)"safe")) safeJsonParse = used = true;

if (!used) break; // must find at start

}

if (used) ++index;

return index;

}

void InitJSONNames()

{

objectcnt = 0; // also for json arrays

jsonIdIncrement = 1;

}

MEANING GetUniqueJsonComposite(char* prefix, unsigned int permanent)

{

char namebuff[MAX_WORD_SIZE];

char* permanence = "";

if (permanent == FACTTRANSIENT) permanence = "t";

else if (jsonPermanent == FACTTRANSIENT) permanence = "t";

else if (jsonPermanent == FACTBOOT) permanence = "b";

while (1)

{

sprintf(namebuff, "%s%s%s%d", prefix, permanence, jsonLabel, objectcnt);

objectcnt += jsonIdIncrement;

WORDP D = FindWord(namebuff);

if (!D) break;

}

jsonIdIncrement = 1; // return to linear hunting of open slots from here

return MakeMeaning(StoreWord(namebuff, AS_IS));

}

bool IsValidJSONName(char* word, char type)

{

size_t n = strlen(word);

if (n < 4) return false;

// must at least start with the right prefix

if (word[0] != 'j' || word[2] != '-') return false;

if (type)

{

if (word[1] != type) return false;

}

else if (word[1] != 'a' && word[1] != 'o') return false;

// the label cannot contain certain characters

if (strchr(word, ' ') ||

strchr(word, '\\') || strchr(word, '"') || strchr(word, '\'') || strchr(word, 0x7f) || strchr(word, '\n') // refuse illegal content

|| strchr(word, '\r') || strchr(word, '\t')) return false; // must be legal unescaped json content and safe CS content

// jo-anne.van.der.ende@signify.com is not a valid JSON name

// last part must be numbers

char* end = word + n - 1;

char* at = end;

while (IsDigit(*at)) --at;

if (at == end) return false;

long len = at - word - 3;

if (word[3] == 't' || word[3] == 'b') --len;

if (len > MAX_JSON_LABEL) return false;

return true;

}

static char* IsJsonNumber(char* str)

{

if (IsDigit(*str) || (*str == '-' && IsDigit(str[1]))) // +number is illegal in json

{

// validate the number

char* at = str;

if (*at != '-') --at;

bool periodseen = false;

bool exponentseen = false;

while (*++at)

{

if (*at == '.' && !periodseen && !exponentseen) periodseen = true;

else if ((*at == 'e' || *at == 'E') && !exponentseen)

{

if (IsSign(at[1])) ++at;

exponentseen = true;

}

else if (*at == ' ' || *at == ',' || *at == '}' || *at == ']') return at;

else if (!IsDigit(*at)) return NULL; // cannot be number

}

return at;

}

return NULL;

}

static bool ConvertUnicode(char* from) // convert \uxxxx to utf8 and escaped characters to normal, leaving utf8 alone

{

char* at = from;

bool converted = false;

while ((at = strchr(at, '\\')))

{

char* base = at++; // base is start of escaped string

if (*at != 'u' || !IsHexDigit(at[1]) || !IsHexDigit(at[2]) || !IsHexDigit(at[3]) || !IsHexDigit(at[4]))

{

if (*at == '"' || *at == '\\') // json escaped \" stuff is not escaped in CS

{

memmove(base, at, strlen(base)); // skip over the escaped char (in case its a \)

at = base + 1;

converted = true;

}

}

else

{

// convert utf16 \unnnn encode from JSON to our std utf8

char* utf8 = UTF16_2_UTF8(at,false );

if (utf8)

{

unsigned int len = strlen(utf8); // how many chars (\uxxxx vs

strcpy(base, utf8); // overwrite

memmove(base + len, base + 6, strlen(base + 5));

converted = true;

}

}

}

return converted;

}

int factsJsonHelper(char* jsontext, jsmntok_t * tokens, int currToken, MEANING * retMeaning, int* flags, bool key, bool nofail) {

// Always build with duplicate on. create a fresh copy of whatever

jsmntok_t curr = tokens[currToken];

char namebuff[256];

*flags = 0;

int retToken = currToken + 1;

int size = curr.end - curr.start;

switch (curr.type) {

case JSMN_PRIMITIVE: { // true false, numbers, null

char str[1000];

if (size >= 1000)

{

if (!nofail) ReportBug((char*)"Bad Json primitive size %s", jsontext); // if we are not expecting it to fail

return 0;

}

strncpy(str, jsontext + curr.start, size);

str[size] = 0;

// see if its string using single quotes instead of double quotes.

if (*str == '\'' && str[size - 1] == '\'')

{

*flags = JSON_STRING_VALUE; // string null

str[size - 1] = 0;

*retMeaning = MakeMeaning(StoreWord(str + 1, AS_IS));

break;

}

if (!strnicmp(str, "ja-", 3)) *flags = JSON_ARRAY_VALUE;

else if (!strnicmp(str, "jo-", 3)) *flags = JSON_OBJECT_VALUE;

else *flags = JSON_PRIMITIVE_VALUE; // json primitive type

if (*str == USERVAR_PREFIX || *str == SYSVAR_PREFIX || *str == '_' || *str == '\'') // variable values from CS

{

// get path to safety if any

char mainpath[MAX_WORD_SIZE];

char* path = strchr(str, '.');

char* pathbracket = strchr(str, '[');

char* first = path;

if (pathbracket && path && pathbracket < path) first = pathbracket;

if (first) strcpy(mainpath, first);

else *mainpath = 0;

if (path) *path = 0;

if (pathbracket) *pathbracket = 0;

char word1[MAX_WORD_SIZE];

FunctionResult result;

ReadShortCommandArg(str, word1, result); // get the basic item

strcpy(str, word1);

char* numberEnd = NULL;

// now see if we must process a path

if (*mainpath) // access field given

{

char word[MAX_WORD_SIZE];

result = JSONpath(word, mainpath, str, true, nofail); // raw mode

if (result != NOPROBLEM_BIT)

{

if (!nofail) ReportBug((char*)"INFO: Bad Json path building facts from template %s%s data", str, mainpath, jsontext); // if we are not expecting it to fail

return 0;

}

else strcpy(str, word);

}

if (!*str) // empty string treated as null

{

strcpy(str, (char*)"null");

*flags = JSON_STRING_VALUE; // string null

}

else if (!strcmp(str, (char*)"true") || !strcmp(str, (char*)"false"))

{

}

else if (!strncmp(str, (char*)"ja-", 3) || !strncmp(str, (char*)"jo-", 3))

{

if (str[1] == 'a') *flags = JSON_ARRAY_VALUE;

else *flags = JSON_OBJECT_VALUE;

MEANING M = jcopy(MakeMeaning(StoreWord(str, AS_IS)));

WORDP D = Meaning2Word(M);

strcpy(str, D->word);

}

else if ((numberEnd = IsJsonNumber(str)) && numberEnd == (str + strlen(str))) { ; }

else *flags = JSON_STRING_VALUE; // cannot be number

}

*retMeaning = MakeMeaning(StoreWord(str, AS_IS));

break;

}

case JSMN_STRING: {

char* limit;

char* str = InfiniteStack(limit, "factsJsonHelper string");

strncpy(str, jsontext + curr.start, size);

str[size] = 0;

if (ConvertUnicode(str)) size = strlen(str);

*flags = JSON_STRING_VALUE; // string null

CompleteBindStack();

if (!PreallocateHeap(size)) return 0;

if (size == 0) *retMeaning = MakeMeaning(StoreWord((char*)"null", AS_IS)); // empty string replaced with null

else *retMeaning = MakeMeaning(StoreWord(str, AS_IS));

ReleaseStack(str);

break;

}

case JSMN_OBJECT: {

// Build the object name

MEANING objectName = GetUniqueJsonComposite((char*)"jo-");

*retMeaning = objectName;

for (int i = 0; i < curr.size / 2; i++) { // each entry takes an id and a value

MEANING keyMeaning = 0;

int jflags = 0;

retToken = factsJsonHelper(jsontext, tokens, retToken, &keyMeaning, &jflags, true, nofail);

if (retToken == 0) return 0;

MEANING valueMeaning = 0;

retToken = factsJsonHelper(jsontext, tokens, retToken, &valueMeaning, &jflags, false, nofail);

if (retToken == 0) return 0;

CreateFact(objectName, keyMeaning, valueMeaning, jsonCreateFlags | jsonPermanent | jflags | JSON_OBJECT_FACT); // only the last value of flags matters. 5 means object fact in subject

}

*flags = JSON_OBJECT_VALUE;

break;

}

case JSMN_ARRAY: {

// Build the array name

MEANING arrayName = GetUniqueJsonComposite((char*)"ja-");

*retMeaning = arrayName;

for (int i = 0; i < curr.size; i++) {

sprintf(namebuff, "%d", i); // Build the array index

MEANING index = MakeMeaning(StoreWord(namebuff, AS_IS));

MEANING arrayMeaning = 0;

int jflags = 0;

retToken = factsJsonHelper(jsontext, tokens, retToken, &arrayMeaning, &jflags, false, nofail);

if (retToken == 0) return 0;

CreateFact(arrayName, index, arrayMeaning, jsonCreateFlags | jsonPermanent | jflags | JSON_ARRAY_FACT); // flag6 means subject is arrayfact

}

*flags = JSON_ARRAY_VALUE;

break;

}

default:

char* str = AllocateBuffer("jsmn default"); // cant use InfiniteStack because ReportBug will.

strncpy(str, jsontext + curr.start, size);

str[size] = 0;

FreeBuffer("jsmn default");

ReportBug((char*)"FATAL: (factsJsonHelper) Unknown JSON type encountered: %s", str);

}

currentFact = NULL;

return retToken;

}

// Define our struct for accepting LCs output

struct CurlBufferStruct {

char* buffer;

size_t size;

};

static void dump(const char* text, FILE * stream, unsigned char* ptr, size_t size) // libcurl callback when verbose is on

{

size_t i;

size_t c;

unsigned int width = 0x10;

(*printer)((char*)"%s, %10.10ld bytes (0x%8.8lx)\n", text, (long)size, (long)size);

for (i = 0; i < size; i += width)

{

(*printer)((char*)"%4.4lx: ", (long)i);

/* show hex to the left */

for (c = 0; c < width; c++)

{

if (i + c < size) (*printer)((char*)"%02x ", ptr[i + c]);

else (*printer)((char*)"%s", (char*)" ");

}

/* show data on the right */

for (c = 0; (c < width) && (i + c < size); c++) (*printer)("%c", (ptr[i + c] >= 0x20) && (ptr[i + c] < 0x80) ? ptr[i + c] : '.');

(*printer)((char*)"%s", (char*)"\n");

}

}

/*

----------------------

FUNCTION: JSONOpenCode

Function arguments :

optional argument(1) - permanent, transient, direct

ARGUMENT(1) - request method : POST, GET, POSTU, GETU

ARGUMENT(2) - URL. The URL to use in the request

ARGUMENT(3) - If a POST request, this argument contains the post data

ARGUMENT(4) - This argument contains any needed extra REQUEST headers for the request(see note above).

e.g.

$$url = "https://api.github.com/users/test/repos"

$$user_agent = ^"User-Agent: Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; WOW64; Trident/6.0)"

^jsonopen(GET $$url "" $$user_agent)

# GitHub requires a valid user agent header or it will reject the request. Note, although

# not shown, if there are multiple extra headers they should be separated by the

# tilde character ("~").

E.g.

$$url = "https://en.wikipedia.org/w/api.php?action=query&titles=Main%20Page&rvprop=content&format=json"

$$user_agent = ^"myemail@hotmail.com User-Agent: Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; WOW64; Trident/6.0)"

*/

#define REQUEST_HEADER_NVP_SEPARATOR "~"

#define REQUEST_NVP_SEPARATOR ':'

// This function reimplements the semi-standard function strlcpy so we can use it on both Windows, Linux and Mac

size_t our_strlcpy(char* dst, const char* src, size_t siz) {

char* d = dst;

const char* s = src;

size_t n = siz;

/* Copy as many bytes as will fit */

if (n != 0 && --n != 0) {

do {

if ((*d++ = *s++) == 0) break;

} while (--n != 0);

}

/* Not enough room in dst, add NUL and traverse rest of src */

if (n == 0) {

if (siz != 0) *d = '\0'; /* NUL-terminate dst */

while (*s++) { ; }

}

return(s - src - 1); /* count does not include NUL */

}

#ifdef WIN32

// If we're on Windows, just use the safe strncpy version, strncpy_s.

# define SAFE_SPRINTF sprintf_s

#else

// Use snprintf for Linux.

# define SAFE_SPRINTF snprintf

#endif

static int EncodingValue(char* name, char* field, int value)

{

size_t len = strlen(name);

char* at = strstr(field, name);

if (!at) return value; // not found

at += len;

if (at[0] != ';') return 2; // autho

if (at[1] != 'q' && at[1] != 'Q') return 2;

if (at[2] != '=' || at[3] != '0') return 2; // gzip;q=1

if (at[4] == '.') return 2; // gzip;q=0.5

return 1;

}

#ifndef DISCARDJSONOPEN

#ifdef WIN32

#include "curl.h"

#ifdef DEBUG

#pragma comment(lib, "../SRC/curl/libcurld.lib")

#else

#pragma comment(lib, "../SRC/curl/libcurl.lib")

#endif

#else

#include <curl/curl.h>

#endif

CURL* curl;

static int my_trace(CURL * handle, curl_infotype type, char* data, size_t size, void* userp)

{

const char* text;

(void)handle; /* prevent compiler warning */

switch (type) {

case CURLINFO_TEXT:

(*printer)("== Info: %s", data);

default: /* in case a new one is introduced to shock us */

return 0;

case CURLINFO_HEADER_OUT:

text = "=> Send header";

break;

case CURLINFO_DATA_OUT:

text = "=> Send data";

break;

case CURLINFO_SSL_DATA_OUT:

text = "=> Send SSL data";

break;

case CURLINFO_HEADER_IN:

text = "<= Recv header";

break;

case CURLINFO_DATA_IN:

text = "<= Recv data";

break;

case CURLINFO_SSL_DATA_IN:

text = "<= Recv SSL data";

break;

}

dump(text, stderr, (unsigned char*)data, size);

return 0;

}

// This is the function we pass to LC, which writes the output to a BufferStruct

static size_t CurlWriteMemoryCallback(void* ptr, size_t size, size_t nmemb, void* data) {

size_t realsize = size * nmemb;

static char* currentLoc;

static char* limit; // where we can allocate to

struct CurlBufferStruct* mem = (struct CurlBufferStruct*)data;

if (!curlBufferBase)

{

curlBufferBase = InfiniteStack(limit, "CurlWriteMemoryCallback"); // can only be released from JSONOpenCode

currentLoc = curlBufferBase;

mem->buffer = curlBufferBase;

}

mem->size += realsize;

if ((int)mem->size > (int)(limit - curlBufferBase)) ReportBug("FATAL: out of curlmemory"); // out of memory

memcpy(currentLoc, ptr, realsize); // add to buffer

currentLoc[realsize] = 0;

currentLoc += realsize;

return realsize;

}

void CurlShutdown()

{

if (curl) {

curl_easy_cleanup(curl);

curl = NULL;

}

if (curl_done_init) curl_global_cleanup();

curl_done_init = false;

}

FunctionResult InitCurl()

{

// Get curl ready -- do this ONCE only during run of CS

if (!curl_done_init) {

#ifdef WIN32

if (InitWinsock() == FAILRULE_BIT) // only init winsock one per any use- we might have done this from TCPOPEN or PGCode

{

ReportBug((char*)"INFO: Winsock init failed");

return FAILRULE_BIT;

}

#endif

curl_global_init(CURL_GLOBAL_SSL);

curl_done_init = true;

}

return NOPROBLEM_BIT;

}

char* UrlEncodePiece(char* input)

{

InitCurl();

CURL* curlEscape = curl_easy_init();

if (!curlEscape)

{

if (trace & TRACE_JSON) Log(USERLOG, "Curl easy init failed");

return NULL;

}

char* fixed = curl_easy_escape(curlEscape, input, 0);

char* limit;

char* buffer = InfiniteStack(limit, "UrlEncode");

strcpy(buffer, fixed);

curl_free(fixed);

curl_easy_cleanup(curlEscape);

ReleaseInfiniteStack();

return buffer;

}

char* encodeSegment(char** fixed, char* at, char* start, CURL * curl)

{

char* segment = *fixed;

if (at[-1] == '\\') return start; // escaped, allow as is

if (at != start) // url encode segment

{

strncpy(segment, start, at - start);

segment[at - start] = 0;

char* coded = curl_easy_escape(curl, segment, 0);

strcpy(segment, coded);

curl_free(coded);

segment += strlen(segment);

}

*segment++ = *at;

*segment = 0;

*fixed = segment;

return(at + 1);

}

// RFC 3986

static char* JSONUrlEncode(char* urlx, char* fixedUrl, CURL * curl)

{

char* fixed = fixedUrl;

*fixed = 0;

char* at = urlx - 1;

char* start = urlx;

char c;

urlSegment currentSegment = URL_SCHEME;

while ((c = *++at)) // url encode segments

{

if (currentSegment == URL_SCHEME) {

if (c == ':' || c == '+' || c == '-' || c == '.') {

start = encodeSegment(&fixed, at, start, curl);

if (start == at) continue;

if (c == ':') {

*fixed++ = *++at; // double /

*fixed++ = *++at;

start += 2;

currentSegment = URL_AUTHORITY;

}

}

}

else if (currentSegment == URL_AUTHORITY) {

if (c == '/' || c == '?' || c == '#' || c == '@' || c == ':' || c == '[' || c == ']' ||

c == '!' || c == '$' || c == '&' || c == '(' || c == ')' ||

c == '*' || c == '+' || c == ',' || c == ';' || c == '=' || c == '\'') {

start = encodeSegment(&fixed, at, start, curl);

if (start == at) continue;

if (c == '/') {

currentSegment = URL_PATH;

}

else if (c == '?') {

currentSegment = URL_QUERY;

}

else if (c == '#') {

currentSegment = URL_FRAGMENT;

}

}

}

else if (currentSegment == URL_PATH) {

if (c == '/' || c == '?' || c == '#' || c == ':' || c == '@' ||

c == '!' || c == '$' || c == '&' || c == '(' || c == ')' ||

c == '*' || c == '+' || c == ',' || c == ';' || c == '=' || c == '\'') {

start = encodeSegment(&fixed, at, start, curl);

if (start == at) continue;

if (c == '?') {

currentSegment = URL_QUERY;

}

else if (c == '#') {

currentSegment = URL_FRAGMENT;

}

}

}

else if (currentSegment == URL_QUERY) {

if (c == '#' || c == '&' || c == '=' || c == '/' || c == '?') {

start = encodeSegment(&fixed, at, start, curl);

if (start == at) continue;

if (c == '#') {

currentSegment = URL_FRAGMENT;

}

}

}

else if (currentSegment == URL_FRAGMENT) {

if (c == '/' || c == '?') {

start = encodeSegment(&fixed, at, start, curl);

}

}

}

// remaining piece

start = encodeSegment(&fixed, at, start, curl);

return fixedUrl;

}

// Open a URL using the given arguments and return the JSON object's returned by querying the given URL as a set of ChatScript facts.

FunctionResult JSONOpenCode(char* buffer)

{

int index = JSONArgs();

size_t len;

curlBufferBase = NULL;

char* arg = NULL;

char* extraRequestHeadersRaw = NULL;

char kind = 0;

char fieldName[1000];

char fieldValue[1000];

char headerLine[1000];

char* raw_kind = ARGUMENT(index++);

if (!stricmp(raw_kind, "POST")) kind = 'P';

else if (!stricmp(raw_kind, "GET")) kind = 'G';

else if (!stricmp(raw_kind, "POSTU")) kind = 'P';

else if (!stricmp(raw_kind, "GETU")) kind = 'G';

else if (!stricmp(raw_kind, "PUT")) kind = 'U';

else if (!stricmp(raw_kind, "DELETE")) kind = 'D';

else {

char* msg = "jsonopen- only POST, GET, PUT AND DELETE allowed\r\n";

SetUserVariable((char*)"$$tcpopen_error", msg); // pass along the error

ReportBug(msg);

return FAILRULE_BIT;

}

char* urlx = ARGUMENT(index++);

// Now fix starting and ending quotes around url if there are any

if (*urlx == '"') ++urlx;

len = strlen(urlx);

if (urlx[len - 1] == '"') urlx[len - 1] = 0;

// convert \" to " within params and remove any wrapper

arg = ARGUMENT(index++);

if (*arg == '"') ++arg;

len = strlen(arg);

if (arg[len - 1] == '"') arg[len - 1] = 0;

if (!stricmp(arg, (char*)"null")) *arg = 0; // empty string replaces null

// convert json ref to text

if (IsValidJSONName(arg))

{

WORDP D = FindWord(arg);

if (!D) return FAILRULE_BIT;

NextInferMark();

unsigned int limit = 1000000;

arg = AllocateStack(NULL, limit);

jwrite(arg, arg, D, 1, false, limit); // it is subject field

}

bool bIsExtraHeaders = false;

extraRequestHeadersRaw = ARGUMENT(index++);

char* timeout = GetUserVariable("$cs_jsontimeout", false, true);

if (IsDigit(*ARGUMENT(index))) timeout = ARGUMENT(index); // local override

long timelimit = (*timeout) ? atoi(timeout) : 300L;

// Make sure the raw extra REQUEST headers parameter value is not empty and

// not the ChatScript empty argument character.

if (*extraRequestHeadersRaw)

{

// If the parameter value is only 1 characters long and it is a question mark,

// then ignore it since it's the "placeholder" (i.e. - "empty") parameter value

// indicating the parameter should be ignored.

if (!((strlen(extraRequestHeadersRaw) == 1) && (*extraRequestHeadersRaw == '?')))

{

// Remove surrounding double-quotes if found.

if (*extraRequestHeadersRaw == '"') ++extraRequestHeadersRaw;

len = strlen(extraRequestHeadersRaw);

if (extraRequestHeadersRaw[len - 1] == '"') extraRequestHeadersRaw[len - 1] = 0;

bIsExtraHeaders = true;

}

} // if (strlen(extraRequestHeadersRaw) > 0)

uint64 start_time = ElapsedMilliseconds();

CURLcode res;

struct CurlBufferStruct output;

output.buffer = NULL;

output.size = 0;

// Get curl ready -- do this ONCE only during run of CS

if (InitCurl() != NOPROBLEM_BIT) return FAILRULE_BIT;

if (trace & TRACE_JSON)

{

Log(USERLOG, "Curl version: %s \r\n", curl_version());

}

// Only need to get one curl easy handle so that can cache connections

if (curl)

{

// reinitialize all the options

// but leaves the connections and DNS cache

curl_easy_reset(curl);

}

else

{

curl = curl_easy_init();

if (!curl)

{

if (trace & TRACE_JSON) Log(USERLOG, (char*)"Curl easy init failed");

return FAILRULE_BIT;

}

}

// url encode as necessary

char* fixedUrl = AllocateBuffer();

JSONUrlEncode(urlx, fixedUrl, curl);

if (trace & TRACE_JSON)

{

Log(USERLOG, "\r\n");

Log(USERLOG, "Json method/url: %s %s\r\n", raw_kind, fixedUrl);

if (kind == 'P' || kind == 'U')

{

Log(USERLOG, "\r\n");

len = strlen(arg);

if (len < (size_t)(logsize - SAFE_BUFFER_MARGIN)) Log(USERLOG, "Json data %d bytes: %s\r\n ", len, arg);

else Log(USERLOG, "Json data %d bytes\r\n ", len);

Log(USERLOG, "");

}

}

// Add the necessary headers for the request.

struct curl_slist* header = NULL;

if (kind == 'P')

{

curl_easy_setopt(curl, CURLOPT_POST, 1);

curl_easy_setopt(curl, CURLOPT_POSTFIELDS, arg);

}

if (kind == 'U')

{

curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "PUT");

curl_easy_setopt(curl, CURLOPT_POSTFIELDS, arg);

}

if (kind == 'D')

{

curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE");

curl_easy_setopt(curl, CURLOPT_POSTFIELDS, arg);

}

// Assuming a content return type of JSON.

int gzip = 0;

int deflate = 0;

int compress = 0;

int identity = 0;

int wild = 0;

bool contentSeen = false;

// If any extra REQUEST headers were specified, add them now.

if (bIsExtraHeaders)

{

// REQUEST header name/value pairs are separated by tildes ("~").

char* p = strtok(extraRequestHeadersRaw, REQUEST_HEADER_NVP_SEPARATOR);

// Process each REQUEST header.

while (p)

{

// Split the REQUEST header label and field value.

char* p2 = strchr(p, REQUEST_NVP_SEPARATOR);

if (p2)

{

// Delimiter found. Split out the field name and it's value.

*p2 = 0;

our_strlcpy(fieldName, p, sizeof(fieldName));

char name[MAX_WORD_SIZE];

MakeLowerCopy(name, fieldName);

p2++;

our_strlcpy(fieldValue, p2, sizeof(fieldValue));

char value[MAX_WORD_SIZE];

MakeLowerCopy(value, fieldValue);

len = strlen(value);

while (value[len - 1] == ' ') value[--len] = 0; // remove trailing blanks, forcing the field to abut the ~

if (!strnicmp(name, "Content-type", 12)) contentSeen = true;

if (strstr(name, (char*)"accept-encoding"))

{

gzip = EncodingValue((char*)"gzip", value, gzip);

deflate = EncodingValue((char*)"deflate", value, deflate);

compress = EncodingValue((char*)"compress", value, compress);

identity = EncodingValue((char*)"identity", value, identity);

wild = EncodingValue((char*)"*", value, wild);

}

}

else

{

// No delimiter found. Use the entire string as the field name and wipe the field value.

our_strlcpy(fieldName, p, sizeof(fieldValue));

strcpy(fieldValue, "");

char value[MAX_WORD_SIZE];

MakeLowerCopy(value, fieldName);

if (strstr(value, (char*)"accept-encoding"))

{

gzip = EncodingValue((char*)"gzip", value, gzip);

deflate = EncodingValue((char*)"deflate", value, deflate);

compress = EncodingValue((char*)"compress", value, compress);

identity = EncodingValue((char*)"identity", value, identity);

wild = EncodingValue((char*)"*", value, wild);

}

}

// Trim trailing spaces from header key and value, https://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html

char* name = TrimSpaces(fieldName, true);

char* value = TrimSpaces(fieldValue, true);

// Build the REQUEST header line for CURL.

SAFE_SPRINTF(headerLine, sizeof(headerLine), "%s: %s", name, value);

// Add the new REQUEST header to the headers list for this request.

header = curl_slist_append(header, headerLine);

// Next REQUEST header.

p = strtok(NULL, REQUEST_HEADER_NVP_SEPARATOR);

} // while (p)

} // if (extraRequestHeadersRaw)

if (!contentSeen) header = curl_slist_append(header, "Content-Type: application/json");

if (trace & TRACE_JSON)

{

Log(USERLOG, "\r\n");

curl_slist* list = header;

while (list)

{

Log(USERLOG, "JSON header: %s\r\n", list->data);

list = list->next;

}

Log(USERLOG, "\r\n");

}

char coding[MAX_WORD_SIZE];

*coding = 0;

if (wild == 2) // authorizes anything not mentioned

{

if (gzip == 0) gzip = 2;

if (compress == 0) compress = 2;

if (identity == 0) identity = 2;

if (deflate == 0) deflate = 2;

}

if (compress == 2)

{

if (gzip == 0) gzip = 2;

if (deflate == 0) deflate = 2;

}

if (gzip == 2) strcat(coding, (char*)"gzip,");

if (deflate == 2) strcat(coding, (char*)"deflate,");

if (identity == 2) strcat(coding, (char*)"identity,");

if (!*coding) strcpy(coding, (char*)"identity,");

size_t len1 = strlen(coding);

coding[len1 - 1] = 0; // remove terminal comma

#if LIBCURL_VERSION_NUM >= 0x071506

// CURLOPT_ACCEPT_ENCODING renamed in curl 7.21.6

curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, coding);

#else

curl_easy_setopt(curl, CURLOPT_ENCODING, coding);

#endif

// Set up the CURL request.

res = CURLE_OK;

// proxy ability

char* proxyuser = GetUserVariable("$cs_proxycredentials", false, true); // "myname:thesecret"

if (res == CURLE_OK && proxyuser && *proxyuser) res = curl_easy_setopt(curl, CURLOPT_PROXYUSERPWD, proxyuser);

char* proxyserver = GetUserVariable("$cs_proxyserver", false, true); // "http://local.example.com:1080"

if (res == CURLE_OK && proxyserver && *proxyserver) res = curl_easy_setopt(curl, CURLOPT_PROXY, proxyserver);

char* proxymethod = GetUserVariable("$cs_proxymethod", false, true); // "CURLAUTH_ANY"

if (res == CURLE_OK && proxymethod && *proxymethod) res = curl_easy_setopt(curl, CURLOPT_PROXYAUTH, atol(proxymethod));

if (res == CURLE_OK) res = curl_easy_setopt(curl, CURLOPT_HTTPHEADER, header);

if (res == CURLE_OK) res = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, CurlWriteMemoryCallback); // callback for memory

if (res == CURLE_OK) res = curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&output); // store output here

if (res == CURLE_OK) res = curl_easy_setopt(curl, CURLOPT_URL, fixedUrl);

if (res == CURLE_OK) curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, my_trace);

if (res == CURLE_OK) curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, timelimit); // 300 second timeout to connect (once connected no effect)

if (res == CURLE_OK) curl_easy_setopt(curl, CURLOPT_TIMEOUT, timelimit * 2);

if (res == CURLE_OK) curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); // dont generate signals in unix

/* the DEBUGFUNCTION has no effect until we enable VERBOSE */

if (trace & TRACE_JSON && deeptrace) curl_easy_setopt(curl, CURLOPT_VERBOSE, (long)1);

if (res == CURLE_OK) res = curl_easy_perform(curl);

curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_response);

char code[MAX_WORD_SIZE];

sprintf(code, (char*)"%ld", http_response);

if (curlBufferBase) CompleteBindStack();

if (trace & TRACE_JSON && res != CURLE_OK)

{

char word[MAX_WORD_SIZE * 10];

char* at = word;

sprintf(at, "Json method/url: %s %s -- ", raw_kind, fixedUrl);

at += strlen(at);

if (bIsExtraHeaders)

{

sprintf(at, "Json header: %s -- ", extraRequestHeadersRaw);

at += strlen(at);

if (kind == 'P' || kind == 'U') sprintf(at, "Json data: %s\r\n ", arg);

}

if (res == CURLE_URL_MALFORMAT) { ReportBug((char*)"\r\nINFO: Json url malformed %s", word); }

else if (res == CURLE_GOT_NOTHING) { ReportBug((char*)"\r\nINFO: Curl got nothing %s", word); }

else if (res == CURLE_UNSUPPORTED_PROTOCOL) { ReportBug((char*)"\r\nINFO: Curl unsupported protocol %s", word); }