diff --git a/.gitignore b/.gitignore index 085fa98..240d5e9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,9 @@ #Eclipse .project +venv +venv_win +TestFiles +test.txt # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/Algorithmia/CLI.py b/Algorithmia/CLI.py index ed6f389..565f5d0 100644 --- a/Algorithmia/CLI.py +++ b/Algorithmia/CLI.py @@ -1,37 +1,37 @@ import Algorithmia import os -from Algorithmia.errors import DataApiError +from Algorithmia.errors import DataApiError, AlgorithmException from Algorithmia.algo_response import AlgoResponse import json, re, requests, six import toml import shutil - -class CLI(): +class CLI: def __init__(self): self.client = Algorithmia.client() # algo auth - def auth(self, apikey, apiaddress, cacert="", profile="default"): + def auth(self, apiaddress, apikey="", cacert="", profile="default", bearer=""): - #store api key in local config file and read from it each time a client needs to be created + # store api key in local config file and read from it each time a client needs to be created key = self.getconfigfile() config = toml.load(key) - if('profiles' in config.keys()): - if(profile in config['profiles'].keys()): + if ('profiles' in config.keys()): + if (profile in config['profiles'].keys()): config['profiles'][profile]['api_key'] = apikey config['profiles'][profile]['api_server'] = apiaddress config['profiles'][profile]['ca_cert'] = cacert + config['profiles'][profile]['bearer_token'] = bearer else: - config['profiles'][profile] = {'api_key':apikey,'api_server':apiaddress,'ca_cert':cacert} + config['profiles'][profile] = {'api_key':apikey,'api_server':apiaddress,'ca_cert':cacert,'bearer_token':bearer} else: - config['profiles'] = {profile:{'api_key':apikey,'api_server':apiaddress,'ca_cert':cacert}} + config['profiles'] = {profile:{'api_key':apikey,'api_server':apiaddress,'ca_cert':cacert,'bearer_token':bearer }} with open(key, "w") as key: toml.dump(config,key) - self.ls(path = None,client = Algorithmia.client(self.getAPIkey(profile))) + self.ls(path = None,client = CLI().getClient(profile)) # algo run run the the specified algo def runalgo(self, options, client): @@ -44,59 +44,59 @@ def runalgo(self, options, client): algo.set_options(timeout=options.timeout, stdout=options.debug) - #handle input type flags - if(options.data != None): - #data + # handle input type flags + if (options.data != None): + # data algo_input = options.data result = algo.pipe(algo_input) - elif(options.text != None): - #text + elif (options.text != None): + # text algo_input = options.text key = self.getAPIkey(options.profile) content = 'text/plain' algo_input = algo_input.encode('utf-8') - elif(options.json != None): - #json + elif (options.json != None): + # json algo_input = options.json key = self.getAPIkey(options.profile) content = 'application/json' - elif(options.binary != None): - #binary + elif (options.binary != None): + # binary algo_input = bytes(options.binary) key = self.getAPIkey(options.profile) content = 'application/octet-stream' - elif(options.data_file != None): - #data file - algo_input = open(options.data_file,"r").read() + elif (options.data_file != None): + # data file + algo_input = open(options.data_file, "r").read() result = algo.pipe(algo_input) - elif(options.text_file != None): - #text file - algo_input = open(options.text_file,"r").read() + elif (options.text_file != None): + # text file + algo_input = open(options.text_file, "r").read() key = self.getAPIkey(options.profile) content = 'text/plain' algo_input = algo_input.encode('utf-8') - elif(options.json_file != None): - #json file - #read json file and run algo with that input bypassing the auto detection of input type in pipe - with open(options.json_file,"r") as f: + elif (options.json_file != None): + # json file + # read json file and run algo with that input bypassing the auto detection of input type in pipe + with open(options.json_file, "r") as f: algo_input = f.read() key = self.getAPIkey(options.profile) content = 'application/json' algo_input = json.dumps(algo_input).encode('utf-8') - elif(options.binary_file != None): - #binary file - with open(options.binary_file,"rb") as f: - algo_inputs = bytes(f.read()) + elif (options.binary_file != None): + # binary file + with open(options.binary_file, "rb") as f: + algo_input = bytes(f.read()) key = self.getAPIkey(options.profile) content = 'application/octet-stream' @@ -104,25 +104,27 @@ def runalgo(self, options, client): else: output = "no valid input detected" - if(content != None): + if (content != None): result = AlgoResponse.create_algo_response(requests.post(url, data=algo_input, - headers={'Authorization':key,'Content-Type':content}, params= algo.query_parameters).json()) + headers={'Authorization': key, + 'Content-Type': content}, + params=algo.query_parameters).json()) - if(result != None): + if (result != None): output = result.result - #handle output flags + # handle output flags - #output to file if there is an output file specified - if(options.output != None): + # output to file if there is an output file specified + if (options.output != None): outputFile = options.output try: if isinstance(result.result, bytearray) or isinstance(result.result, bytes): - out = open(outputFile,"wb") + out = open(outputFile, "wb") out.write(result.result) out.close() else: - out = open(outputFile,"w") + out = open(outputFile, "w") out.write(result.result) out.close() output = "" @@ -132,18 +134,17 @@ def runalgo(self, options, client): return output - # algo mkdir def mkdir(self, path, client): - #make a dir in data collection + # make a dir in data collection newDir = client.dir(path) if newDir.exists() is False: newDir.create() # algo rmdir - def rmdir(self, path, client, force = False): - #remove a dir in data collection + def rmdir(self, path, client, force=False): + # remove a dir in data collection Dir = client.dir(path) @@ -153,10 +154,9 @@ def rmdir(self, path, client, force = False): except Algorithmia.errors.DataApiError as e: print(e) - def rm(self, path, client): - #for f in path + # for f in path file = client.file(path) try: if file.exists(): @@ -179,7 +179,7 @@ def ls(self, path, client, longlist=False): response = client.getHelper(f.url, **{}) if response.status_code != 200: - raise DataApiError("failed to get file info: " + str(response.content)) + raise DataApiError("failed to get file info: " + str(response.content)) responseContent = response.content if isinstance(responseContent, six.binary_type): @@ -225,14 +225,14 @@ def cat(self, path, client): for f in path: if '://' in f and not f.startswith("http"): if f[-1] == '*': - path += ['data://'+file.path for file in client.dir(f[:len(f)-2]).files()] + path += ['data://' + file.path for file in client.dir(f[:len(f) - 2]).files()] else: file = client.file(f) if file.exists(): result += file.getString() else: - result = "file does not exist "+f + result = "file does not exist " + f break else: print("operands must be a path to a remote data source data://") @@ -240,128 +240,157 @@ def cat(self, path, client): return result + # algo freeze + def freezeAlgo(self, client, manifest_path="model_manifest.json"): + client.freeze(manifest_path) + # algo cp def cp(self, src, dest, client): - if(src is None or dest is None): + if (src is None or dest is None): print("expected algo cp ") else: destLocation = client.file(dest) for f in src: - - #if dest is a directory apend the src name - #if there are multiple src files only the final one will be copied if dest is not a directory + # if dest is a directory apend the src name + # if there are multiple src files only the final one will be copied if dest is not a directory destPath = dest path = dest.split('/') - if(os.path.isdir(dest) or client.dir(dest).exists() and len(path) <= 5): - if(dest[-1] == '/' and path[-1] == ''): - destPath+=client.file(f).getName() - elif(len(path) == 4 or "data://" not in dest): - destPath+='/'+client.file(f).getName() + if (os.path.isdir(dest) or client.dir(dest).exists() and len(path) <= 5): + if (dest[-1] == '/' and path[-1] == ''): + destPath += client.file(f).getName() + elif (len(path) == 4 or "data://" not in dest): + destPath += '/' + client.file(f).getName() - if(f[-1] == '*'): - src += ['data://'+file.path for file in client.dir(f[:len(f)-2]).files()] + if (f[-1] == '*'): + src += ['data://' + file.path for file in client.dir(f[:len(f) - 2]).files()] - #if src is local and dest is remote - elif("data://" not in f and "data://" in dest): + # if src is local and dest is remote + elif ("data://" not in f and "data://" in dest): client.file(destPath).putFile(f) - #if src and dest are remote - elif("data://" in f and "data://" in dest): + # if src and dest are remote + elif ("data://" in f and "data://" in dest): file = client.file(f).getFile() filename = file.name file.close() client.file(destPath).putFile(filename) - #if src is remote and dest is local - elif("data://" in f and "data://" not in dest): + # if src is remote and dest is local + elif ("data://" in f and "data://" not in dest): file = client.file(f).getFile() filename = file.name file.close() - shutil.move(filename,destPath) + shutil.move(filename, destPath) else: print("at least one of the operands must be a path to a remote data source data://") - def get_environment_by_language(self,language,client): + def get_environment_by_language(self, language, client): response = client.get_environment(language) if "error" in response: return json.dumps(response) - return json.dumps(response['environments'],indent=1) - + return json.dumps(response['environments'], indent=1) def list_languages(self, client): response = client.get_supported_languages() - return response - + table = [] + if "error" not in response: + table.append("{:<25} {:<35}".format('Name', 'Description')) + for lang in response: + table.append("{:<25} {:<35}".format(lang['name'], lang['display_name'])) + else: + table.append(json.dumps(response)) + return table def getBuildLogs(self, user, algo, client): - api_response = client.algo(user+'/'+algo).build_logs() - - if "error" in api_response: - return json.dumps(api_response) + api_response = client.algo(user + '/' + algo).get_builds() return json.dumps(api_response['results'], indent=1) def getconfigfile(self): - if(os.name == "posix"): - #if!windows - #~/.algorithmia/config - #create the api key file if it does not exist - keyPath = os.environ['HOME']+"/.algorithmia/" - - elif(os.name == "nt"): - #ifwindows - #%LOCALAPPDATA%\Algorithmia\config - #create the api key file if it does not exist + if (os.name == "posix"): + # if!windows + # ~/.algorithmia/config + # create the api key file if it does not exist + keyPath = os.environ['HOME'] + "/.algorithmia/" + + elif (os.name == "nt"): + # ifwindows + # %LOCALAPPDATA%\Algorithmia\config + # create the api key file if it does not exist keyPath = os.path.expandvars("%LOCALAPPDATA%\\Algorithmia\\") keyFile = "config" - if(not os.path.exists(keyPath)): + if (not os.path.exists(keyPath)): os.mkdir(keyPath) - if(not os.path.exists(keyPath+keyFile)): - with open(keyPath+keyFile,"w") as file: + if (not os.path.exists(keyPath + keyFile)): + with open(keyPath + keyFile, "w") as file: file.write("[profiles]\n") file.write("[profiles.default]\n") file.write("api_key = ''\n") file.write("api_server = ''\n") file.write("ca_cert = ''\n") + file.write("bearer_token = ''\n") - - key = keyPath+keyFile + key = keyPath + keyFile return key - def get_template(self,envid,dest,client): - response = client.get_template(envid,dest) + def get_template(self, envid, dest, client): + response = client.get_template(envid, dest) return response - def getAPIkey(self,profile): + def getAPIkey(self, profile): key = self.getconfigfile() config_dict = toml.load(key) - apikey = None - if('profiles' in config_dict.keys() and profile in config_dict['profiles'].keys()): - apikey = config_dict['profiles'][profile]['api_key'] - return apikey - - def getAPIaddress(self,profile): + if 'profiles' in config_dict and profile in config_dict['profiles'] and \ + config_dict['profiles'][profile]['api_key'] != "": + return config_dict['profiles'][profile]['api_key'] + else: + return None + + def getBearerToken(self,profile): key = self.getconfigfile() config_dict = toml.load(key) + if 'profiles' in config_dict and profile in config_dict['profiles'] and \ + config_dict['profiles'][profile]['bearer_token'] != "": + return config_dict['profiles'][profile]['bearer_token'] + else: + return None + - apiaddress = config_dict['profiles'][profile]['api_server'] + def getAPIaddress(self, profile): + key = self.getconfigfile() + config_dict = toml.load(key) - return apiaddress + if config_dict['profiles'][profile]['api_server'] != "": + return config_dict['profiles'][profile]['api_server'] + else: + return None - def getCert(self,profile): + def getCert(self, profile): key = self.getconfigfile() config_dict = toml.load(key) - cert = None - if('profiles' in config_dict.keys() and profile in config_dict['profiles'].keys()): - cert = config_dict['profiles'][profile]['ca_cert'] - return cert + if 'profiles' in config_dict and profile in config_dict['profiles'] and \ + config_dict['profiles'][profile]['ca_cert'] != "": + return config_dict['profiles'][profile]['ca_cert'] + else: + return None + + def getClient(self,profile): + apiAddress = self.getAPIaddress(profile) + apiKey = self.getAPIkey(profile) + caCert = self.getCert(profile) + bearer = None + + if apiKey is None: + bearer = self.getBearerToken(profile) + + return Algorithmia.client(api_key=apiKey,api_address=apiAddress,ca_cert=caCert,bearer_token = bearer) diff --git a/Algorithmia/__init__.py b/Algorithmia/__init__.py index 05ed6dc..38e7ed6 100644 --- a/Algorithmia/__init__.py +++ b/Algorithmia/__init__.py @@ -23,8 +23,8 @@ def file(dataUrl): def dir(dataUrl): return getDefaultClient().dir(dataUrl) -def client(api_key=None, api_address=None, ca_cert=None): - return Client(api_key, api_address, ca_cert) +def client(api_key=None, api_address=None, ca_cert=None, bearer_token=None): + return Client(api_key, api_address, ca_cert, bearer_token) def handler(apply_func, load_func=lambda: None): return Handler(apply_func, load_func) diff --git a/Algorithmia/__main__.py b/Algorithmia/__main__.py index d8edb35..1b5f7b5 100644 --- a/Algorithmia/__main__.py +++ b/Algorithmia/__main__.py @@ -6,6 +6,7 @@ import six from Algorithmia.CLI import CLI import argparse +import re #bind input to raw input try: @@ -17,7 +18,7 @@ usage = """CLI for interaction with Algorithmia\n Usage:\n algo [] [options] [...]\n -algo[] [--help | --version]\n\n +algo [] [--help | --version]\n\n General commands include:\n auth configure authentication\n\n @@ -108,7 +109,7 @@ def main(): parser_cat.add_argument('--profile', action = 'store', type = str, default = 'default') #sub parser for getting environment template - parser_template = subparsers.add_parser('template',help='template downloads an environment template to the destination') + parser_template = subparsers.add_parser('template', help='template downloads an environment template to the destination') parser_template.add_argument('envid',help='environment specification id') parser_template.add_argument('dest',help='destination for template download') @@ -130,8 +131,12 @@ def main(): subparsers.add_parser('help') parser.add_argument('--profile', action = 'store', type = str, default = 'default') + #sub parser for freeze + subparsers.add_parser('freeze', help="freezes a model_manifest.json file into a model_manifest.json.freeze") + args = parser.parse_args() + #run auth before trying to create a client if args.cmd == 'auth': @@ -141,27 +146,26 @@ def main(): APIkey = input("enter API key: ") CACert = input('(optional) enter path to custom CA certificate: ') - if len(APIkey) == 28 and APIkey.startswith("sim"): - if APIaddress == "" or not APIaddress.startswith("https://api."): - APIaddress = "https://api.algorithmia.com" - - CLI().auth(apikey=APIkey, apiaddress=APIaddress, cacert=CACert, profile=args.profile) + if APIaddress == "" or not APIaddress.startswith("https://api."): + print("invalid API address") else: - print("invalid api key") - + if len(APIkey) == 28 and APIkey.startswith("sim"): + CLI().auth(apikey=APIkey, apiaddress=APIaddress, cacert=CACert, profile=args.profile) + else: + jwt = re.compile(r"^([a-zA-Z0-9_=]+)\.([a-zA-Z0-9_=]+)\.([a-zA-Z0-9_\-\+\/=]*)") + Bearer = input("enter JWT token: ") + if jwt.match(Bearer): + CLI().auth(apikey=APIkey, bearer=Bearer, apiaddress=APIaddress, cacert=CACert, profile=args.profile) + else: + print("invalid authentication") + + + if args.cmd == 'help': parser.parse_args(['-h']) #create a client with the appropreate api address and key - client = Algorithmia.client() - if len(CLI().getAPIaddress(args.profile)) > 1: - client = Algorithmia.client(CLI().getAPIkey(args.profile), CLI().getAPIaddress(args.profile)) - elif len(CLI().getAPIaddress(args.profile)) > 1 and len(CLI().getCert(args.profile)) > 1: - client = Algorithmia.client(CLI().getAPIkey(args.profile), CLI().getAPIaddress(args.profile),CLI().getCert(args.profile)) - elif len(CLI().getAPIaddress(args.profile)) < 1 and len(CLI().getCert(args.profile)) > 1: - client = Algorithmia.client(CLI().getAPIkey(args.profile), CLI().getAPIaddress(args.profile),CLI().getCert(args.profile)) - else: - client = Algorithmia.client(CLI().getAPIkey(args.profile)) + client = CLI().getClient(args.profile) if args.cmd == 'run': @@ -202,9 +206,8 @@ def main(): elif args.cmd == 'languages': response = CLI().list_languages(client) - print("{:<25} {:<35}".format('Name','Description')) - for lang in response: - print("{:<25} {:<35}".format(lang['name'],lang['display_name'])) + for line in response: + print(line) elif args.cmd == 'template': CLI().get_template(args.envid,args.dest,client) @@ -216,6 +219,9 @@ def main(): elif args.cmd == 'builds': print(CLI().getBuildLogs(args.user, args.algo, client)) + elif args.cmd == "freeze": + print(CLI().freezeAlgo(client)) + else: parser.parse_args(['-h']) diff --git a/Algorithmia/algo_response.py b/Algorithmia/algo_response.py index dfce6c1..d5abbc4 100644 --- a/Algorithmia/algo_response.py +++ b/Algorithmia/algo_response.py @@ -1,5 +1,6 @@ import base64 from Algorithmia.errors import raiseAlgoApiError +from Algorithmia.async_response import AsyncResponse import sys @@ -19,8 +20,12 @@ def __repr__(self): @staticmethod def create_algo_response(response): + + # Check if request is async + if 'async_protocol' in response and 'request_id' in response: + return AsyncResponse(response) # Parse response JSON, if it's indeed JSON - if 'error' in response or 'metadata' not in response: + elif 'error' in response or 'metadata' not in response: # Failure raise raiseAlgoApiError(response) else: diff --git a/Algorithmia/algorithm.py b/Algorithmia/algorithm.py index e76b910..40be378 100644 --- a/Algorithmia/algorithm.py +++ b/Algorithmia/algorithm.py @@ -1,17 +1,18 @@ 'Algorithmia Algorithm API Client (python)' -import base64 import json import re from Algorithmia.async_response import AsyncResponse from Algorithmia.algo_response import AlgoResponse -from Algorithmia.errors import ApiError, ApiInternalError, raiseAlgoApiError +from Algorithmia.errors import ApiError, ApiInternalError, raiseAlgoApiError, AlgorithmException from enum import Enum from algorithmia_api_client.rest import ApiException -from algorithmia_api_client import CreateRequest, UpdateRequest, VersionRequest, Details, Settings, SettingsMandatory, SettingsPublish, \ +from algorithmia_api_client import CreateRequest, UpdateRequest, VersionRequest, Details, Settings, SettingsMandatory, \ + SettingsPublish, \ CreateRequestVersionInfo, VersionInfo, VersionInfoPublish -OutputType = Enum('OutputType','default raw void') +OutputType = Enum('OutputType', 'default raw void') + class Algorithm(object): def __init__(self, client, algoRef): @@ -32,112 +33,129 @@ def __init__(self, client, algoRef): raise ValueError('Invalid algorithm URI: ' + algoRef) def set_options(self, timeout=300, stdout=False, output=OutputType.default, **query_parameters): - self.query_parameters = {'timeout':timeout, 'stdout':stdout} + self.query_parameters = {'timeout': timeout, 'stdout': stdout} self.output_type = output self.query_parameters.update(query_parameters) return self + def get_algorithm_id(self): + url = '/v1/algorithms/' + self.username + '/' + self.algoname + print(url) + api_response = self.client.getJsonHelper(url) + if 'id' in api_response: + return api_response['id'] + else: + raise Exception("field 'id' not found in response: ", api_response) + + + def get_secrets(self): + algorithm_id = self.get_algorithm_id() + url = "/v1/algorithms/" + algorithm_id + "/secrets" + api_response = self.client.getJsonHelper(url) + return api_response + + + def set_secret(self, short_name, secret_key, secret_value, description=None): + algorithm_id = self.get_algorithm_id() + url = "/v1/algorithms/" + algorithm_id + "/secrets" + secret_providers = self.client.get_secret_providers() + provider_id = secret_providers[0]['id'] + + create_parameters = { + "owner_type": "algorithm", + "owner_id": algorithm_id, + "short_name": short_name, + "provider_id": provider_id, + "secret_key": secret_key, + "secret_value": secret_value, + } + if description: + create_parameters['description'] = description + else: + create_parameters['description'] = " " + + print(create_parameters) + api_response = self.client.postJsonHelper(url, create_parameters, parse_response_as_json=True) + return api_response + + # Create a new algorithm - def create(self, details={}, settings={}, version_info={}): - detailsObj = Details(**details) - settingsObj = SettingsMandatory(**settings) - createRequestVersionInfoObj = CreateRequestVersionInfo(**version_info) - create_parameters = {"name": self.algoname, "details": detailsObj, "settings": settingsObj, "version_info": createRequestVersionInfoObj} - create_request = CreateRequest(**create_parameters) - try: - # Create Algorithm - api_response = self.client.manageApi.create_algorithm(self.username, create_request) - return api_response - except ApiException as e: - error_message = json.loads(e.body) - raise raiseAlgoApiError(error_message) + def create(self, details, settings, version_info=None, source=None, scmsCredentials=None): + url = "/v1/algorithms/" + self.username + create_parameters = {"name": self.algoname, "details": details, "settings": settings} + if version_info: + create_parameters['version_info'] = version_info + if source: + create_parameters['source'] = source + if scmsCredentials: + create_parameters['scmsCredentials'] = scmsCredentials + + api_response = self.client.postJsonHelper(url, create_parameters, parse_response_as_json=True) + return api_response # Update the settings in an algorithm - def update(self, details={}, settings={}, version_info={}): - detailsObj = Details(**details) - settingsObj = Settings(**settings) - createRequestVersionInfoObj = CreateRequestVersionInfo(**version_info) - update_parameters = {"details": detailsObj, "settings": settingsObj, "version_info": createRequestVersionInfoObj} - update_request = UpdateRequest(**update_parameters) - try: - # Update Algorithm - api_response = self.client.manageApi.update_algorithm(self.username, self.algoname, update_request) - return api_response - except ApiException as e: - error_message = json.loads(e.body) - raise raiseAlgoApiError(error_message) + def update(self, details={}, settings={}, version_info={}, source={}, scmsCredentials={}): + url = "/v1/algorithms/" + self.username + "/" + self.algoname + update_parameters = {"details": details, "settings": settings, + "version_info": version_info, "source": source, "scmsCredentials": scmsCredentials} + api_response = self.client.putHelper(url, update_parameters) + return api_response # Publish an algorithm - def publish(self, details={}, settings={}, version_info={}): - detailsObj = Details(**details) - settingsObj = SettingsPublish(**settings) - versionRequestObj = VersionInfoPublish(**version_info) - publish_parameters = {"details": detailsObj, "settings": settingsObj, "version_info": versionRequestObj} - version_request = VersionRequest(**publish_parameters) # VersionRequest | Publish Version Request - try: - # Publish Algorithm - api_response = self.client.manageApi.publish_algorithm(self.username, self.algoname, version_request) - return api_response - except ApiException as e: - error_message = json.loads(e.body) - raise raiseAlgoApiError(error_message) - - def builds(self, limit=56, marker=None): - try: - if marker is not None: - api_response = self.client.manageApi.get_algorithm_builds(self.username, self.algoname, limit=limit, marker=marker) - else: - api_response = self.client.manageApi.get_algorithm_builds(self.username, self.algoname, limit=limit) - return api_response - except ApiException as e: - error_message = json.loads(e.body) - raise raiseAlgoApiError(error_message) + def publish(self, details={}, settings={}, version_info={}, source={}, scmsCredentials={}): + url = "/v1/algorithms/" + self.username + "/" + self.algoname + "/versions" + publish_parameters = {"details": details, "settings": settings, + "version_info": version_info, "source": source, "scmsCredentials": scmsCredentials} + api_response = self.client.postJsonHelper(url, publish_parameters, parse_response_as_json=True, retry=True) + return api_response + + def get_builds(self, limit=56, marker=None): + kwargs = {"limit": limit, "marker": marker} + url = "/v1/algorithms/" + self.username + "/" + self.algoname + '/builds' + response = self.client.getJsonHelper(url, **kwargs) + return response def get_build(self, build_id): # Get the build object for a given build_id # The build status can have one of the following value: succeeded, failed, in-progress - try: - api_response = self.client.manageApi.get_algorithm_build_by_id(self.username, self.algoname, build_id) - return api_response - except ApiException as e: - error_message = json.loads(e.body) - raise raiseAlgoApiError(error_message) + url = '/v1/algorithms/' + self.username + '/' + self.algoname + '/builds/' + build_id + response = self.client.getJsonHelper(url) + return response def get_build_logs(self, build_id): # Get the algorithm build logs for a given build_id - try: - api_response = self.client.manageApi.get_algorithm_build_logs(self.username, self.algoname, build_id) - return api_response - except ApiException as e: - error_message = json.loads(e.body) - raise raiseAlgoApiError(error_message) - - def build_logs(self): - url = '/v1/algorithms/'+self.username+'/'+self.algoname+'/builds' - response = json.loads(self.client.getHelper(url).content.decode('utf-8')) + url = '/v1/algorithms/' + self.username + '/' + self.algoname + '/builds/' + build_id + '/logs' + response = self.client.getJsonHelper(url) return response - def get_scm_status(self): - try: - api_response = self.client.manageApi.get_algorithm_scm_connection_status(self.username, self.algoname) - return api_response - except ApiException as e: - error_message = json.loads(e.body) - raise raiseAlgoApiError(error_message) + url = '/v1/algorithms/' + self.username + '/' + self.algoname + '/scm/status' + response = self.client.getJsonHelper(url) + return response # Get info on an algorithm def info(self, algo_hash=None): + # Get Algorithm + if algo_hash: + url = '/v1/algorithms/' + self.username + '/' + self.algoname + '/versions/' + algo_hash + else: + url = '/v1/algorithms/' + self.username + '/' + self.algoname + '/versions' + response = self.client.getJsonHelper(url) + return response + + # Check if an Algorithm exists + def exists(self): try: - # Get Algorithm - if algo_hash: - api_response = self.client.manageApi.get_algorithm_hash_version(self.username, self.algoname, algo_hash) + url = '/v1/algorithms/' + self.username + '/' + self.algoname + _ = self.client.getJsonHelper(url) + return True + except AlgorithmException as e: + if "404" in str(e) or "No such algorithm" in str(e): + return False + elif "403" in str(e): + raise Exception("unable to check exists on algorithms you don't own.") else: - api_response = self.client.manageApi.get_algorithm(self.username, self.algoname) - return api_response - except ApiException as e: - error_message = json.loads(e.body) - raise raiseAlgoApiError(error_message) + raise e # Get all versions of the algorithm, with the given filters def versions(self, limit=None, marker=None, published=None, callable=None): @@ -153,24 +171,17 @@ def versions(self, limit=None, marker=None, published=None, callable=None): if callable: c = callable kwargs["callable"] = str(c).lower() if str(c) in bools else c - try: - # Get Algorithm versions - api_response = self.client.manageApi.get_algorithm_versions(self.username, self.algoname, **kwargs) - return api_response - except ApiException as e: - error_message = json.loads(e.body) - raise raiseAlgoApiError(error_message) - + # Get Algorithm versions + url = '/v1/algorithms/' + self.username + '/' + self.algoname + '/versions' + response = self.client.getJsonHelper(url, **kwargs) + return response # Compile an algorithm def compile(self): - try: - # Compile algorithm - api_response = self.client.manageApi.algorithms_username_algoname_compile_post(self.username, self.algoname) - return api_response - except ApiException as e: - error_message = json.loads(e.body) - raise raiseAlgoApiError(error_message) + # Compile algorithm + url = '/v1/algorithms/' + self.username + '/' + self.algoname + '/compile' + response = self.client.postJsonHelper(url, {}, parse_response_as_json=True, retry=True) + return response # Pipe an input into this algorithm def pipe(self, input1): @@ -180,25 +191,26 @@ def pipe(self, input1): elif self.output_type == OutputType.void: return self._postVoidOutput(input1) else: - return AlgoResponse.create_algo_response(self.client.postJsonHelper(self.url, input1, **self.query_parameters)) + return AlgoResponse.create_algo_response( + self.client.postJsonHelper(self.url, input1, **self.query_parameters)) def _postRawOutput(self, input1): - # Don't parse response as json - self.query_parameters['output'] = 'raw' - response = self.client.postJsonHelper(self.url, input1, parse_response_as_json=False, **self.query_parameters) - # Check HTTP code and throw error as needed - if response.status_code == 400: - # Bad request - raise ApiError(response.text) - elif response.status_code == 500: - raise ApiInternalError(response.text) - else: - return response.text + # Don't parse response as json + self.query_parameters['output'] = 'raw' + response = self.client.postJsonHelper(self.url, input1, parse_response_as_json=False, **self.query_parameters) + # Check HTTP code and throw error as needed + if response.status_code == 400: + # Bad request + raise ApiError(response.text) + elif response.status_code == 500: + raise ApiInternalError(response.text) + else: + return response.text def _postVoidOutput(self, input1): - self.query_parameters['output'] = 'void' - responseJson = self.client.postJsonHelper(self.url, input1, **self.query_parameters) - if 'error' in responseJson: - raise ApiError(responseJson['error']['message']) - else: - return AsyncResponse(responseJson) + self.query_parameters['output'] = 'void' + responseJson = self.client.postJsonHelper(self.url, input1, **self.query_parameters) + if 'error' in responseJson: + raise ApiError(responseJson['error']['message']) + else: + return AsyncResponse(responseJson) diff --git a/Algorithmia/client.py b/Algorithmia/client.py index faabbe8..dc26e1a 100644 --- a/Algorithmia/client.py +++ b/Algorithmia/client.py @@ -2,39 +2,49 @@ import Algorithmia from Algorithmia.insights import Insights +from Algorithmia.errors import raiseAlgoApiError from Algorithmia.algorithm import Algorithm -from Algorithmia.datafile import DataFile, LocalDataFile -from Algorithmia.datadirectory import DataDirectory, LocalDataDirectory +from Algorithmia.datafile import DataFile, LocalDataFile, AdvancedDataFile +from Algorithmia.datadirectory import DataDirectory, LocalDataDirectory, AdvancedDataDirectory from algorithmia_api_client import Configuration, DefaultApi, ApiClient - +from Algorithmia.util import md5_for_file, md5_for_str from tempfile import mkstemp import atexit import json, re, requests, six, certifi import tarfile import os +from time import time class Client(object): 'Algorithmia Common Library' - - handle, ca_cert = None,None + handle, ca_cert = None, None apiKey = None apiAddress = None requestSession = None + bearerToken = None - - def __init__(self, apiKey = None, apiAddress = None, caCert = None): + def __init__(self, apiKey=None, apiAddress=None, caCert=None, bearerToken=None): # Override apiKey with environment variable + config = None self.requestSession = requests.Session() if apiKey is None and 'ALGORITHMIA_API_KEY' in os.environ: apiKey = os.environ['ALGORITHMIA_API_KEY'] + elif bearerToken is None and 'ALGORITHMIA_BEARER_TOKEN' in os.environ: + bearerToken = os.environ['ALGORITHMIA_BEARER_TOKEN'] + + self.bearerToken = bearerToken self.apiKey = apiKey if apiAddress is not None: self.apiAddress = apiAddress else: self.apiAddress = Algorithmia.getApiAddress() - if caCert is None and 'REQUESTS_CA_BUNDLE' in os.environ: + if caCert == False: + self.requestSession.verify = False + self.requestSession.trust_env = False + config = Configuration(use_ssl=False) + elif caCert is None and 'REQUESTS_CA_BUNDLE' in os.environ: caCert = os.environ.get('REQUESTS_CA_BUNDLE') self.catCerts(caCert) self.requestSession.verify = self.ca_cert @@ -42,16 +52,17 @@ def __init__(self, apiKey = None, apiAddress = None, caCert = None): self.catCerts(caCert) self.requestSession.verify = self.ca_cert elif caCert is not None and 'REQUESTS_CA_BUNDLE' in os.environ: - #if both are available, use the one supplied in the constructor. I assume that a user supplying a cert in initialization wants to use that one. + # if both are available, use the one supplied in the constructor. I assume that a user supplying a cert in initialization wants to use that one. self.catCerts(caCert) self.requestSession.verify = self.ca_cert + if not config: + config = Configuration() - config = Configuration() config.api_key['Authorization'] = self.apiKey config.host = "{}/v1".format(self.apiAddress) self.manageApi = DefaultApi(ApiClient(config)) - + def algo(self, algoRef): return Algorithm(self, algoRef) @@ -59,17 +70,26 @@ def username(self): username = next(self.dir("").list()).path return username - def file(self, dataUrl): - if dataUrl.startswith('file://'): return LocalDataFile(self, dataUrl) - else: return DataFile(self, dataUrl) + def scms(self): + url = "/v1/scms" + response = self.getJsonHelper(url) + return response + + def file(self, dataUrl, cleanup=False): + if dataUrl.startswith('file://'): + return LocalDataFile(self, dataUrl) + else: + return AdvancedDataFile(self, dataUrl, cleanup) def dir(self, dataUrl): - if dataUrl.startswith('file://'): return LocalDataDirectory(self, dataUrl) - else: return DataDirectory(self, dataUrl) + if dataUrl.startswith('file://'): + return LocalDataDirectory(self, dataUrl) + else: + return AdvancedDataDirectory(self, dataUrl) def create_user(self, requestString): - url = "/v1/users" - response = self.postJsonHelper(url,input_object=requestString) + url = "/v1/users" + response = self.postJsonHelper(url, input_object=requestString) return response def get_org_types(self): @@ -77,48 +97,47 @@ def get_org_types(self): response = self.getHelper(url) return json.loads(response.content.decode("utf-8")) - def create_org(self,requestString): + def create_org(self, requestString): url = "/v1/organizations" type = requestString["type_id"] - id,error = self.convert_type_id(type) + id, error = self.convert_type_id(type) requestString["type_id"] = id - - response = self.postJsonHelper(url=url,input_object=requestString) + + response = self.postJsonHelper(url=url, input_object=requestString) if (error != "") and (response["error"] is not None): response["error"]["message"] = error return response - - def get_org(self,org_name): - url = "/v1/organizations/"+org_name + + def get_org(self, org_name): + url = "/v1/organizations/" + org_name response = self.getHelper(url) return json.loads(response.content.decode("utf-8")) - def edit_org(self,org_name,requestString): - url = "/v1/organizations/"+org_name + def edit_org(self, org_name, requestString): + url = "/v1/organizations/" + org_name type = requestString["type_id"] - id,error = self.convert_type_id(type) + id, error = self.convert_type_id(type) requestString["type_id"] = id data = json.dumps(requestString).encode('utf-8') - response = self.putHelper(url,data) + response = self.putHelper(url, data) if (error != "") and (response["error"] is not None): response["error"]["message"] = error return response - def invite_to_org(self,orgname,username): - url = "/v1/organizations/"+orgname+"/members/"+username - response = self.putHelper(url,data={}) + def invite_to_org(self, orgname, username): + url = "/v1/organizations/" + orgname + "/members/" + username + response = self.putHelper(url, data={}) return response - - def get_template(self,envid,dest,save_tar=False): - url = "/v1/algorithm-environments/edge/environment-specifications/"+envid+"/template" - filename="template.tar.gz" + def get_template(self, envid, dest, save_tar=False): + url = "/v1/algorithm-environments/edge/environment-specifications/" + envid + "/template" + filename = "template.tar.gz" if not os.path.exists(dest): os.makedirs(dest) @@ -144,31 +163,79 @@ def get_template(self,envid,dest,save_tar=False): except OSError as e: print(e) return response - else: + else: return json.loads(response.content.decode("utf-8")) - def get_environment(self,language): - url = "/v1/algorithm-environments/edge/languages/"+language+"/environments" + def get_environment(self, language): + url = "/v1/algorithm-environments/edge/languages/" + language + "/environments" response = self.getHelper(url) return response.json() def get_supported_languages(self): - url ="/v1/algorithm-environments/edge/languages" + url = "/v1/algorithm-environments/edge/languages" + response = self.getHelper(url) + return response.json() + + def get_secret_providers(self): + url = "/v1/secret-provider" + api_response = self.getJsonHelper(url) + return api_response + + def get_organization_errors(self, org_name): + """Gets the errors for the organization. + + Args: + self (Client): The instance of the Client class. + org_name (str): The identifier for the organization. + + Returns: + Any: A JSON-encoded response from the API. + """ + + url = '/v1/organizations/%s/errors' % org_name + response = self.getHelper(url) + return response.json() + + def get_user_errors(self, user_id): + """Gets the errors for a specific user. + + Args: + self (Client): The instance of the Client class. + user_id (str): The identifier for the user. + + Returns: + Any: A JSON-encoded response from the API. + """ + + url = '/v1/users/%s/errors' % user_id response = self.getHelper(url) return response.json() + def get_algorithm_errors(self, algorithm_id): + """Gets the errors for a specific algorithm. + + Args: + self (Client): The instance of the Client class. + algorithm_id (str): The identifier for the algorithm. + Returns: + Any: A JSON-encoded response from the API. + """ + + url = '/v1/algorithms/%s/errors' % algorithm_id + return self.getJsonHelper(url) # Used to send insight data to Algorithm Queue Reader in cluster def report_insights(self, insights): return Insights(insights) - # Used internally to post json to the api and parse json response - def postJsonHelper(self, url, input_object, parse_response_as_json=True, **query_parameters): + def postJsonHelper(self, url, input_object, parse_response_as_json=True, retry=False, **query_parameters): headers = {} if self.apiKey is not None: headers['Authorization'] = self.apiKey + elif self.bearerToken is not None: + headers['Authorization'] = 'Bearer ' + self.bearerToken input_json = None if input_object is None: @@ -184,29 +251,63 @@ def postJsonHelper(self, url, input_object, parse_response_as_json=True, **query input_json = json.dumps(input_object).encode('utf-8') headers['Content-Type'] = 'application/json' - response = self.requestSession.post(self.apiAddress + url, data=input_json, headers=headers, params=query_parameters) - - if parse_response_as_json and response.status_code == 200: - return response.json() - return response + response = self.requestSession.post(self.apiAddress + url, data=input_json, headers=headers, + params=query_parameters) + if 200 <= response.status_code <= 299: + if parse_response_as_json: + response = response.json() + if 'error' in response: + raise raiseAlgoApiError(response) + else: + return response + else: + return response + elif retry: + return self.postJsonHelper(url, input_object, parse_response_as_json, False, **query_parameters) + else: + raise raiseAlgoApiError(response) # Used internally to http get a file def getHelper(self, url, **query_parameters): headers = {} if self.apiKey is not None: headers['Authorization'] = self.apiKey + elif self.bearerToken is not None: + headers['Authorization'] = 'Bearer ' + self.bearerToken return self.requestSession.get(self.apiAddress + url, headers=headers, params=query_parameters) + def getJsonHelper(self, url, **query_parameters): + headers = {} + if self.apiKey is not None: + headers['Authorization'] = self.apiKey + elif self.bearerToken is not None: + headers['Authorization'] = 'Bearer ' + self.bearerToken + response = self.requestSession.get(self.apiAddress + url, headers=headers, params=query_parameters) + if 200 <= response.status_code <= 299: + response = response.json() + if 'error' in response: + raise raiseAlgoApiError(response) + else: + return response + else: + if response.content is not None: + response = response.json() + raise raiseAlgoApiError(response) + def getStreamHelper(self, url, **query_parameters): headers = {} if self.apiKey is not None: headers['Authorization'] = self.apiKey + elif self.bearerToken is not None: + headers['Authorization'] = 'Bearer ' + self.bearerToken return self.requestSession.get(self.apiAddress + url, headers=headers, params=query_parameters, stream=True) def patchHelper(self, url, params): headers = {'content-type': 'application/json'} if self.apiKey is not None: headers['Authorization'] = self.apiKey + elif self.bearerToken is not None: + headers['Authorization'] = 'Bearer ' + self.bearerToken return self.requestSession.patch(self.apiAddress + url, headers=headers, data=json.dumps(params)) # Used internally to get http head result @@ -214,58 +315,68 @@ def headHelper(self, url): headers = {} if self.apiKey is not None: headers['Authorization'] = self.apiKey + elif self.bearerToken is not None: + headers['Authorization'] = 'Bearer ' + self.bearerToken return self.requestSession.head(self.apiAddress + url, headers=headers) - # Used internally to http put a file def putHelper(self, url, data): headers = {} if self.apiKey is not None: headers['Authorization'] = self.apiKey + elif self.bearerToken is not None: + headers['Authorization'] = 'Bearer ' + self.bearerToken if isJson(data): headers['Content-Type'] = 'application/json' - response = self.requestSession.put(self.apiAddress + url, data=data, headers=headers) if response._content == b'': return response - return response.json() + if 200 <= response.status_code <= 299: + response = response.json() + if 'error' in response: + raise raiseAlgoApiError(response) + else: + return response + else: + raise raiseAlgoApiError(response) # Used internally to http delete a file def deleteHelper(self, url): headers = {} if self.apiKey is not None: headers['Authorization'] = self.apiKey + elif self.bearerToken is not None: + headers['Authorization'] = 'Bearer ' + self.bearerToken response = self.requestSession.delete(self.apiAddress + url, headers=headers) if response.reason == "No Content": return response return response.json() # Used internally to concatonate given custom cert with built in certificate store. - def catCerts(self,customCert): - self.handle, self.ca_cert = mkstemp(suffix = ".pem") - #wrapped all in the with context handler to prevent unclosed files - with open(customCert,'r') as custom_cert, \ - open(self.ca_cert,'w') as ca,\ - open(certifi.where(),'r') as cert: - new_cert = custom_cert.read() + cert.read() - ca.write(new_cert) + def catCerts(self, customCert): + self.handle, self.ca_cert = mkstemp(suffix=".pem") + # wrapped all in the with context handler to prevent unclosed files + with open(customCert, 'r') as custom_cert, \ + open(self.ca_cert, 'w') as ca, \ + open(certifi.where(), 'r') as cert: + new_cert = custom_cert.read() + cert.read() + ca.write(new_cert) atexit.register(self.exit_handler) - - #User internally to convert type id name to uuid - def convert_type_id(self,type): - id="" - error="" + + # User internally to convert type id name to uuid + def convert_type_id(self, type): + id = "" + error = "" types = self.get_org_types() for enumtype in types: if type == enumtype["name"]: id = enumtype["id"] - error="" + error = "" break else: error = "invalid type_id" - - return(id,error) + return (id, error) # Used internally to clean up temporary files def exit_handler(self): @@ -273,12 +384,38 @@ def exit_handler(self): os.close(self.handle) os.unlink(self.ca_cert) except OSError as e: - print(e) + print(e) + + # Used by CI/CD automation for freezing model manifest files, and by the CLI for manual freezing + def freeze(self, manifest_path, manifest_output_dir="."): + if os.path.exists(manifest_path): + with open(manifest_path, 'r') as f: + manifest_file = json.load(f) + manifest_file['timestamp'] = str(time()) + required_files = manifest_file['required_files'] + optional_files = manifest_file['optional_files'] + for i in range(len(required_files)): + uri = required_files[i]['source_uri'] + local_file = self.file(uri).getFile(as_path=True) + md5_checksum = md5_for_file(local_file) + required_files[i]['md5_checksum'] = md5_checksum + for i in range(len(optional_files)): + uri = required_files[i]['source_uri'] + local_file = self.file(uri).getFile(as_path=True) + md5_checksum = md5_for_file(local_file) + required_files[i]['md5_checksum'] = md5_checksum + lock_md5_checksum = md5_for_str(str(manifest_file)) + manifest_file['lock_checksum'] = lock_md5_checksum + with open(manifest_output_dir + '/' + 'model_manifest.json.freeze', 'w') as f: + json.dump(manifest_file, f) + else: + print("Expected to find a model_manifest.json file, none was discovered in working directory") + def isJson(myjson): try: json_object = json.loads(myjson) - except (ValueError,TypeError) as e: + except (ValueError, TypeError) as e: return False - return True \ No newline at end of file + return True diff --git a/Algorithmia/datadirectory.py b/Algorithmia/datadirectory.py index 73838d4..f7d8193 100644 --- a/Algorithmia/datadirectory.py +++ b/Algorithmia/datadirectory.py @@ -5,14 +5,15 @@ import os import six import tempfile - import Algorithmia -from Algorithmia.datafile import DataFile + +from Algorithmia.datafile import DataFile, AdvancedDataFile, LocalDataFile from Algorithmia.data import DataObject, DataObjectType from Algorithmia.errors import DataApiError from Algorithmia.util import getParentAndBase, pathJoin from Algorithmia.acl import Acl + class DataDirectory(DataObject): def __init__(self, client, dataUrl): super(DataDirectory, self).__init__(DataObjectType.directory) @@ -41,7 +42,7 @@ def exists(self): def create(self, acl=None): '''Creates a directory, optionally include Acl argument to set permissions''' parent, name = getParentAndBase(self.path) - json = { 'name': name } + json = {'name': name} if acl is not None: json['acl'] = acl.to_api_param() response = self.client.postJsonHelper(DataDirectory._getUrl(parent), json, False) @@ -72,6 +73,15 @@ def dir(self, name): def dirs(self): return self._get_directory_iterator(DataObjectType.directory) + def getDir(self): + directory = tempfile.mkdtemp() + for file in self.files(): + correct_filename = file.getName() + correct_file_path = os.path.join(directory, correct_filename) + local_file = file.getFile(as_path=True) + os.rename(local_file, correct_file_path) + return directory + def list(self): return self._get_directory_iterator() @@ -90,7 +100,7 @@ def get_permissions(self): return None def update_permissions(self, acl): - params = {'acl':acl.to_api_param()} + params = {'acl': acl.to_api_param()} response = self.client.patchHelper(self.url, params) if response.status_code != 200: raise DataApiError('Unable to update permissions: ' + response.json()['error']['message']) @@ -102,7 +112,7 @@ def _get_directory_iterator(self, type_filter=None): while first or (marker is not None and len(marker) > 0): first = False url = self.url - query_params= {} + query_params = {} if marker: query_params['marker'] = marker response = self.client.getHelper(url, **query_params) @@ -177,8 +187,17 @@ def list(self): def dirs(self, content): for x in os.listdir(self.path): - if os.path.isdir(self.path+'/'+x): yield x + if os.path.isdir(self.path + '/' + x): yield x def files(self, content): for x in os.listdir(self.path): - if os.path.isfile(self.path+'/'+x): yield x + if os.path.isfile(self.path + '/' + x): + yield x + + +class AdvancedDataDirectory(DataDirectory): + def __init__(self, client, dataUrl): + super(AdvancedDataDirectory, self).__init__(client, dataUrl) + + def file(self, name, cleanup=True): + return AdvancedDataFile(self.client, pathJoin(self.path, name), cleanup) diff --git a/Algorithmia/datafile.py b/Algorithmia/datafile.py index 4844599..bb786e0 100644 --- a/Algorithmia/datafile.py +++ b/Algorithmia/datafile.py @@ -7,10 +7,12 @@ from datetime import datetime import os.path import pkgutil +import zipfile from Algorithmia.util import getParentAndBase from Algorithmia.data import DataObject, DataObjectType from Algorithmia.errors import DataApiError, raiseDataApiError +from io import RawIOBase class DataFile(DataObject): @@ -24,7 +26,7 @@ def __init__(self, client, dataUrl): self.size = None def set_attributes(self, attributes): - self.last_modified = datetime.strptime(attributes['last_modified'],'%Y-%m-%dT%H:%M:%S.%fZ') + self.last_modified = datetime.strptime(attributes['last_modified'], '%Y-%m-%dT%H:%M:%S.%fZ') self.size = attributes['size'] # Deprecated: @@ -32,20 +34,40 @@ def get(self): return self.client.getHelper(self.url) # Get file from the data api - def getFile(self): + def getFile(self, as_path=False): exists, error = self.existsWithError() if not exists: raise DataApiError('unable to get file {} - {}'.format(self.path, error)) # Make HTTP get request response = self.client.getHelper(self.url) - with tempfile.NamedTemporaryFile(delete = False) as f: + with tempfile.NamedTemporaryFile(delete=False) as f: for block in response.iter_content(1024): if not block: - break; + break f.write(block) f.flush() + if as_path: + return f.name + else: return open(f.name) + def getAsZip(self): + """Download/decompress file/directory and return path to file/directory. + + Expects the `DataFile` object to contain a data API path pointing to a file/directory compressed with a zip-based compression algorithm. + Either returns the directory or a path to the file, depending on whether a directory or file was zipped. + """ + local_file_path = self.getFile(as_path=True) + directory_path = tempfile.mkdtemp() + with zipfile.ZipFile(local_file_path, 'r') as ziph: + ziph.extractall(directory_path) + if len(ziph.namelist()) > 1: + output_path = directory_path + else: + filename = ziph.namelist()[0] + output_path = os.path.join(directory_path, filename) + return output_path + def getName(self): _, name = getParentAndBase(self.path) return name @@ -129,6 +151,7 @@ def putFile(self, path): raise raiseDataApiError(result) else: return self + def putNumpy(self, array): # Post numpy array as json payload np_loader = pkgutil.find_loader('numpy') @@ -140,6 +163,24 @@ def putNumpy(self, array): else: raise DataApiError("Attempted to .putNumpy() a file without numpy available, please install numpy.") + def putAsZip(self, path): + """Zip file/directory and upload to data API location defined by `DataFile` object. + + Accepts either a single file or a directory containing other files and directories. + """ + temp = tempfile.NamedTemporaryFile(delete=False).name + if os.path.isdir(path): + with zipfile.ZipFile(temp, 'w') as ziph: + for root, dirs, files in os.walk(path): + for file in files: + f_path = os.path.join(root, file) + arc_path = os.path.relpath(os.path.join(root, file), path) + ziph.write(f_path, arc_path) + else: + with zipfile.ZipFile(temp, 'w') as ziph: + ziph.write(path) + return self.putFile(temp) + def delete(self): # Delete from data api result = self.client.deleteHelper(self.url) @@ -148,6 +189,7 @@ def delete(self): else: return True + class LocalDataFile(): def __init__(self, client, filePath): self.client = client @@ -158,7 +200,7 @@ def __init__(self, client, filePath): self.size = None def set_attributes(self, attributes): - self.last_modified = datetime.strptime(attributes['last_modified'],'%Y-%m-%dT%H:%M:%S.%fZ') + self.last_modified = datetime.strptime(attributes['last_modified'], '%Y-%m-%dT%H:%M:%S.%fZ') self.size = attributes['size'] # Get file from the data api @@ -229,9 +271,78 @@ def delete(self): except: raise DataApiError('Failed to delete local file ' + self.path) + def localPutHelper(path, contents): try: with open(path, 'wb') as f: f.write(contents) return dict(status='success') - except Exception as e: return dict(error=str(e)) + except Exception as e: + return dict(error=str(e)) + + +class AdvancedDataFile(DataFile, RawIOBase): + def __init__(self, client, dataUrl, cleanup=True): + super(AdvancedDataFile, self).__init__(client, dataUrl) + self.cleanup = cleanup + self.local_file = None + + def __del__(self): + if self.local_file: + filepath = self.local_file.name + self.local_file.close() + if self.cleanup: + os.remove(filepath) + + def readable(self): + return True + + def seekable(self): + return True + + def writable(self): + return False + + def read(self, __size=None): + if not self.local_file: + self.local_file = self.getFile() + output = self.local_file.read() + elif __size: + output = self.local_file.read(__size) + else: + output = self.local_file.read() + return output + + def readline(self, __size=None): + if not self.local_file: + self.local_file = self.getFile() + with self.local_file as f: + if __size: + output = f.readline(__size) + else: + output = f.readline() + return output + + def readlines(self, __hint=None): + if not self.local_file: + self.local_file = self.getFile() + if __hint: + output = self.local_file.readlines(__hint) + else: + output = self.local_file.readlines() + return output + + def tell(self): + if not self.local_file: + self.local_file = self.getFile() + output = self.local_file.tell() + return output + + def seek(self, __offset, __whence=None): + if not self.local_file: + self.local_file = self.getFile() + if __whence: + output = self.local_file.seek(__offset, __whence) + else: + output = self.local_file.seek(__offset) + return output diff --git a/Algorithmia/errors.py b/Algorithmia/errors.py index 9356e24..22ad68e 100644 --- a/Algorithmia/errors.py +++ b/Algorithmia/errors.py @@ -36,7 +36,7 @@ def raiseAlgoApiError(result): if 'message' in result['error']: message = result['error']['message'] else: - message = None + message = result['error'] if 'error_type' in result['error']: err_type = result['error']['error_type'] else: @@ -47,4 +47,4 @@ def raiseAlgoApiError(result): stacktrace = None return AlgorithmException(message=message, stack_trace=stacktrace, error_type=err_type) else: - return Exception(result) + return Exception("Non-Algorithm related Failure: " + str(result)) diff --git a/Algorithmia/util.py b/Algorithmia/util.py index 382b586..92aa3b3 100644 --- a/Algorithmia/util.py +++ b/Algorithmia/util.py @@ -1,8 +1,10 @@ import re -from Algorithmia.errors import DataApiError +import hashlib FNAME_MATCH = re.compile(r'/([^/]+)$') # From the last slash to the end of the string -PREFIX = re.compile(r'([^:]+://)(/)?(.+)') # Check for a prefix like data:// +PREFIX = re.compile(r'([^:]+://)(/)?(.+)') # Check for a prefix like data:// + + def getParentAndBase(path): match = PREFIX.match(path) if match is None: @@ -26,7 +28,22 @@ def getParentAndBase(path): parent_path = '{prefix}{uri}'.format(prefix=prefix, uri='/'.join(parts[:-1])) return parent_path, parts[-1] + def pathJoin(parent, base): if parent.endswith('/'): return parent + base return parent + '/' + base + + +def md5_for_file(fname): + hash_md5 = hashlib.md5() + with open(fname, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hash_md5.update(chunk) + return str(hash_md5.hexdigest()) + + +def md5_for_str(content): + hash_md5 = hashlib.md5() + hash_md5.update(content.encode()) + return str(hash_md5.hexdigest()) diff --git a/README.md b/README.md index a7f3214..59a80dc 100644 --- a/README.md +++ b/README.md @@ -72,9 +72,9 @@ If the algorithm output is text, then the `result` field of the response will be ```python algo = client.algo('demo/Hello/0.1.1') response = algo.pipe("HAL 9000") -print response.result # Hello, world! -print response.metadata # Metadata(content_type='text',duration=0.0002127) -print response.metadata.duration # 0.0002127 +print(response.result) # Hello, world! +print(response.metadata) # Metadata(content_type='text',duration=0.0002127) +print(response.metadata.duration) # 0.0002127 ``` ### JSON input/output @@ -119,7 +119,7 @@ This includes support for changing the timeout or indicating that the API should ```python from Algorithmia.algorithm import OutputType response = client.algo('util/echo').set_options(timeout=60, stdout=False) -print response.metadata.stdout +print(response.metadata.stdout) ``` Note: `stdout=True` is only supported if you have access to the algorithm source. @@ -186,15 +186,15 @@ foo = client.dir("data://.my/foo") # List files in "foo" for file in foo.files(): - print file.path + " at URL: " + file.url + " last modified " + file.last_modified + print(file.path + " at URL: " + file.url + " last modified " + file.last_modified) # List directories in "foo" for file in foo.dirs(): - print dir.path + " at URL: " + file.url + print(dir.path + " at URL: " + file.url) # List everything in "foo" for entry in foo.list(): - print entry.path + " at URL: " + entry.url + print(entry.path + " at URL: " + entry.url) ``` ### Manage directory permissions @@ -230,7 +230,7 @@ $ algo auth Configuring authentication for profile: 'default' Enter API Endpoint [https://api.algorithmia.com]: Enter API Key: -(optional) enter path to custom CA certificate: +(optional) enter path to custom CA certificate: Profile is ready to use. Test with 'algo ls' ``` @@ -332,7 +332,7 @@ algo auth --profile second_user Configuring authentication for profile: 'second_user' Enter API Endpoint [https://api.algorithmia.com]: Enter API Key: -(optional) enter path to custom CA certificate: +(optional) enter path to custom CA certificate: ``` Now you may use `algo ls --profile second_user` to list files in your `second_user` account. For more information, see the auth command help with `algo auth --help`. @@ -342,7 +342,7 @@ Now you may use `algo ls --profile second_user` to list files in your `second_us When running commands, the Algorithmia CLI will use the default profile unless otherwise specified with the `--profile ` option. See the following example: ```text -$ algo run kenny/factor -d 17 --profile second_user +$ algo run kenny/factor -d 17 --profile second_user [17] ``` diff --git a/Test/CLI_test.py b/Test/CLI_test.py deleted file mode 100644 index 5ff5700..0000000 --- a/Test/CLI_test.py +++ /dev/null @@ -1,222 +0,0 @@ -import sys -# look in ../ BEFORE trying to import Algorithmia. If you append to the -# you will load the version installed on the computer. -sys.path = ['../'] + sys.path - -import unittest -import os -import json -import Algorithmia -from Algorithmia.CLI import CLI -import argparse -import shutil - -class CLITest(unittest.TestCase): - def setUp(self): - # create a directory to use in testing the cp command - self.client = Algorithmia.client() - CLI().mkdir("data://.my/moredata", self.client) - if(not os.path.exists("./TestFiles/")): - os.mkdir("./TestFiles/") - - def test_ls(self): - parentDir = "data://.my/" - newDir = "test" - - CLI().mkdir(parentDir+newDir, self.client) - result = CLI().ls(parentDir, self.client) - self.assertTrue(result is not None and "moredata" in result and newDir in result) - - CLI().rmdir(parentDir+newDir, self.client) - - - def test_mkdir(self): - - parentDir = "data://.my/" - newDir = "test" - - CLI().mkdir(parentDir+newDir, self.client) - result = CLI().ls(parentDir, self.client) - self.assertTrue(newDir in result) - - CLI().rmdir(parentDir+newDir, self.client) - - def test_rmdir(self): - parentDir = "data://.my/" - newDir = "testRmdir" - - CLI().mkdir(parentDir+newDir, self.client) - result = CLI().ls(parentDir, self.client) - self.assertTrue(newDir in result) - - CLI().rmdir(parentDir+newDir, self.client) - - result = CLI().ls(parentDir, self.client) - self.assertTrue(newDir not in result) - - def test_cat(self): - file = "data://.my/moredata/test.txt" - localfile = "./TestFiles/test.txt" - fileContents = "some text in test file" - - CLI().rm(file, self.client) - testfile = open(localfile, "w") - testfile.write(fileContents) - testfile.close() - - CLI().cp([localfile],file,self.client) - - result = CLI().cat([file],self.client) - self.assertEqual(result, fileContents) - - def test_get_build_logs(self): - user=os.environ.get('ALGO_USER_NAME') - algo="Echo" - - result = json.loads(CLI().getBuildLogs(user,algo,self.client)) - if "error" in result: - print(result) - self.assertTrue("error" not in result) - - -#local to remote - def test_cp_L2R(self): - localfile = "./TestFiles/test.txt" - testfile = open(localfile, "w") - testfile.write("some text") - testfile.close() - - src = [localfile] - dest = "data://.my/moredata/test.txt" - CLI().cp(src,dest,self.client) - - result = CLI().ls("data://.my/moredata/",self.client) - self.assertTrue("test.txt" in result) - -#remote to remote - def test_cp_R2R(self): - - src = ["data://.my/moredata/test.txt"] - dest = "data://.my/moredata/test2.txt" - CLI().cp(src,dest,self.client) - - result = CLI().ls("data://.my/moredata/",self.client) - self.assertTrue("test2.txt" in result) - -#remote to local - def test_cp_R2L(self): - src = ["data://.my/moredata/test.txt"] - dest = "./test.txt" - - CLI().cp(src,dest,self.client) - self.assertTrue(os.path.isfile(dest)) - - def test_run(self): - name = "util/Echo" - inputs = "test" - - parser = argparse.ArgumentParser('CLI for interacting with Algorithmia') - - subparsers = parser.add_subparsers(help = 'sub cmd',dest = 'subparser_name') - parser_run = subparsers.add_parser('run', help = 'algo run [input options] [output options]') - - parser_run.add_argument('algo') - parser_run.add_argument('-d','--data', action = 'store', help = 'detect input type', default = None) - parser_run.add_argument('-t','--text', action = 'store', help = 'treat input as text', default = None) - parser_run.add_argument('-j','--json', action = 'store', help = 'treat input as json data', default = None) - parser_run.add_argument('-b','--binary', action = 'store', help = 'treat input as binary data', default = None) - parser_run.add_argument('-D','--data-file', action = 'store', help = 'specify a path to an input file', default = None) - parser_run.add_argument('-T','--text-file', action = 'store', help = 'specify a path to a text file', default = None) - parser_run.add_argument('-J','--json-file', action = 'store', help = 'specify a path to a json file', default = None) - parser_run.add_argument('-B','--binary-file', action = 'store', help = 'specify a path to a binary file', default = None) - parser_run.add_argument('--timeout', action = 'store',type = int, default = 300, help = 'specify a timeout (seconds)') - parser_run.add_argument('--debug', action = 'store_true', help = 'print the stdout from the algo ') - parser_run.add_argument('--profile', action = 'store', type = str, default = 'default') - parser_run.add_argument('-o', '--output', action = 'store', default = None, type = str) - - args = parser.parse_args(['run',name,'-d',inputs]) - - result = CLI().runalgo(args, self.client) - self.assertEqual(result, inputs) - - def test_auth(self): - #key for test account - key = os.getenv('ALGORITHMIA_API_KEY') - address = 'apiAddress' - profile = 'default' - CLI().auth(key,address,profile=profile) - resultK = CLI().getAPIkey(profile) - resultA = CLI().getAPIaddress(profile) - self.assertEqual(resultK, key) - self.assertEqual(resultA, address) - - def test_auth_cert(self): - - localfile = "./TestFiles/fakecert.pem" - - testfile = open(localfile, "w") - testfile.write("") - testfile.close() - - #key for test account - key = os.getenv('ALGORITHMIA_API_KEY') - address = 'apiAddress' - cacert = localfile - profile = 'test' - - CLI().auth(key,address,cacert,profile) - resultK = CLI().getAPIkey(profile) - resultA = CLI().getAPIaddress(profile) - resultC = CLI().getCert(profile) - self.assertEqual(resultK, key) - self.assertEqual(resultA, address) - self.assertEqual(resultC, cacert) - - def test_get_environment(self): - result = CLI().get_environment_by_language("python2",self.client) - print(result) - if("error" in result): - print(result) - self.assertTrue(result is not None and "display_name" in result) - - def test_list_languages(self): - result = CLI().list_languages(self.client) - if("error" in result): - print(result) - self.assertTrue(result is not None and "name" in result[0]) - - - def test_rm(self): - localfile = "./TestFiles/testRM.txt" - - testfile = open(localfile, "w") - testfile.write("some text") - testfile.close() - - src = [localfile] - dest = "data://.my/moredata/" - CLI().cp(src,dest,self.client) - - result1 = CLI().ls(dest,self.client) - - CLI().rm("data://.my/moredata/testRM.txt",self.client) - - result2 = CLI().ls(dest,self.client) - - self.assertTrue("testRM.txt" in result1 and "testRM.txt" not in result2) - - def test_get_template(self): - filename = "./temptest" - envid = "36fd467e-fbfe-4ea6-aa66-df3f403b7132" - response = CLI().get_template(envid,filename,self.client) - print(response) - self.assertTrue(response.ok) - try: - shutil.rmtree(filename) - except OSError as e: - print(e) - - - -if __name__ == '__main__': - unittest.main() \ No newline at end of file diff --git a/Test/algo_test.py b/Test/algo_test.py deleted file mode 100644 index b14e99b..0000000 --- a/Test/algo_test.py +++ /dev/null @@ -1,77 +0,0 @@ -import sys -import os -from Algorithmia.errors import AlgorithmException -# look in ../ BEFORE trying to import Algorithmia. If you append to the -# you will load the version installed on the computer. -sys.path = ['../'] + sys.path - -import unittest - -import Algorithmia - -class AlgoTest(unittest.TestCase): - def setUp(self): - self.client = Algorithmia.client() - - def test_call_customCert(self): - open("./test.pem",'w') - c = Algorithmia.client(ca_cert="./test.pem") - result = c.algo('util/Echo').pipe(bytearray('foo','utf-8')) - self.assertEquals('binary', result.metadata.content_type) - self.assertEquals(bytearray('foo','utf-8'), result.result) - try: - os.remove("./test.pem") - except OSError as e: - print(e) - - def test_call_binary(self): - result = self.client.algo('util/Echo').pipe(bytearray('foo','utf-8')) - self.assertEquals('binary', result.metadata.content_type) - self.assertEquals(bytearray('foo','utf-8'), result.result) - - def test_text_unicode(self): - telephone = u"\u260E" - - #Unicode input to pipe() - result1 = self.client.algo('util/Echo').pipe(telephone) - self.assertEquals('text', result1.metadata.content_type) - self.assertEquals(telephone, result1.result) - - #Unicode return in .result - result2 = self.client.algo('util/Echo').pipe(result1.result) - self.assertEquals('text', result2.metadata.content_type) - self.assertEquals(telephone, result2.result) - - def test_get_build_by_id(self): - result = self.client.algo("J_bragg/Echo").get_build("1a392e2c-b09f-4bae-a616-56c0830ac8e5") - self.assertTrue(result.build_id is not None) - - def test_get_build_logs(self): - result = self.client.algo("J_bragg/Echo").get_build_logs("1a392e2c-b09f-4bae-a616-56c0830ac8e5") - self.assertTrue(result.logs is not None) - - def test_get_scm_status(self): - result = self.client.algo("J_bragg/Echo").get_scm_status() - self.assertTrue(result.scm_connection_status is not None) - - def test_exception_ipa_algo(self): - try: - result = self.client.algo('zeryx/raise_exception').pipe("") - except AlgorithmException as e: - self.assertEqual(e.message, "This is an exception") - - # def test_json_unicode(self): - # telephone = [u"\u260E"] - # - # #Unicode input to pipe() - # result1 = self.client.algo('util/Echo').pipe(telephone) - # self.assertEquals('json', result1.metadata.content_type) - # self.assertEquals(telephone, result1.result) - # - # #Unicode return in .result - # result2 = self.client.algo('util/Echo').pipe(result1.result) - # self.assertEquals('json', result2.metadata.content_type) - # self.assertEquals(telephone, result2.result) - -if __name__ == '__main__': - unittest.main() diff --git a/Test/api/__init__.py b/Test/api/__init__.py index 5ca3185..c5ff73e 100644 --- a/Test/api/__init__.py +++ b/Test/api/__init__.py @@ -1,16 +1,3 @@ -import importlib -from fastapi import FastAPI, Response - -app = FastAPI() - -@app.post("/v1/{username}/{algoname}/{version}") -async def throw_error(username, algoname, version): - return Response("Internal Server Error", status_code=500) - - -def create_endpoint(algoname): - module = importlib.import_module(algoname) - @app.get("/invocations") - def invocations(data): - return module.apply(data) +from .app import start_webserver_reg as start_webserver_reg +from .self_signed_app import start_webserver_self_signed as start_webserver_self_signed diff --git a/Test/api/app.py b/Test/api/app.py new file mode 100644 index 0000000..db7efd2 --- /dev/null +++ b/Test/api/app.py @@ -0,0 +1,549 @@ +from fastapi import FastAPI, Request, status +from typing import Optional +from fastapi.responses import Response, JSONResponse +import json +import base64 +from multiprocessing import Process +import uvicorn + +regular_app = FastAPI() + + +def start_webserver_reg(): + def _start_webserver(): + uvicorn.run(regular_app, host="127.0.0.1", port=8080, log_level="debug") + + p = Process(target=_start_webserver) + p.start() + return p + + +@regular_app.post("/v1/algo/{username}/{algoname}") +async def process_algo_req(request: Request, username, algoname, output: Optional[str] = None): + metadata = {"request_id": "req-55c0480d-6af3-4a21-990a-5c51d29f5725", "duration": 0.000306774} + content_type = request.headers['Content-Type'] + auth = request.headers.get('Authorization', None) + if auth is None: + return {"error": {"message": "authorization required"}} + request = await request.body() + if output and output == "void": + return {"async": "abcd123", "request_id": "req-55c0480d-6af3-4a21-990a-5c51d29f5725"} + elif output and output == "raw": + return Response(request.decode(), status_code=200) + elif algoname == "500": + return Response("Internal Server Error", status_code=500) + elif algoname == "raise_exception": + return {"error": {"message": "This is an exception"}} + else: + if content_type != "application/octet-stream": + request = request.decode('utf-8') + if content_type == "text/plain": + metadata['content_type'] = "text" + elif content_type == "application/json": + request = json.loads(request) + metadata['content_type'] = "json" + else: + metadata['content_type'] = "binary" + request = base64.b64encode(request) + output = {"result": request, "metadata": metadata} + return output + + +@regular_app.post("/v1/algo/{username}/{algoname}/{githash}", status_code=status.HTTP_200_OK) +async def process_hello_world(request: Request, username, algoname, githash): + metadata = {"request_id": "req-55c0480d-6af3-4a21-990a-5c51d29f5725", "duration": 0.000306774, + 'content_type': "text"} + request = await request.body() + request = request.decode('utf-8') + return {"result": f"hello {request}", "metadata": metadata} + + +### Algorithm Routes +@regular_app.get('/v1/algorithms/{username}/{algoname}') +async def process_get_algo(username, algoname): + if algoname == "echo" and username == 'quality': + return {"id": "21df7a38-eab8-4ac8-954c-41a285535e69", "name": "echo", + "details": {"summary": "", "label": "echo", "tagline": ""}, + "settings": {"algorithm_callability": "public", "source_visibility": "closed", + "package_set": "python36", "license": "apl", "royalty_microcredits": 0, + "network_access": "full", "pipeline_enabled": True, "insights_enabled": False, + "algorithm_environment": "067110e7-8969-4441-b3d6-5333f18a3db3"}, + "version_info": {"semantic_version": "0.1.0", "git_hash": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "version_uuid": "e06d2808-bb5e-46ae-b7bc-f3d9968e3c6b"}, + "build": {"build_id": "a9ae2c93-6f4e-42c0-ac54-baa4a66e53d3", "status": "succeeded", + "commit_sha": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "started_at": "2022-05-08T22:43:09.050Z", "finished_at": "2022-05-08T22:43:28.646Z", + "version_info": {"semantic_version": "0.1.0"}, "resource_type": "algorithm_build"}, + "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, + "compilation": {"successful": True, "output": ""}, + "self_link": "https://api.algorithmia.com/v1/algorithms/quality/echo/versions/0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "resource_type": "algorithm"} + elif algoname == "echo": + return JSONResponse(content={"error": {"id": "1cfb98c5-532e-4cbf-9192-fdd45b86969c", "code": 2001, + "message": "Caller is not authorized to perform the operation"}}, + status_code=403) + else: + return JSONResponse(content={"error": "No such algorithm"}, status_code=404) + + +@regular_app.get("/v1/algorithms/{username}/{algoname}/builds/{buildid}") +async def get_build_id(username, algoname, buildid): + return {"status": "succeeded", "build_id": buildid, "commit_sha": "bcdadj", + "started_at": "2021-09-27T22:54:20.786Z", "finished_at": "2021-09-27T22:54:40.898Z", + "version_info": {"semantic_version": "0.1.1"}} + + +@regular_app.get("/v1/algorithms/{username}/{algoname}/builds/{buildid}/logs") +async def get_build_log(username, algoname, buildid): + return {"logs": "This is a log"} + + +@regular_app.get("/v1/algorithms/{username}/{algoname}/scm/status") +async def get_scm_status(username, algoname): + return {"scm_connection_status": "active"} + + +@regular_app.get("/v1/scms") +async def get_scms(): + return {'results': [{'default': True, 'enabled': True, 'id': 'internal', 'name': '', 'provider': 'internal'}, + {'default': False, 'enabled': True, 'id': 'github', 'name': 'https://github.com', + 'provider': 'github', 'scm': {'client_id': '0ff25ba21ec67dbed6e2'}, + 'oauth': {'client_id': '0ff25ba21ec67dbed6e2'}, + 'urls': {'web': 'https://github.com', 'api': 'https://api.github.com', + 'ssh': 'ssh://git@github.com'}}, + {'default': False, 'enabled': True, 'id': 'aadebe70-007f-48ff-ba38-49007c6e0377', + 'name': 'https://gitlab.com', 'provider': 'gitlab', + 'scm': {'client_id': 'ca459576279bd99ed480236a267cc969f8322caad292fa5147cc7fdf7b530a7e'}, + 'oauth': {'client_id': 'ca459576279bd99ed480236a267cc969f8322caad292fa5147cc7fdf7b530a7e'}, + 'urls': {'web': 'https://gitlab.com', 'api': 'https://gitlab.com', + 'ssh': 'ssh://git@gitlab.com'}}, + {'default': False, 'enabled': True, 'id': '24ad1496-5a1d-43e2-9d96-42fce8e5484f', + 'name': 'IQIVA Public GitLab', 'provider': 'gitlab', + 'scm': {'client_id': '3341c989f9d28043d2597388aa4f43ce60a74830b981c4b7d79becf641959376'}, + 'oauth': {'client_id': '3341c989f9d28043d2597388aa4f43ce60a74830b981c4b7d79becf641959376'}, + 'urls': {'web': 'https://gitlab.com', 'api': 'https://gitlab.com', + 'ssh': 'ssh://git@gitlab.com'}}, + {'default': False, 'enabled': False, 'id': '83cd96ae-b1f4-4bd9-b9ca-6f7f25c37708', + 'name': 'GitlabTest', 'provider': 'gitlab', + 'scm': {'client_id': '5e257d6e168d579d439b7d38cdfa647e16573ae1dace6d93a30c5c60b4e5dd32'}, + 'oauth': {'client_id': '5e257d6e168d579d439b7d38cdfa647e16573ae1dace6d93a30c5c60b4e5dd32'}, + 'urls': {'web': 'https://gitlab.com', 'api': 'https://gitlab.com', + 'ssh': 'ssh://git@gitlab.com'}}]} + + +@regular_app.get("/v1/algorithms/{algo_id}/errors") +async def get_algo_errors(algo_id): + return JSONResponse(content={"error": {"message": "not found"}}, status_code=404) + + +@regular_app.post("/v1/algorithms/{username}") +async def create_algorithm(request: Request, username): + payload = await request.json() + return {"id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", "name": payload["name"], + "details": {"label": payload["details"]["label"]}, + "settings": {"algorithm_callability": "private", "source_visibility": "open", + "package_set": "tensorflow-gpu-2.3-python38", "license": "apl", "network_access": "isolated", + "pipeline_enabled": False, "insights_enabled": False, + "algorithm_environment": "fd980f4f-1f1c-4b2f-a128-d60b40c6567a"}, + "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, + "resource_type": "algorithm"} + + +@regular_app.put('/v1/algorithms/{username}/{algoname}') +async def update_algorithm(request: Request, username, algoname): + return { + "id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", + "name": algoname, + "details": { + "summary": "Example Summary", + "label": "QA", + "tagline": "Example Tagline" + }, + "settings": { + "algorithm_callability": "private", + "source_visibility": "open", + "package_set": "tensorflow-gpu-2.3-python38", + "license": "apl", + "network_access": "isolated", + "pipeline_enabled": False, + "insights_enabled": False, + "algorithm_environment": "fd980f4f-1f1c-4b2f-a128-d60b40c6567a" + }, + "version_info": { + "git_hash": "e85db9bca2fad519f540b445f30d12523e4dec9c", + "version_uuid": "1d9cb91d-11ca-49cb-a7f4-28f67f277654" + }, + "source": { + "scm": { + "id": "internal", + "provider": "internal", + "default": True, + "enabled": True + } + }, + "compilation": { + "successful": True, + "output": "" + }, + "self_link": f"http://localhost:8080/v1/algorithms/{username}/{algoname}/versions/e85db9bca2fad519f540b445f30d12523e4dec9c", + "resource_type": "algorithm" + } + + +@regular_app.post("/v1/algorithms/{username}/{algoname}/compile") +async def compile_algorithm(username, algoname): + return { + "id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", + "name": algoname, + "details": { + "summary": "Example Summary", + "label": "QA", + "tagline": "Example Tagline" + }, + "settings": { + "algorithm_callability": "private", + "source_visibility": "open", + "package_set": "tensorflow-gpu-2.3-python38", + "license": "apl", + "network_access": "isolated", + "pipeline_enabled": False, + "insights_enabled": False, + "algorithm_environment": "fd980f4f-1f1c-4b2f-a128-d60b40c6567a" + }, + "version_info": { + "git_hash": "e85db9bca2fad519f540b445f30d12523e4dec9c", + "version_uuid": "1d9cb91d-11ca-49cb-a7f4-28f67f277654" + }, + "source": { + "scm": { + "id": "internal", + "provider": "internal", + "default": True, + "enabled": True + } + }, + "compilation": { + "successful": True, + "output": "" + }, + "self_link": f"http://localhost:8080/v1/algorithms/{username}/{algoname}/versions/e85db9bca2fad519f540b445f30d12523e4dec9c", + "resource_type": "algorithm" + } + +fail_cnt = 0 + +@regular_app.post("/v1/algorithms/{username}/{algoname}/versions") +async def publish_algorithm(request: Request, username, algoname): + global fail_cnt + if "failonce" == algoname and fail_cnt == 0: + fail_cnt +=1 + return JSONResponse(content="This is an expected failure mode, try again", status_code=400) + elif "failalways" == algoname: + return JSONResponse(status_code=500) + return {"id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", "name": algoname, + "details": {"summary": "Example Summary", "label": "QA", "tagline": "Example Tagline"}, + "settings": {"algorithm_callability": "private", "source_visibility": "open", + "package_set": "tensorflow-gpu-2.3-python38", "license": "apl", "network_access": "isolated", + "pipeline_enabled": False, "insights_enabled": False, + "algorithm_environment": "fd980f4f-1f1c-4b2f-a128-d60b40c6567a"}, + "version_info": {"semantic_version": "0.1.0", "git_hash": "e85db9bca2fad519f540b445f30d12523e4dec9c", + "release_notes": "created programmatically", "sample_input": "payload", + "version_uuid": "e85db9bca2fad519f540b445f30d12523e4dec9c"}, + "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, + "compilation": {"successful": True}, + "self_link": f"http://localhost:8080/v1/algorithms/{username}/{algoname}/versions/e85db9bca2fad519f540b445f30d12523e4dec9c", + "resource_type": "algorithm"} + + +@regular_app.get("/v1/algorithms/{username}/{algoname}/versions") +async def versions_of_algorithm(request: Request, username, algoname): + return {"marker": None, "next_link": None, "results": [ + {"id": "21df7a38-eab8-4ac8-954c-41a285535e69", "name": algoname, + "details": {"summary": "", "label": algoname, "tagline": ""}, + "settings": {"algorithm_callability": "public", "source_visibility": "closed", "package_set": "python36", + "license": "apl", "royalty_microcredits": 0, "network_access": "full", "pipeline_enabled": True, + "insights_enabled": False, "algorithm_environment": "067110e7-8969-4441-b3d6-5333f18a3db3"}, + "version_info": {"semantic_version": "0.1.0", "git_hash": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "version_uuid": "e06d2808-bb5e-46ae-b7bc-f3d9968e3c6b"}, + "build": {"build_id": "a9ae2c93-6f4e-42c0-ac54-baa4a66e53d3", "status": "succeeded", + "commit_sha": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", "started_at": "2022-05-08T22:43:09.050Z", + "finished_at": "2022-05-08T22:43:28.646Z", "version_info": {"semantic_version": "0.1.0"}, + "resource_type": "algorithm_build"}, + "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, + "compilation": {"successful": True}, + "self_link": f"https://api.algorithmia.com/v1/algorithms/{username}/{algoname}/versions" + "/0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "resource_type": "algorithm"}]} + + +@regular_app.get("/v1/algorithms/{username}/{algoname}/versions/{algohash}") +async def get_algorithm_info(username, algoname, algohash): + if algohash == "e85db9bca2fad519f540b445f30d12523e4dec9c": + return {"id": "21df7a38-eab8-4ac8-954c-41a285535e69", "name": algoname, + "details": {"summary": "", "label": algoname, "tagline": ""}, + "settings": {"algorithm_callability": "public", "source_visibility": "closed", "language": "python3", + "environment": "cpu", "package_set": "python36", "license": "apl", + "royalty_microcredits": 0, "network_access": "full", "pipeline_enabled": True, + "insights_enabled": False, + "algorithm_environment": "067110e7-8969-4441-b3d6-5333f18a3db3"}, + "version_info": {"semantic_version": "0.1.0", "git_hash": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "version_uuid": "e06d2808-bb5e-46ae-b7bc-f3d9968e3c6b"}, + "build": {"build_id": "a9ae2c93-6f4e-42c0-ac54-baa4a66e53d3", "status": "succeeded", + "commit_sha": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "started_at": "2022-05-08T22:43:09.050Z", "finished_at": "2022-05-08T22:43:28.646Z", + "version_info": {"semantic_version": "0.1.0"}, "resource_type": "algorithm_build"}, + "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, + "compilation": {"successful": True, "output": ""}, "resource_type": "algorithm"} + else: + return JSONResponse(content={"error": {"message": "not found"}}, status_code=404) + + +### Admin Routes +@regular_app.post("/v1/users") +async def create_user(request: Request): + payload = await request.body() + data = json.loads(payload) + username = data['username'] + email = data['email'] + return { + "id": "1e5c89ab-3d5c-4bad-b8a3-6c8a294d4418", + "username": username, + "email": email, + "fullname": username, + "self_link": f"http://localhost:8080/v1/users/{username}", "resource_type": "user" + } + + +@regular_app.get("/v1/users/{user_id}/errors") +async def get_user_errors(user_id): + return [] + + +@regular_app.get("/v1/organization/types") +async def get_org_types(): + return [ + {"id": "d0c85ea6-ddfa-11ea-a0c8-12a811be4db3", "name": "basic"}, + {"id": "d0bff917-ddfa-11ea-a0c8-12a811be4db3", "name": "legacy"}, + {"id": "d0c9d825-ddfa-11ea-a0c8-12a811be4db3", "name": "pro"} + ] + + +@regular_app.post("/v1/organizations") +async def create_org(request: Request): + payload = await request.body() + data = json.loads(payload) + org_name = data["org_name"] + org_email = data["org_email"] + return {"id": "55073c92-5f8e-4d7e-a14d-568f94924fd9", + "org_name": org_name, + "org_label": "some label", + "org_contact_name": "Some owner", + "org_email": org_email, + "org_created_at": "2021-10-22T16:41:32", + "org_url": None, + "type_id": "d0c85ea6-ddfa-11ea-a0c8-12a811be4db3", + "stripe_customer_id": None, + "external_admin_group": None, + "external_member_group": None, + "external_id": None, + "owner_ids": None, + "resource_type": "organization", + "self_link": "http://localhost:8080/v1/organizations/a_myOrg1542" + } + + +@regular_app.put("/v1/organizations/{orgname}/members/{username}") +async def add_user_to_org(orgname, username): + return Response(status_code=200) + + +@regular_app.get("/v1/organizations/{orgname}/errors") +async def org_errors(orgname): + return [] + + +@regular_app.put("/v1/organizations/{org_name}") +async def edit_org(org_name): + return Response(status_code=204) + + +@regular_app.get("/v1/organizations/{org_name}") +async def get_org_by_name(org_name): + return { + "id": "55073c92-5f8e-4d7e-a14d-568f94924fd9", + "org_name": org_name, + "org_label": "some label", + "org_contact_name": "Some owner", + "org_email": "a_myOrg1542@algo.com", + "org_created_at": "2021-10-22T16:41:32", + "org_url": None, + "type_id": "d0c85ea6-ddfa-11ea-a0c8-12a811be4db3", + "stripe_customer_id": None, + "external_admin_group": None, + "external_member_group": None, + "external_id": None, + "owner_ids": None, + "resource_type": "organization", + "self_link": "http://localhost:8080/v1/organizations/a_myOrg1542" + } + + +@regular_app.get("/v1/algorithms/{username}/{algoname}/builds/{buildid}/logs") +async def get_build_log(username, algoname, buildid): + return {"logs": "This is a log"} + + +@regular_app.get("/v1/algorithm-environments/edge/languages") +async def get_supported_langs(): + return [{"name": "anaconda3", "display_name": "Conda (Environments) - beta", + "configuration": "{\n \"display_name\": \"Conda (Environments) - beta\",\n \"req_files\": [\n \"environment.yml\"\n ],\n \"artifacts\": [\n {\"source\":\"/home/algo/.cache\", \"destination\":\"/home/algo/.cache/\"},\n {\"source\":\"/home/algo/anaconda_environment\", \"destination\": \"/home/algo/anaconda_environment/\"},\n {\"source\":\"/opt/algorithm\", \"destination\":\"/opt/algorithm/\"}\n ]\n}\n"}, + {"name": "csharp-dotnet-core2", "display_name": "C# .NET Core 2.x+ (Environments)", + "configuration": "{\n \"display_name\": \"C# .NET Core 2.x+ (Environments)\",\n \"artifacts\": [\n {\"source\":\"/opt/algorithm/bin/Release/*/*\", \"destination\":\"/opt/algorithm/\"},\n {\"source\":\"/opt/algorithm/resources\", \"destination\":\"/opt/algorithm/resources/\"},\n {\"source\":\"/home/algo/.nuget\", \"destination\":\"/home/algo/.nuget/\"}\n ]\n}\n"}, + {"name": "java11", "display_name": "Java OpenJDK 11.0 (Environments)", + "configuration": "{\n \"display_name\": \"Java OpenJDK 11.0 (Environments)\",\n \"artifacts\": [\n {\"source\":\"/opt/algorithm/target/*.jar\", \"destination\":\"/opt/algorithm/target/algorithm.jar\"},\n {\"source\":\"/opt/algorithm/target/lib\", \"destination\":\"/opt/algorithm/target/lib/\"}\n ]\n}\n"}, + {"name": "python2", "display_name": "Python 2.x (Environments)", + "configuration": "{\n \"display_name\": \"Python 2.x (Environments)\",\n \"req_files\": [\n \"requirements.txt\"\n ],\n \"artifacts\": [\n {\"source\":\"/home/algo/.local\", \"destination\":\"/home/algo/.local/\"},\n {\"source\":\"/opt/algorithm\", \"destination\":\"/opt/algorithm/\"}\n ]\n}\n"}, + {"name": "python3", "display_name": "Python 3.x (Environments)", + "configuration": "{\n \"display_name\": \"Python 3.x (Environments)\",\n \"req_files\": [\n \"requirements.txt\"\n ],\n \"artifacts\": [\n {\"source\":\"/home/algo/.local\", \"destination\":\"/home/algo/.local/\"},\n {\"source\":\"/opt/algorithm\", \"destination\":\"/opt/algorithm/\"}\n ]\n}\n"}, + {"name": "r36", "display_name": "R 3.6.x (Environments)", + "configuration": "{\n \"display_name\": \"R 3.6.x (Environments)\",\n \"req_files\": [\n \"packages.txt\"\n ],\n \"artifacts\": [\n {\"source\":\"/opt/algorithm\", \"destination\":\"/opt/algorithm/\"},\n {\"source\":\"/usr/local/lib/R/site-library\", \"destination\":\"/usr/local/lib/R/site-library/\"}\n ]\n}\n\n"}, + {"name": "scala-2", "display_name": "Scala 2.x & sbt 1.3.x (Environments)", + "configuration": "{\n \"display_name\": \"Scala 2.x & sbt 1.3.x (Environments)\",\n \"artifacts\": [\n {\"source\":\"/opt/algorithm/target/universal/stage\", \"destination\":\"/opt/algorithm/stage/\"}\n ]\n}\n\n"}] + + +@regular_app.get("/v1/algorithm-environments/edge/languages/{language}/environments") +async def get_environments_by_lang(language): + return { + "environments": [ + { + "id": "717d36e0-222c-44a0-9aa8-06f4ebc1b82a", + "environment_specification_id": "f626effa-e519-431e-9d7a-0d3a7563ae1e", + "display_name": "Python 2.7", + "description": "Generic Python 2.7 installation", + "created_at": "2020-12-21T21:47:53.239", + "language": { + "name": language, + "display_name": "Python 2.x (Environments)", + "configuration": "{\n \"display_name\": \"Python 2.x (Environments)\",\n \"req_files\": [\n " + " \"requirements.txt\"\n ],\n \"artifacts\": [\n {" + "\"source\":\"/home/algo/.local\", \"destination\":\"/home/algo/.local/\"}," + "\n {\"source\":\"/opt/algorithm\", " + "\"destination\":\"/opt/algorithm/\"}\n ]\n}\n " + }, + "machine_type": "CPU" + }, + { + "id": "6f57e041-54e0-4e1a-8b2f-4589bb2c06f8", + "environment_specification_id": "faf81400-eb15-4f64-81c0-3d4ed7181e77", + "display_name": "Python 2.7 + GPU support", + "description": "Python2.7 installation with CUDA 9.0 and CUDNN7", + "created_at": "2020-08-14T07:22:32.955", + "language": { + "name": language, + "display_name": "Python 2.x (Environments)", + "configuration": "{\n \"display_name\": \"Python 2.x (Environments)\",\n \"req_files\": [\n " + " \"requirements.txt\"\n ],\n \"artifacts\": [\n {" + "\"source\":\"/home/algo/.local\", \"destination\":\"/home/algo/.local/\"}," + "\n {\"source\":\"/opt/algorithm\", " + "\"destination\":\"/opt/algorithm/\"}\n ]\n}\n " + }, + "machine_type": "GPU" + } + ] + } + + +@regular_app.get("/v1/secret-provider") +async def get_service_providers(): + return [ + { + "id": "dee00b6c-05c4-4de7-98d8-e4a3816ed75f", + "name": "algorithmia_internal_secret_provider", + "description": "Internal Secret Provider", + "moduleName": "module", + "factoryClassName": "com.algorithmia.plugin.sqlsecretprovider.InternalSecretProviderFactory", + "interfaceVersion": "1.0", + "isEnabled": True, + "isDefault": True, + "created": "2021-03-11T20:42:23Z", + "modified": "2021-03-11T20:42:23Z" + } + ] + + +@regular_app.get("/v1/algorithms/{algorithm_id}/secrets") +async def get_secrets_for_algorithm(algorithm_id): + return { + "secrets": [ + { + "id": "45e97c47-3ae6-46be-87ee-8ab23746706b", + "short_name": "MLOPS_SERVICE_URL", + "description": "", + "secret_key": "MLOPS_SERVICE_URL", + "owner_type": "algorithm", + "owner_id": "fa2cd80b-d22a-4548-b16a-45dbad2d3499", + "provider_id": "dee00b6c-05c4-4de7-98d8-e4a3816ed75f", + "created": "2022-07-22T14:36:01Z", + "modified": "2022-07-22T14:36:01Z" + }, + { + "id": "50dca60e-317f-4582-8854-5b83b4d182d0", + "short_name": "deploy_id", + "description": "", + "secret_key": "DEPLOYMENT_ID", + "owner_type": "algorithm", + "owner_id": "fa2cd80b-d22a-4548-b16a-45dbad2d3499", + "provider_id": "dee00b6c-05c4-4de7-98d8-e4a3816ed75f", + "created": "2022-07-21T19:04:31Z", + "modified": "2022-07-21T19:04:31Z" + }, + { + "id": "5a75cdc8-ecc8-4715-8c4b-8038991f1608", + "short_name": "model_path", + "description": "", + "secret_key": "MODEL_PATH", + "owner_type": "algorithm", + "owner_id": "fa2cd80b-d22a-4548-b16a-45dbad2d3499", + "provider_id": "dee00b6c-05c4-4de7-98d8-e4a3816ed75f", + "created": "2022-07-21T19:04:31Z", + "modified": "2022-07-21T19:04:31Z" + }, + { + "id": "80e51ed3-f6db-419d-9349-f59f4bbfdcbb", + "short_name": "model_id", + "description": "", + "secret_key": "MODEL_ID", + "owner_type": "algorithm", + "owner_id": "fa2cd80b-d22a-4548-b16a-45dbad2d3499", + "provider_id": "dee00b6c-05c4-4de7-98d8-e4a3816ed75f", + "created": "2022-07-21T19:04:30Z", + "modified": "2022-07-21T19:04:30Z" + }, + { + "id": "8773c654-ea2f-4ac5-9ade-55dfc47fec9d", + "short_name": "datarobot_api_token", + "description": "", + "secret_key": "DATAROBOT_MLOPS_API_TOKEN", + "owner_type": "algorithm", + "owner_id": "fa2cd80b-d22a-4548-b16a-45dbad2d3499", + "provider_id": "dee00b6c-05c4-4de7-98d8-e4a3816ed75f", + "created": "2022-07-21T19:04:31Z", + "modified": "2022-07-21T19:04:31Z" + } + ] + } + + +@regular_app.post("/v1/algorithms/{algorithm_id}/secrets") +async def set_algorithm_secret(algorithm_id): + return { + "id":"959af771-7cd8-4981-91c4-70def15bbcdc", + "short_name":"tst", + "description":"", + "secret_key":"test", + "owner_type":"algorithm", + "owner_id":"fa2cd80b-d22a-4548-b16a-45dbad2d3499", + "provider_id":"dee00b6c-05c4-4de7-98d8-e4a3816ed75f", + "created":"2022-07-22T18:28:42Z", + "modified":"2022-07-22T18:28:42Z" +} \ No newline at end of file diff --git a/Test/api/self_signed_app.py b/Test/api/self_signed_app.py new file mode 100644 index 0000000..693d486 --- /dev/null +++ b/Test/api/self_signed_app.py @@ -0,0 +1,374 @@ +from fastapi import FastAPI, Request +from typing import Optional +from fastapi.responses import Response +import json +import base64 +from multiprocessing import Process +import uvicorn + +self_signed_app = FastAPI() + + +def start_webserver_self_signed(): + def _start_webserver(): + uvicorn.run(self_signed_app, host="127.0.0.1", port=8090, log_level="debug", + ssl_certfile="Test/resources/cert.cert", ssl_keyfile="Test/resources/cert.key") + + p = Process(target=_start_webserver) + p.start() + return p + + +@self_signed_app.post("/v1/algo/{username}/{algoname}") +async def process_algo_req(request: Request, username, algoname, output: Optional[str] = None): + metadata = {"request_id": "req-55c0480d-6af3-4a21-990a-5c51d29f5725", "duration": 0.000306774} + content_type = request.headers['Content-Type'] + auth = request.headers.get('Authorization', None) + if auth is None: + return {"error": {"message": "authorization required"}} + request = await request.body() + if output and output == "void": + return {"async": "abcd123", "request_id": "req-55c0480d-6af3-4a21-990a-5c51d29f5725"} + elif output and output == "raw": + return Response(request.decode(), status_code=200) + elif algoname == "500": + return Response("Internal Server Error", status_code=500) + elif algoname == "raise_exception": + return {"error": {"message": "This is an exception"}} + else: + if content_type != "application/octet-stream": + request = request.decode('utf-8') + if content_type == "text/plain": + metadata['content_type'] = "text" + elif content_type == "application/json": + request = json.loads(request) + metadata['content_type'] = "json" + else: + metadata['content_type'] = "binary" + request = base64.b64encode(request) + output = {"result": request, "metadata": metadata} + return output + + +@self_signed_app.post("/v1/algo/{username}/{algoname}/{githash}") +async def process_hello_world(request: Request, username, algoname, githash): + metadata = {"request_id": "req-55c0480d-6af3-4a21-990a-5c51d29f5725", "duration": 0.000306774, + 'content_type': "text"} + request = await request.body() + request = request.decode('utf-8') + return {"result": f"hello {request}", "metadata": metadata} + + +### Algorithm Routes +@self_signed_app.get('/v1/algorithms/{username}/{algoname}') +async def process_get_algo(request: Request, username, algoname): + if algoname == "echo": + return {"id": "21df7a38-eab8-4ac8-954c-41a285535e69", "name": "echo", + "details": {"summary": "", "label": "echo", "tagline": ""}, + "settings": {"algorithm_callability": "public", "source_visibility": "closed", + "package_set": "python36", "license": "apl", "royalty_microcredits": 0, + "network_access": "full", "pipeline_enabled": True, "insights_enabled": False, + "algorithm_environment": "067110e7-8969-4441-b3d6-5333f18a3db3"}, + "version_info": {"semantic_version": "0.1.0", "git_hash": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "version_uuid": "e06d2808-bb5e-46ae-b7bc-f3d9968e3c6b"}, + "build": {"build_id": "a9ae2c93-6f4e-42c0-ac54-baa4a66e53d3", "status": "succeeded", + "commit_sha": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "started_at": "2022-05-08T22:43:09.050Z", "finished_at": "2022-05-08T22:43:28.646Z", + "version_info": {"semantic_version": "0.1.0"}, "resource_type": "algorithm_build"}, + "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, + "compilation": {"successful": True, "output": ""}, + "self_link": "https://api.algorithmia.com/v1/algorithms/quality/echo/versions/0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "resource_type": "algorithm"} + else: + return {"error": "No such algorithm"} + + +@self_signed_app.get("/v1/algorithms/{username}/{algoname}/builds/{buildid}") +async def get_build_id(username, algoname, buildid): + return {"status": "succeeded", "build_id": buildid, "commit_sha": "bcdadj", + "started_at": "2021-09-27T22:54:20.786Z", "finished_at": "2021-09-27T22:54:40.898Z", + "version_info": {"semantic_version": "0.1.1"}} + + +@self_signed_app.get("/v1/algorithms/{username}/{algoname}/builds/{buildid}/logs") +async def get_build_log(username, algoname, buildid): + return {"logs": "This is a log"} + + +@self_signed_app.get("/v1/algorithms/{username}/{algoname}/scm/status") +async def get_scm_status(username, algoname): + return {"scm_connection_status": "active"} + + +@self_signed_app.get("/v1/algorithms/{algo_id}/errors") +async def get_algo_errors(algo_id): + return {"error": {"message": "not found"}} + + +@self_signed_app.post("/v1/algorithms/{username}") +async def create_algorithm(request: Request, username): + payload = await request.json() + return {"id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", "name": payload["name"], + "details": {"label": payload["details"]["label"]}, + "settings": {"algorithm_callability": "private", "source_visibility": "open", + "package_set": "tensorflow-gpu-2.3-python38", "license": "apl", "network_access": "isolated", + "pipeline_enabled": False, "insights_enabled": False, + "algorithm_environment": "fd980f4f-1f1c-4b2f-a128-d60b40c6567a"}, + "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, + "resource_type": "algorithm"} + + +@self_signed_app.post("/v1/algorithms/{username}/{algoname}/compile") +async def compile_algorithm(username, algoname): + return { + "id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", + "name": algoname, + "details": { + "summary": "Example Summary", + "label": "QA", + "tagline": "Example Tagline" + }, + "settings": { + "algorithm_callability": "private", + "source_visibility": "open", + "package_set": "tensorflow-gpu-2.3-python38", + "license": "apl", + "network_access": "isolated", + "pipeline_enabled": False, + "insights_enabled": False, + "algorithm_environment": "fd980f4f-1f1c-4b2f-a128-d60b40c6567a" + }, + "version_info": { + "git_hash": "e85db9bca2fad519f540b445f30d12523e4dec9c", + "version_uuid": "1d9cb91d-11ca-49cb-a7f4-28f67f277654" + }, + "source": { + "scm": { + "id": "internal", + "provider": "internal", + "default": True, + "enabled": True + } + }, + "compilation": { + "successful": True, + "output": "" + }, + "self_link": f"http://localhost:8080/v1/algorithms/{username}/{algoname}/versions/e85db9bca2fad519f540b445f30d12523e4dec9c", + "resource_type": "algorithm" + } + + +@self_signed_app.post("/v1/algorithms/{username}/{algoname}/versions") +async def publish_algorithm(request: Request, username, algoname): + return {"id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", "name": algoname, + "details": {"summary": "Example Summary", "label": "QA", "tagline": "Example Tagline"}, + "settings": {"algorithm_callability": "private", "source_visibility": "open", + "package_set": "tensorflow-gpu-2.3-python38", "license": "apl", "network_access": "isolated", + "pipeline_enabled": False, "insights_enabled": False, + "algorithm_environment": "fd980f4f-1f1c-4b2f-a128-d60b40c6567a"}, + "version_info": {"semantic_version": "0.1.0", "git_hash": "e85db9bca2fad519f540b445f30d12523e4dec9c", + "release_notes": "created programmatically", "sample_input": "payload", + "version_uuid": "e85db9bca2fad519f540b445f30d12523e4dec9c"}, + "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, + "compilation": {"successful": True}, + "self_link": f"http://localhost:8080/v1/algorithms/{username}/{algoname}/versions/e85db9bca2fad519f540b445f30d12523e4dec9c", + "resource_type": "algorithm"} + + +@self_signed_app.get("/v1/algorithms/{username}/{algoname}/versions") +async def versions_of_algorithm(request: Request, username, algoname): + return {"marker": None, "next_link": None, "results": [ + {"id": "21df7a38-eab8-4ac8-954c-41a285535e69", "name": algoname, + "details": {"summary": "", "label": algoname, "tagline": ""}, + "settings": {"algorithm_callability": "public", "source_visibility": "closed", "package_set": "python36", + "license": "apl", "royalty_microcredits": 0, "network_access": "full", "pipeline_enabled": True, + "insights_enabled": False, "algorithm_environment": "067110e7-8969-4441-b3d6-5333f18a3db3"}, + "version_info": {"semantic_version": "0.1.0", "git_hash": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "version_uuid": "e06d2808-bb5e-46ae-b7bc-f3d9968e3c6b"}, + "build": {"build_id": "a9ae2c93-6f4e-42c0-ac54-baa4a66e53d3", "status": "succeeded", + "commit_sha": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", "started_at": "2022-05-08T22:43:09.050Z", + "finished_at": "2022-05-08T22:43:28.646Z", "version_info": {"semantic_version": "0.1.0"}, + "resource_type": "algorithm_build"}, + "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, + "compilation": {"successful": True}, + "self_link": f"https://api.algorithmia.com/v1/algorithms/{username}/{algoname}/versions" + "/0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "resource_type": "algorithm"}]} + + +@self_signed_app.get("/v1/algorithms/{username}/{algoname}/versions/{algohash}") +async def get_algorithm_info(username, algoname, algohash): + if algohash == "e85db9bca2fad519f540b445f30d12523e4dec9c": + return {"id": "21df7a38-eab8-4ac8-954c-41a285535e69", "name": algoname, + "details": {"summary": "", "label": algoname, "tagline": ""}, + "settings": {"algorithm_callability": "public", "source_visibility": "closed", "language": "python3", + "environment": "cpu", "package_set": "python36", "license": "apl", + "royalty_microcredits": 0, "network_access": "full", "pipeline_enabled": True, + "insights_enabled": False, + "algorithm_environment": "067110e7-8969-4441-b3d6-5333f18a3db3"}, + "version_info": {"semantic_version": "0.1.0", "git_hash": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "version_uuid": "e06d2808-bb5e-46ae-b7bc-f3d9968e3c6b"}, + "build": {"build_id": "a9ae2c93-6f4e-42c0-ac54-baa4a66e53d3", "status": "succeeded", + "commit_sha": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "started_at": "2022-05-08T22:43:09.050Z", "finished_at": "2022-05-08T22:43:28.646Z", + "version_info": {"semantic_version": "0.1.0"}, "resource_type": "algorithm_build"}, + "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, + "compilation": {"successful": True, "output": ""}, "resource_type": "algorithm"} + else: + return {"error": {"message": "not found"}} + + +### Admin Routes +@self_signed_app.post("/v1/users") +async def create_user(request: Request): + payload = await request.body() + data = json.loads(payload) + username = data['username'] + email = data['email'] + return { + "id": "1e5c89ab-3d5c-4bad-b8a3-6c8a294d4418", + "username": username, + "email": email, + "fullname": username, + "self_link": f"http://localhost:8080/v1/users/{username}", "resource_type": "user" + } + + +@self_signed_app.get("/v1/users/{user_id}/errors") +async def get_user_errors(user_id): + return [] + + +@self_signed_app.get("/v1/organization/types") +async def get_org_types(): + return [ + {"id": "d0c85ea6-ddfa-11ea-a0c8-12a811be4db3", "name": "basic"}, + {"id": "d0bff917-ddfa-11ea-a0c8-12a811be4db3", "name": "legacy"}, + {"id": "d0c9d825-ddfa-11ea-a0c8-12a811be4db3", "name": "pro"} + ] + + +@self_signed_app.post("/v1/organizations") +async def create_org(request: Request): + payload = await request.body() + data = json.loads(payload) + org_name = data["org_name"] + org_email = data["org_email"] + return {"id": "55073c92-5f8e-4d7e-a14d-568f94924fd9", + "org_name": org_name, + "org_label": "some label", + "org_contact_name": "Some owner", + "org_email": org_email, + "org_created_at": "2021-10-22T16:41:32", + "org_url": None, + "type_id": "d0c85ea6-ddfa-11ea-a0c8-12a811be4db3", + "stripe_customer_id": None, + "external_admin_group": None, + "external_member_group": None, + "external_id": None, + "owner_ids": None, + "resource_type": "organization", + "self_link": "http://localhost:8080/v1/organizations/a_myOrg1542" + } + + +@self_signed_app.put("/v1/organizations/{orgname}/members/{username}") +async def add_user_to_org(orgname, username): + return Response(status_code=200) + + +@self_signed_app.get("/v1/organizations/{orgname}/errors") +async def org_errors(orgname): + return [] + + +@self_signed_app.put("/v1/organizations/{org_name}") +async def edit_org(org_name): + return Response(status_code=204) + + +@self_signed_app.get("/v1/organizations/{org_name}") +async def get_org_by_name(org_name): + return { + "id": "55073c92-5f8e-4d7e-a14d-568f94924fd9", + "org_name": org_name, + "org_label": "some label", + "org_contact_name": "Some owner", + "org_email": "a_myOrg1542@algo.com", + "org_created_at": "2021-10-22T16:41:32", + "org_url": None, + "type_id": "d0c85ea6-ddfa-11ea-a0c8-12a811be4db3", + "stripe_customer_id": None, + "external_admin_group": None, + "external_member_group": None, + "external_id": None, + "owner_ids": None, + "resource_type": "organization", + "self_link": "http://localhost:8080/v1/organizations/a_myOrg1542" + } + + +@self_signed_app.get("/v1/algorithms/{username}/{algoname}/builds/{buildid}/logs") +async def get_build_log(username, algoname, buildid): + return {"logs": "This is a log"} + + +@self_signed_app.get("/v1/algorithm-environments/edge/languages") +async def get_supported_langs(): + return [{"name": "anaconda3", "display_name": "Conda (Environments) - beta", + "configuration": "{\n \"display_name\": \"Conda (Environments) - beta\",\n \"req_files\": [\n \"environment.yml\"\n ],\n \"artifacts\": [\n {\"source\":\"/home/algo/.cache\", \"destination\":\"/home/algo/.cache/\"},\n {\"source\":\"/home/algo/anaconda_environment\", \"destination\": \"/home/algo/anaconda_environment/\"},\n {\"source\":\"/opt/algorithm\", \"destination\":\"/opt/algorithm/\"}\n ]\n}\n"}, + {"name": "csharp-dotnet-core2", "display_name": "C# .NET Core 2.x+ (Environments)", + "configuration": "{\n \"display_name\": \"C# .NET Core 2.x+ (Environments)\",\n \"artifacts\": [\n {\"source\":\"/opt/algorithm/bin/Release/*/*\", \"destination\":\"/opt/algorithm/\"},\n {\"source\":\"/opt/algorithm/resources\", \"destination\":\"/opt/algorithm/resources/\"},\n {\"source\":\"/home/algo/.nuget\", \"destination\":\"/home/algo/.nuget/\"}\n ]\n}\n"}, + {"name": "java11", "display_name": "Java OpenJDK 11.0 (Environments)", + "configuration": "{\n \"display_name\": \"Java OpenJDK 11.0 (Environments)\",\n \"artifacts\": [\n {\"source\":\"/opt/algorithm/target/*.jar\", \"destination\":\"/opt/algorithm/target/algorithm.jar\"},\n {\"source\":\"/opt/algorithm/target/lib\", \"destination\":\"/opt/algorithm/target/lib/\"}\n ]\n}\n"}, + {"name": "python2", "display_name": "Python 2.x (Environments)", + "configuration": "{\n \"display_name\": \"Python 2.x (Environments)\",\n \"req_files\": [\n \"requirements.txt\"\n ],\n \"artifacts\": [\n {\"source\":\"/home/algo/.local\", \"destination\":\"/home/algo/.local/\"},\n {\"source\":\"/opt/algorithm\", \"destination\":\"/opt/algorithm/\"}\n ]\n}\n"}, + {"name": "python3", "display_name": "Python 3.x (Environments)", + "configuration": "{\n \"display_name\": \"Python 3.x (Environments)\",\n \"req_files\": [\n \"requirements.txt\"\n ],\n \"artifacts\": [\n {\"source\":\"/home/algo/.local\", \"destination\":\"/home/algo/.local/\"},\n {\"source\":\"/opt/algorithm\", \"destination\":\"/opt/algorithm/\"}\n ]\n}\n"}, + {"name": "r36", "display_name": "R 3.6.x (Environments)", + "configuration": "{\n \"display_name\": \"R 3.6.x (Environments)\",\n \"req_files\": [\n \"packages.txt\"\n ],\n \"artifacts\": [\n {\"source\":\"/opt/algorithm\", \"destination\":\"/opt/algorithm/\"},\n {\"source\":\"/usr/local/lib/R/site-library\", \"destination\":\"/usr/local/lib/R/site-library/\"}\n ]\n}\n\n"}, + {"name": "scala-2", "display_name": "Scala 2.x & sbt 1.3.x (Environments)", + "configuration": "{\n \"display_name\": \"Scala 2.x & sbt 1.3.x (Environments)\",\n \"artifacts\": [\n {\"source\":\"/opt/algorithm/target/universal/stage\", \"destination\":\"/opt/algorithm/stage/\"}\n ]\n}\n\n"}] + + +@self_signed_app.get("/v1/algorithm-environments/edge/languages/{language}/environments") +async def get_environments_by_lang(language): + return { + "environments": [ + { + "id": "717d36e0-222c-44a0-9aa8-06f4ebc1b82a", + "environment_specification_id": "f626effa-e519-431e-9d7a-0d3a7563ae1e", + "display_name": "Python 2.7", + "description": "Generic Python 2.7 installation", + "created_at": "2020-12-21T21:47:53.239", + "language": { + "name": language, + "display_name": "Python 2.x (Environments)", + "configuration": "{\n \"display_name\": \"Python 2.x (Environments)\",\n \"req_files\": [\n " + " \"requirements.txt\"\n ],\n \"artifacts\": [\n {" + "\"source\":\"/home/algo/.local\", \"destination\":\"/home/algo/.local/\"}," + "\n {\"source\":\"/opt/algorithm\", " + "\"destination\":\"/opt/algorithm/\"}\n ]\n}\n " + }, + "machine_type": "CPU" + }, + { + "id": "6f57e041-54e0-4e1a-8b2f-4589bb2c06f8", + "environment_specification_id": "faf81400-eb15-4f64-81c0-3d4ed7181e77", + "display_name": "Python 2.7 + GPU support", + "description": "Python2.7 installation with CUDA 9.0 and CUDNN7", + "created_at": "2020-08-14T07:22:32.955", + "language": { + "name": language, + "display_name": "Python 2.x (Environments)", + "configuration": "{\n \"display_name\": \"Python 2.x (Environments)\",\n \"req_files\": [\n " + " \"requirements.txt\"\n ],\n \"artifacts\": [\n {" + "\"source\":\"/home/algo/.local\", \"destination\":\"/home/algo/.local/\"}," + "\n {\"source\":\"/opt/algorithm\", " + "\"destination\":\"/opt/algorithm/\"}\n ]\n}\n " + }, + "machine_type": "GPU" + } + ] + } diff --git a/Test/client_test.py b/Test/client_test.py deleted file mode 100644 index fe39143..0000000 --- a/Test/client_test.py +++ /dev/null @@ -1,104 +0,0 @@ -from datetime import datetime, time -import shutil -import sys -import os -from random import seed -from random import random -# look in ../ BEFORE trying to import Algorithmia. If you append to the -# you will load the version installed on the computer. -sys.path = ['../'] + sys.path - -import unittest - -import Algorithmia - -class client_test(unittest.TestCase): - seed(datetime.now().microsecond) - - username = "a_Mrtest" - orgname = "a_myOrg" - - def setUp(self): - self.username = self.username + str(int(random()*10000)) - self.orgname = self.orgname + str(int(random()*10000)) - self.c = Algorithmia.client(api_address="https://test.algorithmia.com",api_key=os.environ.get('ALGORITHMIA_A_KEY')) - - def test_create_user(self): - response = self.c.create_user({"username":self.username, "email": self.username+"@algo.com", "passwordHash":"", "shouldCreateHello": False}) - self.assertEqual(self.username,response['username']) - - def test_get_org_types(self): - response = self.c.get_org_types() - self.assertTrue(len(response)>0) - - def test_create_org(self): - response = self.c.create_org({"org_name": self.orgname, "org_label": "some label", "org_contact_name": "Some owner", "org_email": self.orgname+"@algo.com","type_id":"basic"}) - self.assertEqual(self.orgname,response['org_name']) - - def test_get_org(self): - response = self.c.get_org("a_myOrg84") - self.assertEqual("a_myOrg84",response['org_name']) - - def test_get_environment(self): - client =Algorithmia.client(api_key=os.environ.get('ALGORITHMIA_API_KEY')) - response = client.get_environment("python2") - print(response) - if("error" in response): - print(response) - self.assertTrue(response is not None and "environments" in response) - - def test_get_build_logs(self): - client = Algorithmia.client(api_key=os.environ.get('ALGORITHMIA_API_KEY')) - user = os.environ.get('ALGO_USER_NAME') - algo = "Echo" - result = client.algo(user+'/'+algo).build_logs() - if "error" in result: - print(result) - self.assertTrue("error" not in result) - - - def test_edit_org(self): - orgname="a_myOrg84" - - obj = { - "id": "b85d8c4e-7f3c-40b9-9659-6adc2cb0e16f", - "org_name": "a_myOrg84", - "org_label": "some label", - "org_contact_name": "Some owner", - "org_email": "a_myOrg84@algo.com", - "org_created_at": "2020-11-30T23:51:40", - "org_url":"https://algorithmia.com", - "type_id": "basic", - "resource_type": "organization" - } - - response = self.c.edit_org(orgname,obj) - self.assertEqual(204,response.status_code) - - def test_get_template(self): - filename = "./temptest" - client =Algorithmia.client(api_key=os.environ.get('ALGORITHMIA_API_KEY')) - response = client.get_template("36fd467e-fbfe-4ea6-aa66-df3f403b7132",filename) - print(response) - self.assertTrue(response.ok) - try: - shutil.rmtree(filename) - except OSError as e: - print(e) - - def test_get_supported_languages(self): - client = Algorithmia.client(api_key=os.environ.get('ALGORITHMIA_API_KEY')) - response = client.get_supported_languages() - language_found = any('anaconda3' in languages['name'] for languages in response) - if("error" in response): - print(response) - self.assertTrue(response is not None and language_found) - - - def test_invite_to_org(self): - response = self.c.invite_to_org("a_myOrg38","a_Mrtest4") - self.assertEqual(200,response.status_code) - - -if __name__ == '__main__': - unittest.main() \ No newline at end of file diff --git a/Test/conftest.py b/Test/conftest.py new file mode 100644 index 0000000..c758814 --- /dev/null +++ b/Test/conftest.py @@ -0,0 +1,15 @@ +import sys +from time import sleep +import os, signal +if sys.version_info.major >= 3: + from Test.api import start_webserver_reg, start_webserver_self_signed + import pytest + + @pytest.fixture(scope='package', autouse=True) + def fastapi_start(): + p_reg = start_webserver_reg() + p_self_signed = start_webserver_self_signed() + sleep(2) + yield p_reg, p_self_signed + os.kill(p_reg.pid, signal.SIGKILL) + os.kill(p_self_signed.pid, signal.SIGKILL) \ No newline at end of file diff --git a/Test/regular/CLI_test.py b/Test/regular/CLI_test.py new file mode 100644 index 0000000..015db70 --- /dev/null +++ b/Test/regular/CLI_test.py @@ -0,0 +1,289 @@ +import sys + +# look in ../ BEFORE trying to import Algorithmia. If you append to the +# you will load the version installed on the computer. +sys.path = ['../'] + sys.path + +import unittest +import os +import json +import Algorithmia +from Algorithmia.CLI import CLI +import argparse +import shutil + +if sys.version_info.major >= 3: + class CLIDummyTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.client = Algorithmia.client(api_address="http://localhost:8080", api_key="simabcd123") + cls.bearerClient = Algorithmia.client(api_address="http://localhost:8080", bearer_token="simabcd123.token.token") + + def test_run(self): + name = "util/Echo" + inputs = "test" + + parser = argparse.ArgumentParser('CLI for interacting with Algorithmia') + + subparsers = parser.add_subparsers(help='sub cmd', dest='subparser_name') + parser_run = subparsers.add_parser('run', help='algo run [input options] [output options]') + + parser_run.add_argument('algo') + parser_run.add_argument('-d', '--data', action='store', help='detect input type', default=None) + parser_run.add_argument('-t', '--text', action='store', help='treat input as text', default=None) + parser_run.add_argument('-j', '--json', action='store', help='treat input as json data', default=None) + parser_run.add_argument('-b', '--binary', action='store', help='treat input as binary data', default=None) + parser_run.add_argument('-D', '--data-file', action='store', help='specify a path to an input file', + default=None) + parser_run.add_argument('-T', '--text-file', action='store', help='specify a path to a text file', + default=None) + parser_run.add_argument('-J', '--json-file', action='store', help='specify a path to a json file', + default=None) + parser_run.add_argument('-B', '--binary-file', action='store', help='specify a path to a binary file', + default=None) + parser_run.add_argument('--timeout', action='store', type=int, default=300, + help='specify a timeout (seconds)') + parser_run.add_argument('--debug', action='store_true', + help='print the stdout from the algo ') + parser_run.add_argument('--profile', action='store', type=str, default='default') + parser_run.add_argument('-o', '--output', action='store', default=None, type=str) + + args = parser.parse_args(['run', name, '-d', inputs]) + + result = CLI().runalgo(args, self.client) + self.assertEqual(result, inputs) + + def test_run_token(self): + name = "util/Echo" + inputs = "test" + + parser = argparse.ArgumentParser('CLI for interacting with Algorithmia') + + subparsers = parser.add_subparsers(help='sub cmd', dest='subparser_name') + parser_run = subparsers.add_parser('run', help='algo run [input options] [output options]') + + parser_run.add_argument('algo') + parser_run.add_argument('-d', '--data', action='store', help='detect input type', default=None) + parser_run.add_argument('-t', '--text', action='store', help='treat input as text', default=None) + parser_run.add_argument('-j', '--json', action='store', help='treat input as json data', default=None) + parser_run.add_argument('-b', '--binary', action='store', help='treat input as binary data', default=None) + parser_run.add_argument('-D', '--data-file', action='store', help='specify a path to an input file', + default=None) + parser_run.add_argument('-T', '--text-file', action='store', help='specify a path to a text file', + default=None) + parser_run.add_argument('-J', '--json-file', action='store', help='specify a path to a json file', + default=None) + parser_run.add_argument('-B', '--binary-file', action='store', help='specify a path to a binary file', + default=None) + parser_run.add_argument('--timeout', action='store', type=int, default=300, + help='specify a timeout (seconds)') + parser_run.add_argument('--debug', action='store_true', + help='print the stdout from the algo ') + parser_run.add_argument('--profile', action='store', type=str, default='default') + parser_run.add_argument('-o', '--output', action='store', default=None, type=str) + + args = parser.parse_args(['run', name, '-d', inputs]) + + result = CLI().runalgo(args, self.bearerClient) + self.assertEqual(result, inputs) + + +class CLIMainTest(unittest.TestCase): + def setUp(self): + # create a directory to use in testing the cp command + self.client = Algorithmia.client() + CLI().mkdir("data://.my/moredata", self.client) + if not os.path.exists("../TestFiles/"): + os.mkdir("../TestFiles/") + + def test_ls(self): + parentDir = "data://.my/" + newDir = "test" + + CLI().mkdir(parentDir + newDir, self.client) + result = CLI().ls(parentDir, self.client) + self.assertTrue(result is not None and "moredata" in result and newDir in result) + + CLI().rmdir(parentDir + newDir, self.client) + + def test_mkdir(self): + + parentDir = "data://.my/" + newDir = "test" + + CLI().mkdir(parentDir + newDir, self.client) + result = CLI().ls(parentDir, self.client) + self.assertTrue(newDir in result) + + CLI().rmdir(parentDir + newDir, self.client) + + def test_rmdir(self): + parentDir = "data://.my/" + newDir = "testRmdir" + + CLI().mkdir(parentDir + newDir, self.client) + result = CLI().ls(parentDir, self.client) + self.assertTrue(newDir in result) + + CLI().rmdir(parentDir + newDir, self.client) + + result = CLI().ls(parentDir, self.client) + self.assertTrue(newDir not in result) + + def test_cat(self): + file = "data://.my/moredata/test.txt" + localfile = "./../TestFiles/test.txt" + fileContents = "some text in test file" + + CLI().rm(file, self.client) + testfile = open(localfile, "w") + testfile.write(fileContents) + testfile.close() + + CLI().cp([localfile], file, self.client) + + result = CLI().cat([file], self.client) + self.assertEqual(result, fileContents) + + def test_get_build_logs(self): + user = os.environ.get('ALGO_USER_NAME') + algo = "Echo" + + result = json.loads(CLI().getBuildLogs(user, algo, self.client)) + if "error" in result: + print(result) + self.assertTrue("error" not in result) + + # local to remote + def test_cp_L2R(self): + localfile = "./../TestFiles/test.txt" + testfile = open(localfile, "w") + testfile.write("some text") + testfile.close() + + src = [localfile] + dest = "data://.my/moredata/test.txt" + CLI().cp(src, dest, self.client) + + result = CLI().ls("data://.my/moredata/", self.client) + self.assertTrue("test.txt" in result) + + # remote to remote + def test_cp_R2R(self): + + src = ["data://.my/moredata/test.txt"] + dest = "data://.my/moredata/test2.txt" + CLI().cp(src, dest, self.client) + + result = CLI().ls("data://.my/moredata/", self.client) + self.assertTrue("test2.txt" in result) + + # remote to local + def test_cp_R2L(self): + src = ["data://.my/moredata/test.txt"] + dest = "./../test.txt" + + CLI().cp(src, dest, self.client) + self.assertTrue(os.path.isfile(dest)) + + def test_auth(self): + # key for test account + key = os.getenv('ALGORITHMIA_API_KEY') + api_address = "https://api.algorithmia.com" + profile = 'default' + CLI().auth(api_address, key, profile=profile) + resultK = CLI().getAPIkey(profile) + resultA = CLI().getAPIaddress(profile) + self.assertEqual(resultK, key) + self.assertEqual(resultA, api_address) + + def test_auth_cert(self): + + localfile = "./../TestFiles/fakecert.pem" + + testfile = open(localfile, "w") + testfile.write("") + testfile.close() + + # key for test account + key = os.getenv('ALGORITHMIA_API_KEY') + address = 'https://api.algorithmia.com' + cacert = localfile + profile = 'test' + + CLI().auth(address, key, cacert=cacert, profile=profile) + resultK = CLI().getAPIkey(profile) + resultA = CLI().getAPIaddress(profile) + resultC = CLI().getCert(profile) + self.assertEqual(resultK, key) + self.assertEqual(resultA, address) + self.assertEqual(resultC, cacert) + + def test_auth_token(self): + address = 'https://api.algorithmia.com' + bearer = 'testtokenabcd' + profile = 'test' + + CLI().auth(apiaddress=address, bearer=bearer, profile=profile) + resultA = CLI().getAPIaddress(profile) + resultT = CLI().getBearerToken(profile) + self.assertEqual(resultA, address) + self.assertEqual(resultT, bearer) + + def test_get_environment(self): + result = CLI().get_environment_by_language("python2", self.client) + print(result) + if ("error" in result): + print(result) + self.assertTrue(result is not None and "display_name" in result) + + def test_list_languages(self): + result = CLI().list_languages(self.client) + if ("error" in result[0]): + print(result) + self.assertTrue(result is not None and "anaconda3" in result[1]) + + def test_rm(self): + localfile = "./../TestFiles/testRM.txt" + + testfile = open(localfile, "w") + testfile.write("some text") + testfile.close() + + src = [localfile] + dest = "data://.my/moredata/" + CLI().cp(src, dest, self.client) + + result1 = CLI().ls(dest, self.client) + + CLI().rm("data://.my/moredata/testRM.txt", self.client) + + result2 = CLI().ls(dest, self.client) + + self.assertTrue("testRM.txt" in result1 and "testRM.txt" not in result2) + + def test_get_template(self): + filename = "./../temptest" + envid = "36fd467e-fbfe-4ea6-aa66-df3f403b7132" + response = CLI().get_template(envid, filename, self.client) + print(response) + self.assertTrue(response.ok) + try: + shutil.rmtree(filename) + except OSError as e: + print(e) + + def test_api_address_auth(self): + api_key = os.getenv('ALGORITHMIA_API_KEY') + api_address = "https://api.algorithmia.com" + CLI().auth(api_address, api_key) + profile = "default" + + client = Algorithmia.client(CLI().getAPIkey(profile), CLI().getAPIaddress(profile), CLI().getCert(profile)) + result2 = CLI().ls("data://.my", client) + print(result2) + self.assertTrue(result2 != "") + + +if __name__ == '__main__': + unittest.main() diff --git a/Test/regular/__init__.py b/Test/regular/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Test/acl_test.py b/Test/regular/acl_test.py similarity index 100% rename from Test/acl_test.py rename to Test/regular/acl_test.py diff --git a/Test/regular/algo_failure_test.py b/Test/regular/algo_failure_test.py new file mode 100644 index 0000000..0ec4fc2 --- /dev/null +++ b/Test/regular/algo_failure_test.py @@ -0,0 +1,42 @@ +import sys + +if sys.version_info[0] >= 3: + import unittest + import Algorithmia + import uvicorn + import time + from multiprocessing import Process + + # look in ../ BEFORE trying to import Algorithmia. If you append to the + # you will load the version installed on the computer. + sys.path = ['../'] + sys.path + from requests import Response + + class AlgoTest(unittest.TestCase): + error_500 = Response() + error_500.status_code = 500 + error_message = "Non-Algorithm related Failure: " + str(error_500) + + @classmethod + def setUpClass(cls): + cls.client = Algorithmia.client(api_address="http://localhost:8080", api_key="simabcd123") + + def test_throw_500_error_HTTP_response_on_algo_request(self): + try: + result = self.client.algo('util/500').pipe(bytearray('foo', 'utf-8')) + except Exception as e: + result = e + pass + self.assertEqual(str(self.error_message), str(result)) + + def test_retry_on_400_error_publish(self): + result = self.client.algo("util/failonce").publish() + self.assertEqual(result['version_info']['semantic_version'], "0.1.0") + + def test_throw_on_always_500_publish(self): + try: + result = self.client.algo("util/failalways").publish() + except Exception as e: + result = e + pass + self.assertEqual(str(self.error_message), str(result)) diff --git a/Test/regular/algo_test.py b/Test/regular/algo_test.py new file mode 100644 index 0000000..ab63c0d --- /dev/null +++ b/Test/regular/algo_test.py @@ -0,0 +1,253 @@ +import sys +import os +from Algorithmia.errors import AlgorithmException +from Algorithmia.algorithm import OutputType +import Algorithmia + +# look in ../ BEFORE trying to import Algorithmia. If you append to the +# you will load the version installed on the computer. +sys.path = ['../'] + sys.path + +import unittest + +if sys.version_info.major >= 3: + + class AlgoDummyTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.client = Algorithmia.client(api_address="http://localhost:8080", api_key="simabcd123") + cls.environment_id = "abcd-123" + + def test_call_customCert(self): + result = self.client.algo('quality/echo').pipe(bytearray('foo', 'utf-8')) + self.assertEqual('binary', result.metadata.content_type) + self.assertEqual(bytearray('foo', 'utf-8'), result.result) + + + + def test_normal_call(self): + result = self.client.algo('quality/echo').pipe("foo") + self.assertEqual("text", result.metadata.content_type) + self.assertEqual("foo", result.result) + + def test_async_call(self): + result = self.client.algo('quality/echo').set_options(output=OutputType.void).pipe("foo") + self.assertTrue(hasattr(result, "async_protocol")) + self.assertTrue(hasattr(result, "request_id")) + + def test_raw_call(self): + result = self.client.algo('quality/echo').set_options(output=OutputType.raw).pipe("foo") + self.assertEqual("foo", result) + + def test_dict_call(self): + result = self.client.algo('quality/echo').pipe({"foo": "bar"}) + self.assertEqual("json", result.metadata.content_type) + self.assertEqual({"foo": "bar"}, result.result) + + def test_algo_exists(self): + result = self.client.algo('quality/echo').exists() + self.assertEqual(True, result) + + def test_algo_no_exists(self): + result = self.client.algo('quality/not_echo').exists() + self.assertEqual(False, result) + + #TODO: add more coverage examples to check kwargs + def test_get_versions(self): + result = self.client.algo('quality/echo').versions() + self.assertTrue('results' in result) + self.assertTrue('version_info' in result['results'][0]) + self.assertTrue('semantic_version' in result['results'][0]['version_info']) + self.assertEqual('0.1.0', result['results'][0]['version_info']['semantic_version']) + + def test_text_unicode(self): + telephone = u"\u260E" + # Unicode input to pipe() + result1 = self.client.algo('quality/echo').pipe(telephone) + self.assertEqual('text', result1.metadata.content_type) + self.assertEqual(telephone, result1.result) + + # Unicode return in .result + result2 = self.client.algo('quality/echo').pipe(result1.result) + self.assertEqual('text', result2.metadata.content_type) + self.assertEqual(telephone, result2.result) + + def test_algo_info(self): + result = self.client.algo('quality/echo').info() + self.assertTrue('results' in result) + self.assertTrue('resource_type' in result['results'][0]) + self.assertTrue(result['results'][0]['resource_type'] == "algorithm") + + def test_update_algo(self): + details = { + "summary": "Example Summary", + "label": "QA", + "tagline": "Example Tagline" + } + settings = { + "source_visibility": "open", + "algorithm_environment": self.environment_id, + "license": "apl", + "network_access": "isolated", + "pipeline_enabled": False + } + version_info = { + "sample_input": "hello" + } + result = self.client.algo('quality/echo').update(details=details, settings=settings, version_info=version_info) + self.assertTrue('id' in result) + + + def test_get_build_by_id(self): + result = self.client.algo("quality/echo").get_build("1a392e2c-b09f-4bae-a616-56c0830ac8e5") + self.assertTrue('commit_sha' in result) + + def test_get_build_logs(self): + result = self.client.algo("quality/echo").get_build_logs("1a392e2c-b09f-4bae-a616-56c0830ac8e5") + self.assertTrue('logs' in result) + + def test_get_scm_status(self): + result = self.client.algo("quality/echo").get_scm_status() + self.assertTrue('scm_connection_status' in result) + + def test_exception_ipa_algo(self): + try: + result = self.client.algo('zeryx/raise_exception').pipe("") + except AlgorithmException as e: + self.assertEqual(e.message, "This is an exception") + + def test_algorithm_programmatic_create_process(self): + algorithm_name = "hello" + payload = "John" + expected_response = "hello John" + full_path = "quality/" + algorithm_name + details = { + "summary": "Example Summary", + "label": "QA", + "tagline": "Example Tagline" + } + settings = { + "source_visibility": "open", + "algorithm_environment": self.environment_id, + "license": "apl", + "network_access": "isolated", + "pipeline_enabled": False + } + version_info = { + "sample_input": "hello" + } + created_algo = self.client.algo(full_path) + print("about to create algo") + response = created_algo.create(details=details, settings=settings, version_info=version_info) + print("created algo") + self.assertEqual(response['name'], algorithm_name, "algorithm creation failed") + + # --- Creation complete, compiling + + response = created_algo.compile() + git_hash = response['version_info']['git_hash'] + algo_with_build = self.client.algo(full_path + "/" + git_hash) + self.assertEqual(response['name'], created_algo.algoname) + + # --- compiling complete, now testing algorithm request + response = algo_with_build.pipe(payload).result + self.assertEqual(response, expected_response, "compiling failed") + + # --- testing complete, now publishing new release. + + pub_settings = {"algorithm_callability": "private"} + pub_version_info = { + "release_notes": "created programmatically", + "sample_input": payload, + "version_type": "minor" + } + pub_details = {"label": "testing123"} + + response = algo_with_build.publish( + details=pub_details, + settings=pub_settings, + version_info=pub_version_info + ) + self.assertEqual(response["version_info"]["semantic_version"], "0.1.0", + "Publishing failed, semantic version is not correct.") + + # --- publishing complete, getting additional information + + response = created_algo.info(git_hash) + + self.assertEqual(response['version_info']['semantic_version'], "0.1.0", "information is incorrect") + + + def test_set_secret(self): + short_name = "tst" + secret_key = "test_key" + secret_value = "test_value" + description = "loreum epsum" + response = self.client.algo("quality/echo").set_secret(short_name, secret_key, secret_value, description) + self.assertEqual(response['id'], "959af771-7cd8-4981-91c4-70def15bbcdc", "invalid ID for created secret") + + +else: + class AlgoTest(unittest.TestCase): + def setUp(self): + self.client = Algorithmia.client() + + def test_call_customCert(self): + open("./test.pem", 'w') + c = Algorithmia.client(ca_cert="./test.pem") + result = c.algo('quality/echo').pipe(bytearray('foo', 'utf-8')) + self.assertEqual('binary', result.metadata.content_type) + self.assertEqual(bytearray('foo', 'utf-8'), result.result) + try: + os.remove("./test.pem") + except OSError as e: + print(e) + + def test_call_binary(self): + result = self.client.algo('quality/echo').pipe(bytearray('foo', 'utf-8')) + self.assertEqual('binary', result.metadata.content_type) + self.assertEqual(bytearray('foo', 'utf-8'), result.result) + + def test_async_call(self): + result = self.client.algo('quality/echo').set_options(output=OutputType.void).pipe("foo") + self.assertTrue(hasattr(result, "async_protocol")) + self.assertTrue(hasattr(result, "request_id")) + + def test_raw_call(self): + result = self.client.algo('quality/echo').set_options(output=OutputType.raw).pipe("foo") + self.assertEqual("foo", result) + + #TODO: add more coverage examples to check kwargs + def test_get_versions(self): + result = self.client.algo('quality/echo').versions() + self.assertTrue('results' in result) + self.assertTrue('version_info' in result['results'][0]) + self.assertTrue('semantic_version' in result['results'][0]['version_info']) + self.assertEqual('0.1.0', result['results'][0]['version_info']['semantic_version']) + + def test_text_unicode(self): + telephone = u"\u260E" + + # Unicode input to pipe() + result1 = self.client.algo('quality/echo').pipe(telephone) + self.assertEqual('text', result1.metadata.content_type) + self.assertEqual(telephone, result1.result) + + # Unicode return in .result + result2 = self.client.algo('quality/echo').pipe(result1.result) + self.assertEqual('text', result2.metadata.content_type) + self.assertEqual(telephone, result2.result) + + + def test_get_scm_status(self): + result = self.client.algo("quality/echo").get_scm_status() + self.assertTrue('scm_connection_status' in result) + + def test_exception_ipa_algo(self): + try: + result = self.client.algo('zeryx/raise_exception').pipe("") + except AlgorithmException as e: + self.assertEqual(e.message, "This is an exception") + +if __name__ == '__main__': + unittest.main() diff --git a/Test/regular/client_test.py b/Test/regular/client_test.py new file mode 100644 index 0000000..9cfc39b --- /dev/null +++ b/Test/regular/client_test.py @@ -0,0 +1,146 @@ +import os +import shutil +import sys +from datetime import datetime +from random import random +from random import seed + +sys.path = ['../'] + sys.path + +import unittest +import Algorithmia +from Algorithmia.errors import AlgorithmException +from uuid import uuid4 + +if sys.version_info.major >= 3: + unicode = str + + + class ClientDummyTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.client = Algorithmia.client(api_address="http://localhost:8080", api_key="simabcd123") + + admin_username = "a_Mrtest" + admin_org_name = "a_myOrg" + environment_name = "Python 3.9" + + def setUp(self): + self.admin_username = self.admin_username + str(int(random() * 10000)) + self.admin_org_name = self.admin_org_name + str(int(random() * 10000)) + + def test_create_user(self): + response = self.client.create_user( + {"username": self.admin_username, "email": self.admin_username + "@algo.com", "passwordHash": "", + "shouldCreateHello": False}) + + if type(response) is dict: + self.assertEqual(self.admin_username, response['username']) + else: + self.assertIsNotNone(response) + + def test_get_org_types(self): + response = self.client.get_org_types() + self.assertTrue(len(response) > 0) + + def test_create_org(self): + response = self.client.create_org( + {"org_name": self.admin_org_name, "org_label": "some label", "org_contact_name": "Some owner", + "org_email": self.admin_org_name + "@algo.com", "type_id": "basic"}) + + self.assertEqual(self.admin_org_name, response[u'org_name']) + + def test_get_org(self): + response = self.client.get_org("a_myOrg84") + self.assertEqual("a_myOrg84", response['org_name']) + + def test_get_environment(self): + response = self.client.get_environment("python2") + + if u'error' not in response: + self.assertTrue(response is not None and u'environments' in response) + + def test_get_scms(self): + response = self.client.scms() + results = response['results'] + internal = [result for result in results if result['id'] == 'internal'] + self.assertTrue(len(internal) == 1) + + def test_edit_org(self): + org_name = "a_myOrg84" + + obj = { + "id": "b85d8c4e-7f3c-40b9-9659-6adc2cb0e16f", + "org_name": "a_myOrg84", + "org_label": "some label", + "org_contact_name": "Some owner", + "org_email": "a_myOrg84@algo.com", + "org_created_at": "2020-11-30T23:51:40", + "org_url": "https://algorithmia.com", + "type_id": "basic", + "resource_type": "organization" + } + + response = self.client.edit_org(org_name, obj) + if type(response) is dict: + print(response) + else: + self.assertEqual(204, response.status_code) + + def test_get_supported_languages(self): + response = self.client.get_supported_languages() + self.assertTrue(response is not None) + + if type(response) is not list: + self.assertTrue(u'error' in response) + else: + language_found = any('anaconda3' in languages['name'] for languages in response) + self.assertTrue(response is not None and language_found) + + def test_invite_to_org(self): + response = self.client.invite_to_org("a_myOrg38", "a_Mrtest4") + if type(response) is dict: + self.assertTrue(u'error' in response) + else: + self.assertEqual(200, response.status_code) + + # This test will require updating after the /v1/organizations/{org_name}/errors endpoint has been + # deployed to the remote environment. + def test_get_organization_errors(self): + response = self.client.get_organization_errors(self.admin_org_name) + self.assertTrue(response is not None) + + if type(response) is list: + self.assertEqual(0, len(response), 'Received unexpected result, should have been 0.') + + def test_get_user_errors(self): + response = self.client.get_user_errors(self.admin_username) + + self.assertTrue(response is not None) + self.assertEqual(0, len(response)) + + def test_get_algorithm_errors(self): + try: + _ = self.client.get_algorithm_errors('hello') + self.assertFalse(True) + except AlgorithmException as e: + self.assertTrue(e.message == "No such algorithm") + + def test_no_auth_client(self): + + key = os.environ.get('ALGORITHMIA_API_KEY', "") + if key != "": + del os.environ['ALGORITHMIA_API_KEY'] + + client = Algorithmia.client(api_address="http://localhost:8080") + error = None + try: + client.algo("demo/hello").pipe("world") + except Exception as e: + error = e + finally: + os.environ['ALGORITHMIA_API_KEY'] = key + self.assertEqual(str(error), str(AlgorithmException(message="authorization required", stack_trace=None, + error_type=None))) +if __name__ == '__main__': + unittest.main() diff --git a/Test/datadirectory_test.py b/Test/regular/datadirectory_test.py similarity index 79% rename from Test/datadirectory_test.py rename to Test/regular/datadirectory_test.py index 28d73e0..9dc2704 100644 --- a/Test/datadirectory_test.py +++ b/Test/regular/datadirectory_test.py @@ -93,12 +93,33 @@ def list_files_small(self, collectionName): dd.delete(True) + def get_files(self, collectionName): + dd = self.client.dir(collectionName) + if dd.exists(): + dd.delete(True) + + dd.create() + + f1 = dd.file('a') + f1.put('data') + + f2 = dd.file('b') + f2.put('data') + + local_path = dd.getDir() + self.assertTrue(os.path.isfile(os.path.join(local_path, "a"))) + self.assertTrue(os.path.isfile(os.path.join(local_path, "b"))) + + def test_list_files_small_without_trailing_slash(self): self.list_files_small('data://.my/test_list_files_small') def test_list_files_small_with_trailing_slash(self): self.list_files_small('data://.my/test_list_files_small/') + def test_get_directory(self): + self.get_files("data://.my/test_list_files_small") + def test_list_folders(self): dd = DataDirectory(self.client, 'data://.my/') @@ -120,32 +141,34 @@ def test_list_folders(self): testDir.delete(True) - def test_list_files_with_paging(self): - NUM_FILES = 1100 - EXTENSION = '.txt' - - dd = DataDirectory(self.client, 'data://.my/pythonLargeDataDirList') - if not dd.exists(): - dd.create() - - for i in range(NUM_FILES): - dd.file(str(i) + EXTENSION).put(str(i)) - - seenFiles = [False] * NUM_FILES - numFiles = 0 - - for f in dd.files(): - numFiles += 1 - name = f.getName() - index = int(name[:-1 * len(EXTENSION)]) - seenFiles[index] = True - - allSeen = True - for cur in seenFiles: - allSeen = (allSeen and cur) - self.assertEqual(NUM_FILES, numFiles) - self.assertTrue(allSeen) +# TODO: replicate this in Marketplace + # def test_list_files_with_paging(self): + # NUM_FILES = 1100 + # EXTENSION = '.txt' + # + # dd = DataDirectory(self.client, 'data://.my/pythonLargeDataDirList') + # if not dd.exists(): + # dd.create() + # + # for i in range(NUM_FILES): + # dd.file(str(i) + EXTENSION).put(str(i)) + # + # seenFiles = [False] * NUM_FILES + # numFiles = 0 + # + # for f in dd.files(): + # numFiles += 1 + # name = f.getName() + # index = int(name[:-1 * len(EXTENSION)]) + # seenFiles[index] = True + # + # allSeen = True + # for cur in seenFiles: + # allSeen = (allSeen and cur) + # + # self.assertEqual(NUM_FILES, numFiles) + # self.assertTrue(allSeen) def test_data_object(self): dd = DataDirectory(self.client, 'data://foo') diff --git a/Test/datafile_test.py b/Test/regular/datafile_test.py similarity index 68% rename from Test/datafile_test.py rename to Test/regular/datafile_test.py index 71bd88d..38a6746 100644 --- a/Test/datafile_test.py +++ b/Test/regular/datafile_test.py @@ -7,7 +7,8 @@ import unittest, os, uuid import numpy as np import Algorithmia -from Algorithmia.datafile import DataFile, LocalDataFile +import json +from Algorithmia.datafile import DataFile, LocalDataFile, AdvancedDataFile class DataFileTest(unittest.TestCase): def setUp(self): @@ -113,5 +114,58 @@ def test_read_types(self): txt = self.client.file(self.EXISTING_FILE).getFile().read() self.assertEqual(txt, self.EXISTING_TEXT) +class AdvancedDataFileTest(unittest.TestCase): + def setUp(self): + self.client = Algorithmia.client() + if not self.client.dir("data://.my/empty").exists(): + self.client.dir("data://.my/empty").create() + + def test_get_nonexistant(self): + try: + with self.client.file('data://.my/nonexistant/nonreal') as f: + _ = f.read() + retrieved_file = True + except Exception as e: + retrieved_file = False + self.assertFalse(retrieved_file) + + def test_get_str(self): + df = self.client.file('data://.my/nonexistant/nonreal', cleanup=True) + try: + print(df.getString()) + retrieved_file = True + except Exception as e: + retrieved_file = False + self.assertFalse(retrieved_file) + + def test_putJson_getJson(self): + file = '.my/empty/test.json' + df = AdvancedDataFile(self.client, 'data://' + file, cleanup=True) + if sys.version_info[0] < 3: + payload = {u"hello":u"world"} + else: + payload = {"hello": "world"} + response = df.putJson(payload) + self.assertEqual(response.path,file) + result = json.loads(df.read()) + self.assertDictEqual(result, payload) + self.assertEqual(str(result), str(payload)) + + def test_putZipDir_getZipDir(self): + local_directory = os.path.join(os.getcwd(), "Test/resources/zip_directory") + remote_directory = "data://.my/empty/datafile.zip" + df = AdvancedDataFile(self.client, remote_directory, cleanup=True) + response = df.putAsZip(local_directory) + self.assertEqual(response, df) + + unzipped_local_path = df.getAsZip() + self.assertTrue(os.path.isdir(unzipped_local_path)) + found_files = [] + for _, _, files in os.walk(unzipped_local_path): + for file in files: + found_files.append(file) + self.assertEqual(len(found_files), 3) + + if __name__ == '__main__': unittest.main() diff --git a/Test/util_test.py b/Test/regular/util_test.py similarity index 100% rename from Test/util_test.py rename to Test/regular/util_test.py diff --git a/Test/resources/cert.cert b/Test/resources/cert.cert new file mode 100644 index 0000000..aeb562f --- /dev/null +++ b/Test/resources/cert.cert @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDazCCAlOgAwIBAgIUTikiwxFBpLW4pC+5VfOis1xCYKcwDQYJKoZIhvcNAQEL +BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMjA1MDMxNzE2MjZaFw0yMjA2 +MDIxNzE2MjZaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw +HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQDBUMZkg/bCJurQIB9znskTjv8URtIK6qqvZpYGTbfI +AzY6HiI0o1gPxjINZW7cBky/9MeEV5zyJghC4WoK099cIUNq2TmAWAjlRgIE8iEy +9z7QVfbSMainuw0RTlD5/8FRWtRe5v8qwbqLICMn3qv/KsG6bRezyS7UVihwFJua +E4dki+y6KSha4RrCtC43inbPlncB4om7PfJQyt5nI7N4KxbY2L3BUa5/+x1ux/ni +C/3y808vLJVQ6nLYgTEg/6K6lFrig0mUIMnCuOiBsrms3NmBPuDdRri/z1ulFHJB +WVQVQ5DgWher0f/dMzHwyRj3ffC8bAPlhrvLHwPQtNeRAgMBAAGjUzBRMB0GA1Ud +DgQWBBRoC77Hql6kEzk7WC6BeaPBu82K/jAfBgNVHSMEGDAWgBRoC77Hql6kEzk7 +WC6BeaPBu82K/jAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQCn +W9acM3+rxsiBBClTYEm2tOiukcXEkI7IzvW/4r7P24SmUiDD3vxVVbZ6nevVkg+P +4/QH+YYE3JUeXaN+xnHYjSy4NKxjd3EHT7BFxLMe0DQaodMj0klHqBtULNzojv8+ +/5tpQsjDLeeeDyOIJNz8r6CU9Gzh7j1EBF8BRdLA1z2UVmt6l6d4o3xOTYpOlZs3 +gI+ASxF9ODQzCCOeMYO2qiuMV3RD0oNdIEHUiMD+iHeC1jFGlxZzaWNeuUzP7Yj/ +MOwbBo8l6Hk2BUuUayLxZFLd0wN28IRkLEU5/SOh3mKz79nfPk6pD9rHUO1a53lI +Ua5xJ5tSwG6bMtNnHYYX +-----END CERTIFICATE----- diff --git a/Test/resources/cert.key b/Test/resources/cert.key new file mode 100644 index 0000000..1746297 --- /dev/null +++ b/Test/resources/cert.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDBUMZkg/bCJurQ +IB9znskTjv8URtIK6qqvZpYGTbfIAzY6HiI0o1gPxjINZW7cBky/9MeEV5zyJghC +4WoK099cIUNq2TmAWAjlRgIE8iEy9z7QVfbSMainuw0RTlD5/8FRWtRe5v8qwbqL +ICMn3qv/KsG6bRezyS7UVihwFJuaE4dki+y6KSha4RrCtC43inbPlncB4om7PfJQ +yt5nI7N4KxbY2L3BUa5/+x1ux/niC/3y808vLJVQ6nLYgTEg/6K6lFrig0mUIMnC +uOiBsrms3NmBPuDdRri/z1ulFHJBWVQVQ5DgWher0f/dMzHwyRj3ffC8bAPlhrvL +HwPQtNeRAgMBAAECggEAPr2OhhTmQ0EKOJ4UVxwTuotQci5CAVUELIUo78bNfNa+ +BMK+60KQVB5JJFvlTPemdS5miqc8wsJhMAOkvPripS4OiWES7nqj+HVuNli3OalQ +86DSyIlhaX6l0RYP5fOBtHu8LUjfS+swNfMqNchpHhmsYmsBpFIJJtUHrsihb7GR +4LpNOZ5go4+LG7FX9KaUE4FvAlS7hi6KLSMua10+3+NAlXggbcVikHr3Uq6eQIvk +z09cs+q2FHaESdTjXSIitmYOfJU5KK3QSfXAr/vaqakjnMvfp8MzQ5dHFsy03HRZ +Sy+LjRKOEOCMCT4DmGIPO4V89i3prbVH4JxixCOaeQKBgQDzuwERWE04JEtvfjxS +OAciQKLIxhfa4t2VB65d3115wxfDPIBYU5Mx5YV4aQyOddNxBwpmX/wYwstx2JDZ +2JM0OjOKLnSvlQfr5UmsY9jUO7CdmgC5HpgbHNhc8uJFw4pd+XypWSjytmVxBSdb +m0+in/iUUQuFNH/+BNLVVgWSiwKBgQDLDBCTEpKQvx2kAc8TEtwrWNhacZILab5D +StQBEL62VfGMdXYaA5dXreo5nqioHfBR3BfAmDecmq3iEFE8/yHJs6pLdcmj0Z1L +034UQedYLCmL9zuAgC6p4SKIMPubnYtMrNJOL3lq0ibogz3rfOhdN2B6S88IYoSL +M6asdoQN0wKBgCd1VPzr4MSAC75nH3joHS+Ma045087Z/6mK7s2/xbBax1QSTWz/ +Sss/L1aJG0FNDgg0bZiZXYTctHcf6oN6Loq8CXALiVSLuhaUrlK8b3QcncFGF2vg +6hspllWl9L/6okIIjAgWqSxyHwYnIXIRONlJMMNCQ60zDK2hNkjXflt1AoGAX0w3 +Tz/NSGBaogozTUlxymp1iOV63R5xLRYmsKVSTTPDHeBXYNhEpOM8ZnS/xb/fdhwt +jbgjib3TVKHB7zXzfr5zc91BmUCdaeRGbW2NDgYULdwIskP3IsZGtdL/lEb6BS+r +uQRxISCnIEPQwQCr8mw2PM/tyIqsmMTSOmmZiv8CgYBAfIC/cNwJyID6AVauZHQo +S3Bii9CPmPnuklBuS7ikX0bmZ93dzv537nqwGr0j9ksxabLWZRcoxx6MhgmXzXVT +dy48TWpqpHiMNorYskB9tcZSrBCl70bu5qKp2owqWHW0d4hqH3lkBNFhfwNWm+qC +54x3T/1fqyaqeapCiE5FGA== +-----END PRIVATE KEY----- diff --git a/Test/resources/manifests/example_manifest.json b/Test/resources/manifests/example_manifest.json new file mode 100644 index 0000000..ba6cbf5 --- /dev/null +++ b/Test/resources/manifests/example_manifest.json @@ -0,0 +1,29 @@ +{ + "required_files" : [ + { "name": "squeezenet", + "source_uri": "data://AlgorithmiaSE/image_cassification_demo/squeezenet1_1-f364aa15.pth", + "fail_on_tamper": true, + "metadata": { + "dataset_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266" + } + }, + { + "name": "labels", + "source_uri": "data://AlgorithmiaSE/image_cassification_demo/imagenet_class_index.json", + "fail_on_tamper": true, + "metadata": { + "dataset_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266" + } + } + ], + "optional_files": [ + { + "name": "mobilenet", + "source_uri": "data://AlgorithmiaSE/image_cassification_demo/mobilenet_v2-b0353104.pth", + "fail_on_tamper": false, + "metadata": { + "dataset_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266" + } + } + ] +} \ No newline at end of file diff --git a/Test/resources/zip_directory/root.json b/Test/resources/zip_directory/root.json new file mode 100644 index 0000000..5eed32d --- /dev/null +++ b/Test/resources/zip_directory/root.json @@ -0,0 +1 @@ +{"location": "root"} \ No newline at end of file diff --git a/Test/resources/zip_directory/subdirectory/__init__.py b/Test/resources/zip_directory/subdirectory/__init__.py new file mode 100644 index 0000000..b1a5ec6 --- /dev/null +++ b/Test/resources/zip_directory/subdirectory/__init__.py @@ -0,0 +1,3 @@ +from .build_wait import get_build +from .publish_algo import publish_algo +from .test_algo import test_algo diff --git a/Test/resources/zip_directory/subdirectory/subdir.json b/Test/resources/zip_directory/subdirectory/subdir.json new file mode 100644 index 0000000..aab19d7 --- /dev/null +++ b/Test/resources/zip_directory/subdirectory/subdir.json @@ -0,0 +1 @@ +{"foo": "bar"} \ No newline at end of file diff --git a/Test/self_signed/__init__.py b/Test/self_signed/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Test/algo_failure_test.py b/Test/self_signed/algo_failure_test.py similarity index 53% rename from Test/algo_failure_test.py rename to Test/self_signed/algo_failure_test.py index 236857a..aed319d 100644 --- a/Test/algo_failure_test.py +++ b/Test/self_signed/algo_failure_test.py @@ -6,30 +6,25 @@ import uvicorn import time from multiprocessing import Process + # look in ../ BEFORE trying to import Algorithmia. If you append to the # you will load the version installed on the computer. sys.path = ['../'] + sys.path from requests import Response - from Test.api import app - - def start_webserver(): - uvicorn.run(app, host="127.0.0.1", port=8080, log_level="debug") class AlgoTest(unittest.TestCase): error_500 = Response() error_500.status_code = 500 + error_message = "Non-Algorithm related Failure: " + str(error_500) + + @classmethod + def setUpClass(cls): + cls.client = Algorithmia.client(api_address="https://localhost:8090", api_key="simabcd123", ca_cert=False) - def setUp(self): - self.client = Algorithmia.client(api_address="http://localhost:8080") - self.uvi_p = Process(target=start_webserver) - self.uvi_p.start() - time.sleep(1) - def tearDown(self): - self.uvi_p.terminate() def test_throw_500_error_HTTP_response_on_algo_request(self): try: - result = self.client.algo('util/Echo').pipe(bytearray('foo','utf-8')) + result = self.client.algo('util/500').pipe(bytearray('foo', 'utf-8')) except Exception as e: result = e pass - self.assertEqual(str(self.error_500), str(result)) + self.assertEqual(str(self.error_message), str(result)) diff --git a/Test/self_signed/algo_test.py b/Test/self_signed/algo_test.py new file mode 100644 index 0000000..b3b377f --- /dev/null +++ b/Test/self_signed/algo_test.py @@ -0,0 +1,97 @@ +import sys +import os +from Algorithmia.errors import AlgorithmException +from Algorithmia.algorithm import OutputType +import Algorithmia + +import unittest + +# look in ../ BEFORE trying to import Algorithmia. If you append to the +# you will load the version installed on the computer. +sys.path = ['../'] + sys.path + +if sys.version_info.major >= 3: + + class AlgoDummyTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.client = Algorithmia.client(api_address="https://localhost:8090", api_key="simabcd123", ca_cert=False) + + def test_call_customCert(self): + result = self.client.algo('quality/echo').pipe(bytearray('foo', 'utf-8')) + self.assertEquals('binary', result.metadata.content_type) + self.assertEquals(bytearray('foo', 'utf-8'), result.result) + + def test_normal_call(self): + result = self.client.algo('quality/echo').pipe("foo") + self.assertEquals("text", result.metadata.content_type) + self.assertEquals("foo", result.result) + + def test_async_call(self): + result = self.client.algo('quality/echo').set_options(output=OutputType.void).pipe("foo") + self.assertTrue(hasattr(result, "async_protocol")) + self.assertTrue(hasattr(result, "request_id")) + + def test_raw_call(self): + result = self.client.algo('quality/echo').set_options(output=OutputType.raw).pipe("foo") + self.assertEquals("foo", result) + + def test_dict_call(self): + result = self.client.algo('quality/echo').pipe({"foo": "bar"}) + self.assertEquals("json", result.metadata.content_type) + self.assertEquals({"foo": "bar"}, result.result) + + def test_algo_exists(self): + result = self.client.algo('quality/echo').exists() + self.assertEquals(True, result) + + def test_algo_no_exists(self): + result = self.client.algo('quality/not_echo').exists() + self.assertEquals(False, result) + + # TODO: add more coverage examples to check kwargs + def test_get_versions(self): + result = self.client.algo('quality/echo').versions() + self.assertTrue('results' in result) + self.assertTrue('version_info' in result['results'][0]) + self.assertTrue('semantic_version' in result['results'][0]['version_info']) + self.assertEquals('0.1.0', result['results'][0]['version_info']['semantic_version']) + + def test_text_unicode(self): + telephone = u"\u260E" + # Unicode input to pipe() + result1 = self.client.algo('quality/echo').pipe(telephone) + self.assertEquals('text', result1.metadata.content_type) + self.assertEquals(telephone, result1.result) + + # Unicode return in .result + result2 = self.client.algo('quality/echo').pipe(result1.result) + self.assertEquals('text', result2.metadata.content_type) + self.assertEquals(telephone, result2.result) + + def test_algo_info(self): + result = self.client.algo('quality/echo').info() + self.assertTrue('results' in result) + self.assertTrue('resource_type' in result['results'][0]) + self.assertTrue(result['results'][0]['resource_type'] == "algorithm") + + def test_get_build_by_id(self): + result = self.client.algo("quality/echo").get_build("1a392e2c-b09f-4bae-a616-56c0830ac8e5") + self.assertTrue('commit_sha' in result) + + def test_get_build_logs(self): + result = self.client.algo("quality/echo").get_build_logs("1a392e2c-b09f-4bae-a616-56c0830ac8e5") + self.assertTrue('logs' in result) + + def test_get_scm_status(self): + result = self.client.algo("quality/echo").get_scm_status() + self.assertTrue('scm_connection_status' in result) + + def test_exception_ipa_algo(self): + try: + result = self.client.algo('zeryx/raise_exception').pipe("") + except AlgorithmException as e: + self.assertEqual(e.message, "This is an exception") + +if __name__ == '__main__': + unittest.main() diff --git a/requirements.txt b/requirements.txt index 6109d06..207a7f0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,8 +3,8 @@ six enum-compat toml argparse -algorithmia-api-client>=1.3,<1.4 -algorithmia-adk>=1.0.2,<1.1 +algorithmia-api-client==1.5.1 +algorithmia-adk>=1.2,<1.4 numpy<2 uvicorn==0.14.0 fastapi==0.65.2 diff --git a/requirements27.txt b/requirements27.txt index 1e7c2a8..8a118ea 100644 --- a/requirements27.txt +++ b/requirements27.txt @@ -3,6 +3,6 @@ six enum-compat toml argparse -algorithmia-api-client>=1.3,<1.4 -algorithmia-adk>=1.0.2,<1.1 +algorithmia-api-client==1.5.1 +algorithmia-adk>=1.2,<1.4 numpy<2 diff --git a/setup.py b/setup.py index 05b17d5..0069b73 100644 --- a/setup.py +++ b/setup.py @@ -21,8 +21,8 @@ 'enum-compat', 'toml', 'argparse', - 'algorithmia-api-client>=1.3,<1.4', - 'algorithmia-adk>=1.0.2,<1.1' + 'algorithmia-api-client==1.5.1', + 'algorithmia-adk>=1.2,<1.4' ], include_package_data=True, classifiers=[