From 5a6b79c5d3a892706d3f65adc3620dfa5915f1a7 Mon Sep 17 00:00:00 2001 From: Matt Daw Date: Tue, 7 Jun 2011 10:22:01 -0700 Subject: [PATCH 001/570] Initial version of JSON API. --- LICENSE | 2 +- shotgun_api3.py | 4945 +++++++++++++++++++++++++++-------------------- 2 files changed, 2798 insertions(+), 2149 deletions(-) diff --git a/LICENSE b/LICENSE index 7225a7ff8..a32a5bdcb 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2009-2010, Shotgun Software Inc +Copyright (c) 2009-2011, Shotgun Software Inc All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/shotgun_api3.py b/shotgun_api3.py index f8a7641bf..e50774f4a 100755 --- a/shotgun_api3.py +++ b/shotgun_api3.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # --------------------------------------------------------------------------------------------- -# Copyright (c) 2009-2010, Shotgun Software Inc +# Copyright (c) 2009-2011, Shotgun Software Inc # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -26,599 +26,762 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# --------------------------------------------------------------------------------------------- -# docs and latest version available for download at -# https://support.shotgunsoftware.com/forums/48807-developer-api-info -# --------------------------------------------------------------------------------------------- - -__version__ = "3.0.6" - -# --------------------------------------------------------------------------------------------- -# SUMMARY -# --------------------------------------------------------------------------------------------- -""" -Python Shotgun API library. -""" - -# --------------------------------------------------------------------------------------------- -# TODO -# --------------------------------------------------------------------------------------------- -""" - - add a configurable timeout duration (python xml-rpc lib never times out by default) - - include a native python https implementation, when native https is not available (e.g. maya's python) - - convert duration fields to/from a native python object? - - make file fields an http link to the file - - add logging functionality - - add scrubbing to text data sent to server to make sure it is all valid unicode - - support removing thumbnails / files (can only create or replace them now) -""" -# --------------------------------------------------------------------------------------------- -# CHANGELOG -# --------------------------------------------------------------------------------------------- -""" -+v3.0.6 - 2010 Jan 25 - + optimization: don't request paging_info unless required (and server support is available) - -+v3.0.5 - 2010 Dec 20 - + officially remove support for old api3_preview controller - + find(): allow requesting a specific page of results instead of returning them all at once - + add support for "session_uuid" parameter for communicating with a web browser session. - -+v3.0.3 - 2010 Nov 12 - + add support for local files. Injects convenience info into returned hash for local file links - + add support for authentication through http proxy server - -+v3.0.2 - 2010 May 10 - + add revive() method to revive deleted entities - -v3.0.1 - 2010 May 10 - + find(): default sorting to ascending, if not set (instead of requiring ascending/descending) - + upload() and upload_thumbnail(): pass auth info through - -v3.0 - 2010 May 5 - + add batch() method to do multiple create, update, and delete requests in one - request to the server (requires Shotgun server to be v1.13.0 or higher) - -v3.0b8 - 2010 Feb 19 - + fix python gotcha about using lists / dictionaries as defaults. See: - http://www.ferg.org/projects/python_gotchas.html#contents_item_6 - + add schema_read method - -v3.0b7 - 2009 November 30 - + add additional retries for connection errors and a catch for broken pipe exceptions - -v3.0b6 - 2009 October 20 - + add support for HTTP/1.1 keepalive, which greatly improves performance for multiple requests - + add more helpful error if server entered is not http or https - + add support assigning tags to file uploads (for Shotgun version >= 1.10.6) - -v3.0b5 - 2009 Sept 29 - + fixed deprecation warnings to raise Exception class for python 2.5 - -v3.0b4 - 2009 July 3 - + made upload() and upload_thumbnail() methods more backwards compatible - + changes to find_one(): - + now defaults to no filter_operators - -v3.0b3 - 2009 June 24 - + fixed upload() and upload_thumbnail() methods - + added download_attachment() method - + added schema_* methods for accessing entities and fields - + added support for http proxy servers - + added __version__ string - + removed RECORDS_PER_PAGE global (can just set records_per_page on the Shotgun object after initializing it) - + removed api_ver from the constructor, as this class is only designed to work with api v3 - -v3.0b2 - 2009 June 2 - + added preliminary support for http proxy servers - -v3.0b1 - 2009 May 25 - + updated to use v3 of the XML-RPC API to communicate with the Shotgun server - + the "limit" option for find() now works fully - + errors from the server are now raised as xml-rpc Fault exceptions (previously just wrote the error into the - results, and you had to check for it explicitly -- which most people didn't do, so they didn't see the errors) - + changes to find(): - + in the "order" param "column" has been renamed to "field_name" to be consistent - + new option for complex filters that allow grouping - + supports linked fields ("sg_project.Project.name") - + changes to create(): - + now accepts "return_fields" param, which is an array of field names to return when creating the entity. - Previously returned only the id. - -v1.2 - 2009 Apr 28 - + updated compatibility for Python 2.4+ - + added convert_datetimes_to_utc flag to assume all datetimes are in local time (disabled by default to maintain - current behavior) - + upload() now returns id of Attachment created - -v1.1 - 2009 Mar 27 - + added retired_only parameter to find() - + fixed bug preventing attachments from being uploaded without linking to a specific field - + minor error message formatting tweaks -""" +# needed for httplib2, future imports must be first +from __future__ import generators -# --------------------------------------------------------------------------------------------- -# Imports -# --------------------------------------------------------------------------------------------- -import cookielib -import cStringIO -import mimetools -import mimetypes +import base64 +import cookielib # used for attachment upload +import cStringIO # used for attachment upload +import datetime +import logging +import mimetools # used for attachment upload +import mimetypes # used for attachment upload import os import platform import re -import stat +import stat # used for attachment upload import sys import time import urllib -import urllib2 -from urlparse import urlparse +import urllib2 # used for image upload +import urlparse -# --------------------------------------------------------------------------------------------- -# Shotgun Object -# --------------------------------------------------------------------------------------------- -class ShotgunError(Exception): pass +log = logging.getLogger("shotgun_api3") -class Shotgun(object): - # Used to split up requests into batches of records_per_page when doing requests. this helps speed tremendously - # when getting lots of results back. doesn't affect the interface of the api at all (you always get the full set - # of results back as one array) but just how the client class communicates with the server. - records_per_page = 500 - schema_expire_mins = 60 +try: + import simplejson as json +except ImportError: + log.debug("simplejson not found, dropping back to json") + import json as json + +# ---------------------------------------------------------------------------- +# Errors + +class ShotgunError(Exception): + """Base for all Shotgun API Errors""" + pass + +class Fault(ShotgunError): + pass - def __init__(self, base_url, script_name, api_key, convert_datetimes_to_utc=True, http_proxy=None): - """ - Initialize Shotgun. +# ---------------------------------------------------------------------------- +# API + +class ServerCapabilities(object): + + def __init__(self, host, meta): + """Container for the servers capabilities, such as version and + paging. + + :param host: Host name for the server excluding protocol. + + :param meta: dict of meta data for the server returned from the + info api method. """ - self.server = None - if base_url.split("/")[0] not in ("http:","https:"): - raise ShotgunError("URL protocol must be http or https. Value was '%s'" % base_url) - self.base_url = "/".join(base_url.split("/")[0:3]) # cheesy way to strip off anything past the domain name, so: - # http://blah.com/asd => http://blah.com - self.script_name = script_name - self.api_key = api_key - self.api_ver = 'api3' - self.api_url = "%s/%s/" % (self.base_url, self.api_ver) - self.convert_datetimes_to_utc = convert_datetimes_to_utc - self.sid = None # only load this if needed - self.http_proxy = http_proxy - - server_options = { - 'server_url': self.api_url, - 'script_name': self.script_name, - 'script_key': self.api_key, - 'http_proxy' : self.http_proxy, - 'convert_datetimes_to_utc': self.convert_datetimes_to_utc - } - self._api3 = ShotgunCRUD(server_options) + #Server host name + self.host = host + try: + meta = dict(meta) + except ValueError: + meta = {} + self.server_info = meta + + #Version from server is major.minor.rev or major.minor.rev."Dev" + #Store version as triple and check dev flag + self.version = meta.get("version", None) + if not self.version: + self.version = (0,0,0) + self.is_dev = False + else: + if len(self.version) > 3 and self.version[3] == "Dev": + self.is_dev = True + else: + self.is_dev = False + self.version = tuple(self.version[:3]) - self.server_info = self._api3.info() - self._determine_features() + self._ensure_json_supported() - self.local_path_string = None - self.platform = self._determine_platform() - if self.platform: - self.local_path_string = "local_path_%s" % (self.platform) + #Flag if this server supports paging + self.has_paging = self._is_paging(self.version) - def _determine_features(self): - self.supports_paging_info = False + def _ensure_json_supported(self): + """Checks the server version supports the JSON api, raises an + exception if it does not. - if self.server_info.has_key('version'): - v = self.server_info['version'] - else: - return + :raises ShotgunError: The current server version does not support json + """ - if v[0] == 2 and v[1] == 3 and v[2] >= 4: - self.supports_paging_info = True - elif v[0] >= 2 and v[1] >= 4: - self.supports_paging_info = True + if not self.version or self.version < (2,4,0): + raise ShotgunError("JSON API requires server version 2.4 or "\ + "higher, server is %s" % (self.version,)) + return + def _is_paging(self, version): + """Determines if the server version supports paging. + :param version: Version to check. + + :returns: True if the server supports paging, False otherwise. + """ + if version >= (2, 3, 4): + return True + return False + + def __str__(self): + return "ServerCapabilities: host %s, version %s, is_dev %s, "\ + "has_paging %s" % (self.host, self.version, self.is_dev, + self.has_paging) + +class ClientCapabilities(object): - def _determine_platform(self): - s = platform.system().lower() - if s in ['windows','linux','darwin']: - if s == 'darwin': - return 'mac' - else: - return s - return None - - def _inject_field_values(self, records): + def __init__(self): + """Container for the client capabilities. + + Detects the current client platform and works out the SG field + used for local data paths. """ - Inject additional information into server results for convenience before returning - records back to the client. Currently this includes: - - 'image' value is rewritten to provide url to thumbnail image - - any local file link fields - 'local_file' key is set to match the current platform's path - 'url' key is set to match the current platform's url + + system = platform.system().lower() + if system =='darwin': + self.platform = "mac" + elif system in ('windows','linux'): + self.platform = system + else: + self.platform = None + + if self.platform: + self.local_path_field = "local_path_%s" % (self.platform) + else: + self.local_path_field = None + + self.py_version = ".".join( str(x) for x in sys.version_info[:2]) + + def __str__(self): + return "ClientCapabilities: platform %s, local_path_field %s, "\ + "py_verison %s" % (self.platform, self.local_path_field, + self.py_version) + +class _Config(object): + + def __init__(self): + """Container for the client configuration.""" + + self.max_rpc_attempts = 3 + self.timeout_secs = 3 + self.api_ver = 'api3' + self.convert_datetimes_to_utc = True + self.records_per_page = 500 + self.api_key = None + self.script_name = None + # uuid as a string + self.session_uuid = None + self.scheme = None + self.server = None + self.api_path = None + self.proxy_server = None + self.proxy_port = None + self.session_token = None + self.authorization = None + +class Shotgun(object): + """Shotgun Client Connection""" + + # reg ex from + # http://underground.infovark.com/2008/07/22/iso-date-validation-regex/ + # Note a length check is done before checking the reg ex + _DATE_PATTERN = re.compile( + "^(\d{4})\D?(0[1-9]|1[0-2])\D?([12]\d|0[1-9]|3[01])$") + _DATE_TIME_PATTERN = re.compile( + "^(\d{4})\D?(0[1-9]|1[0-2])\D?([12]\d|0[1-9]|3[01])"\ + "(\D?([01]\d|2[0-3])\D?([0-5]\d)\D?([0-5]\d)?\D?(\d{3})?)?$") + + def __init__(self, base_url, script_name, api_key, + convert_datetimes_to_utc=True, http_proxy=None): + """Initialises a new instance of the Shotgun client. + + :param base_url: http or https url to the shotgun server. + + :param script_name: name of the client script, used to authenticate + to the server. + + :param api_key: key assigned to the client script, used to + authenticate to the server. + + :param convert_datetimes_to_utc: If True date time values are + converted from local time to UTC time before been sent to the server. + Datetimes received from the server are converted back to local time. + If False the client should use UTC date time values. + Default is True. + + :param http_proxy: Optional, URL for the http proxy server, of the + form http://proxy.com:8080 """ - if len(records) == 0: - return records - for i,r in enumerate(records): - # skip results that aren't entity dictionaries - if type(r) is not dict: - continue + self.config = _Config() + self.config.api_key = api_key + self.config.script_name = script_name + self.config.convert_datetimes_to_utc = convert_datetimes_to_utc + self.config.proxy_info = http_proxy + + self._connection = None + + base_url = (base_url or "").lower() + self.config.scheme, self.config.server, api_base, _, _ = \ + urlparse.urlsplit(base_url) + if self.config.scheme not in ("http", "https"): + raise ValueError("base_url must use http or https got '%s'" % + base_url) + self.config.api_path = urlparse.urljoin(urlparse.urljoin( + api_base or "/", self.config.api_ver + "/"), "json") + + # if the service contains user information strip it out + # copied from the xmlrpclib which turned the user:password into + # and auth header + auth, self.config.server = urllib.splituser(self.config.server) + if auth: + auth = base64.encodestring(urllib.unquote(auth)) + self.config.authorization = "Basic " + auth.strip() + + if http_proxy: + _, proxy_netloc, _, _, _ = urlparse.urlsplit(http_proxy) + self.config.proxy_server, _, proxy_port = proxy_netloc.partition( + ":") + self.config.proxy_port = int(proxy_port or 8080) - # iterate over each item and check each field for possible injection - for k, v in r.items(): - # check for thumbnail - if k == 'image' and v: - records[i]['image'] = self._get_thumb_url(r['type'], r['id']) - - if type(v) == dict and 'link_type' in v and v['link_type'] == 'local' \ - and self.platform and self.local_path_string in r[k]: - records[i][k]['local_path'] = r[k][self.local_path_string] - records[i][k]['url'] = "file://%s" % (r[k]['local_path']) + self.client_caps = ClientCapabilities() + self._server_caps = None + #test to ensure the the server supports the json API + #call to server will only be made once and will raise error + sc = self.server_caps - return records + # ======================================================================== + # API Functions - def _get_thumb_url(self, entity_type, entity_id): + @property + def server_info(self): + return self.server_caps.server_info + + @property + def server_caps(self): """ - Returns the URL for the thumbnail of an entity given the - entity type and the entity id + :returns: ServerCapabilities that describe the server the client is + connected to. """ - url = self.base_url + "/upload/get_thumbnail_url?entity_type=%s&entity_id=%d"%(entity_type,entity_id) - for i in range(3): - f = urllib.urlopen(url) - response_code = f.readline().strip() - # something else happened. try again. found occasional connection errors still spit out html but not - # the correct response codes. usually trying again will right the ship. if not, we catch for it later. - if response_code not in ('0','1'): - continue - elif response_code == '1': - path = f.readline().strip() - if path: - return self.base_url + path - elif response_code == '0': - break - # if it's an error, message is printed on second line - raise ValueError, "%s:%s " % (entity_type,entity_id)+f.read().strip() - - def set_session_uuid(self, session_uuid): - server_options = { - 'server_url': self.api_url, - 'script_name': self.script_name, - 'script_key': self.api_key, - 'http_proxy' : self.http_proxy, - 'convert_datetimes_to_utc': self.convert_datetimes_to_utc, - 'session_uuid': session_uuid - } + if not self._server_caps or ( + self._server_caps.host != self.config.server): + self._server_caps = ServerCapabilities(self.config.server, + self.info()) + return self._server_caps + + def connect(self): + """Forces the client to connect to the server if it is not already + connected. + + NOTE: The client will automatically connect to the server. Only + call this function if you wish to confirm the client can connect. + """ + self._get_connection() + self.info() + return - self._api3 = ShotgunCRUD(server_options) - - def schema_read(self): - resp = self._api3.schema_read() - return resp["results"] - - def schema_field_read(self, entity_type, field_name=None): - args = { - "type":entity_type - } - if field_name: - args["field_name"] = field_name - resp = self._api3.schema_field_read(args) - return resp["results"] - - def schema_field_create(self, entity_type, data_type, display_name, properties=None): - if properties == None: - properties = {} - - args = { - "type":entity_type, - "data_type":data_type, - "properties":[{'property_name': 'name', 'value': display_name}] - } - for f,v in properties.items(): - args["properties"].append( {"property_name":f,"value":v} ) - resp = self._api3.schema_field_create(args) - return resp["results"] - - def schema_field_update(self, entity_type, field_name, properties): - args = { - "type":entity_type, - "field_name":field_name, - "properties":[] - } - for f,v in properties.items(): - args["properties"].append( {"property_name":f,"value":v} ) - resp = self._api3.schema_field_update(args) - return resp["results"] - - def schema_field_delete(self, entity_type, field_name): - args = { - "type":entity_type, - "field_name":field_name - } - resp = self._api3.schema_field_delete(args) - return resp["results"] - - def schema_entity_read(self): - resp = self._api3.schema_entity_read() - return resp["results"] - - def find(self, entity_type, filters, fields=None, order=None, filter_operator=None, limit=0, retired_only=False, page=0): + def close(self): + """Closes the current connection to the server. + + If the client needs to connect again it will do so automatically. + """ + self._close_connection() + return + + def info(self): + """Calls the Info function on the Shotgun API to get the server meta. + + :returns: dict of the server meta data. + """ + + return self._call_rpc("info", None, include_script_name=False) + + def find_one(self, entity_type, filters, fields=None, order=None, + filter_operator=None, retired_only=False): + """Calls the find() method and returns the first result, or None. + + The params are the same as for find(). """ - Find entities of entity_type matching the given filters. - The columns returned for each entity match the 'fields' - parameter provided, or just the id if nothing is specified. + results = self.find(entity_type, filters, fields, order, + filter_operator, 1, retired_only) + + if results: + return results[0] + return None + + def find(self, entity_type, filters, fields=None, order=None, + filter_operator=None, limit=0, retired_only=False, page=0): + """Find entities matching the given filters. + + :param entity_type: Required, entity type (string) to find. + + :param filters: Required, list of filters to apply. + + :param fields: Optional list of fields from the matched entities to + return. Defaults to id. + + :param order: Optional list of fields to order the results by, list + has the form [{'field_name':'foo','direction':'asc or desc'},] - Limit constrains the total results to its value. + :param filter_operator: Optional operator to apply to the filters, + supported values are 'all' and 'any'. Defaults to 'all'. - Returns an array of dict entities sorted by the optional - 'order' parameter. + :param limit: Optional, number of entities to return per page. + Defaults to 0 which returns all entities that match. + + :param page: Optional, page of results to return. By default all + results are returned. Use together with limit. + + :param retired_only: Optional, flag to return only entities that have + been retried. Defaults to False which returns only entities which + have not been retired. + + :returns: list of the dicts for each entity with the requested fields, + and their id and type. """ - if fields == None: - fields = ['id'] - if order == None: - order = [] - if type(filters) == type([]): + if not isinstance(limit, int) or limit < 0: + raise ValueError("limit parameter must be a positive integer") + + if not isinstance(page, int) or page < 0: + raise ValueError("page parameter must be a positive integer") + + if isinstance(filters, (list, tuple)): new_filters = {} + if not filter_operator or filter_operator == "all": new_filters["logical_operator"] = "and" else: new_filters["logical_operator"] = "or" - new_filters["conditions"] = [] - for f in filters: - new_filters["conditions"].append( {"path":f[0],"relation":f[1],"values":f[2:]} ) + new_filters["conditions"] = [ + {"path" : f[0], "relation" : f[1], "values" : f[2:]} + for f in filters + ] filters = new_filters elif filter_operator: - raise ShotgunError("Deprecated: Use of filter_operator for find() is not valid any more. See the documention on find()") - - if retired_only: - return_only = 'retired' - else: - return_only = 'active' - - req = { - "type": entity_type, - "return_fields": fields, + #TODO:Not sure if this test is correct, replicated from prev api + raise ShotgunError("Deprecated: Use of filter_operator for find()" + " is not valid any more. See the documentation on find()") + + params = { + "type" : entity_type, + "return_fields": fields or ["id"], "filters": filters, - "return_only" : return_only, - "paging": {"entities_per_page": self.records_per_page, "current_page": 1} + "return_only" : (retired_only and 'retired') or "active", + "paging": { + "entities_per_page": self.config.records_per_page, + "current_page": 1 + } } - - if self.supports_paging_info: - req["return_paging_info"] = True - + + if self.server_caps.has_paging: + params["return_paging_info"] = True + if order: - req['sorts'] = [] - for sort in order: - if sort.has_key('column'): - # TODO: warn about deprecation of 'column' param name - sort['field_name'] = sort['column'] - if not sort.has_key('direction'): - sort['direction'] = 'asc' - req['sorts'].append({'field_name': sort['field_name'],'direction' : sort['direction']}) - - if type(limit) != int or limit < 0: - raise ValueError("find() 'limit' parameter must be a positive integer") - elif (limit and limit > 0 and limit <= self.records_per_page): - req["paging"]["entities_per_page"] = limit - - # If page isn't set and the limit doesn't require pagination, then trigger the - # faster code path. + sort_list = [] + for sort in order: + if sort.has_key('column'): + # TODO: warn about deprecation of 'column' param name + sort['field_name'] = sort['column'] + sort.setdefault("direction", "asc") + sort_list.append({ + 'field_name': sort['field_name'], + 'direction' : sort['direction'] + }) + params['sorts'] = sort_list + + if limit and limit <= self.config.records_per_page: + params["paging"]["entities_per_page"] = limit + # If page isn't set and the limit doesn't require pagination, + # then trigger the faster code path. if page == 0: page = 1 - records = [] - # if page is specified, then only return the page of records requested - if type(page) != int or page < 0: - raise ValueError("find() 'page' parameter must be a positive integer") - elif page != 0: + if page != 0: # No paging_info needed, so optimize it out. - if self.supports_paging_info: - req["return_paging_info"] = False + if self.server_caps.has_paging: + params["return_paging_info"] = False - req["paging"]["current_page"] = page - resp = self._api3.read(req) - results = resp["results"]["entities"] - records.extend(results) - else: - done = False - while not done: - resp = self._api3.read(req) - results = resp["results"]["entities"] - if results: - records.extend(results) - if ( len(records) >= limit and limit > 0 ): - records = records[:limit] - done = True - elif len(records) == resp["results"]["paging_info"]["entity_count"]: - done = True - else: - req['paging']['current_page'] += 1 - else: - done = True - - records = self._inject_field_values(records) - - return records - - def find_one(self, entity_type, filters, fields=None, order=None, filter_operator=None, retired_only=False): - """ - Same as find, but only returns 1 result as a dict - """ - result = self.find(entity_type, filters, fields, order, filter_operator, 1, retired_only) - if len(result) > 0: - return result[0] - else: - return None - - def _required_keys(self, message, required_keys, data): - missing = set(required_keys) - set(data.keys()) - if missing: - raise ShotgunError("%s missing required key: %s. Value was: %s." % (message, ", ".join(missing), data)) - - def batch(self, requests): - if type(requests) != type([]): - raise ShotgunError("batch() expects a list. Instead was sent a %s"%type(requests)) + params["paging"]["current_page"] = page + records = self._call_rpc("read", params).get("entities", []) + return self._parse_records(records) + + records = [] + result = self._call_rpc("read", params) + while result.get("entities"): + records.extend(result.get("entities")) + + if limit and len(records) >= limit: + records = records[:limit] + break + if len(records) == result["paging_info"]["entity_count"]: + break + + params['paging']['current_page'] += 1 + result = self._call_rpc("read", params) - reqs = [] + return self._parse_records(records) + + def create(self, entity_type, data, return_fields=None): + """Create a new entity of the specified entity_type. - for r in requests: - self._required_keys("Batched request",['request_type','entity_type'],r) - - if r["request_type"] == "create": - self._required_keys("Batched create request",['data'],r) - - nr = { - "request_type": "create", - "type": r["entity_type"], - "fields": [] - } - - if "return_fields" in r: - nr["return_fields"] = r - - for f,v in r["data"].items(): - nr["fields"].append( { "field_name": f, "value": v } ) - - reqs.append(nr) - elif r["request_type"] == "update": - self._required_keys("Batched create request",['entity_id','data'],r) - - nr = { - "request_type": "update", - "type": r["entity_type"], - "id": r["entity_id"], - "fields": [] - } - - for f,v in r["data"].items(): - nr["fields"].append( { "field_name": f, "value": v } ) - - reqs.append(nr) - elif r["request_type"] == "delete": - self._required_keys("Batched delete request",['entity_id'],r) - - nr = { - "request_type": "delete", - "type": r["entity_type"], - "id": r["entity_id"] - } - - reqs.append(nr) - else: - raise ShotgunError("Invalid request_type for batch") + :param entity_type: Required, entity type (string) to create. - resp = self._api3.batch(reqs) - records = self._inject_field_values(resp["results"]) + :param data: Required, dict fields to set on the new entity. - return records - - def create(self, entity_type, data, return_fields=None): - """ - Create a new entity of entity_type type. + :param return_fields: Optional list of fields from the new entity + to return. Defaults to 'id' field. - 'data' is a dict of key=>value pairs of fieldname and value - to set the field to. + :returns: dict of the requested fields. """ - if return_fields == None: - return_fields = ['id'] - args = { - "type":entity_type, - "fields":[], - "return_fields":return_fields + params = { + "type" : entity_type, + "fields" : self._dict_to_list(data), + "return_fields" : return_fields or ["id"] } - for f,v in data.items(): - args["fields"].append( {"field_name":f,"value":v} ) - resp = self._api3.create(args) - record = self._inject_field_values([resp["results"]])[0] - return record - + record = self._call_rpc("create", params, first=True) + return self._parse_records(record)[0] + def update(self, entity_type, entity_id, data): - """ - Update an entity given the entity_type, and entity_id + """Updates the specified entity with the supplied data. - 'data' is a dict of key=>value pairs of fieldname and value - to set the field to. + :param entity_type: Required, entity type (string) to update. + + :param entity_id: Required, id of the entity to update. + + :param data: Required, dict fields to update on the entity. + + :returns: dict of the fields updated, with the entity_type and + id added. """ - args = {"type":entity_type,"id":entity_id,"fields":[]} - for f,v in data.items(): - args["fields"].append( {"field_name":f,"value":v} ) - resp = self._api3.update(args) - records = self._inject_field_values([resp["results"]]) - return records - + params = { + "type" : entity_type, + "id" : entity_id, + "fields" : self._dict_to_list(data) + } + + record = self._call_rpc("update", params) + return self._parse_records(record)[0] + def delete(self, entity_type, entity_id): + """Retire the specified entity. + + The entity can be brought back to life using the revive function. + + :param entity_type: Required, entity type (string) to delete. + + :param entity_id: Required, id of the entity to delete. + + :returns: True if the entity was deleted, False otherwise e.g. if the + entity has previously been deleted. """ - Retire an entity given the entity_type, and entity_id - """ - resp = self._api3.delete( {"type":entity_type, "id":entity_id} ) - return resp["results"] - + + params = { + "type" : entity_type, + "id" : entity_id + } + + return self._call_rpc("delete", params) + def revive(self, entity_type, entity_id): + """Revive an entity that has previously been deleted. + + :param entity_type: Required, entity type (string) to revive. + + :param entity_id: Required, id of the entity to revive. + + :returns: True if the entity was revived, False otherwise e.g. if the + entity has previously been revived (or was not deleted). """ - Revive an entity given the entity_type, and entity_id + + params = { + "type" : entity_type, + "id" : entity_id + } + + return self._call_rpc("revive", params) + + def batch(self, requests): + """Make a batch request of several create, update and delete calls. + + All requests are performed within a transaction, so either all will + complete or none will. + + :param requests: A list of dict's of the form which have a + request_type key and also specifies: + - create: entity_type, data dict of fields to set + - update: entity_type, entity_id, data dict of fields to set + - delete: entity_type and entity_id + + :returns: A list of values for each operation, create and update + requests return a dict of the fields updated. Delete requests + return True if the entity was deleted. """ - resp = self._api3.revive( {"type":entity_type, "id":entity_id} ) - return resp["results"] - - def upload(self, entity_type, entity_id, path, field_name=None, display_name=None, tag_list=None): + + if not isinstance(requests, list): + raise ShotgunError("batch() expects a list. Instead was sent "\ + "a %s" % type(requests)) + + calls = [] + + def _required_keys(message, required_keys, data): + if set(required_keys) - set(data.keys()): + raise ShotgunError("%s missing required key: %s. "\ + "Value was: %s." % (message, ", ".join(missing), data)) + + for req in requests: + _required_keys("Batched request", ['request_type','entity_type'], + req) + + if req["request_type"] == "create": + _required_keys("Batched create request", ['data'], req) + + calls.append({ + "request_type" : "create", + "type" : req["entity_type"], + "fields" : self._dict_to_list(req["data"]), + "return_fields" : req.get("return_fields") or["id"] + }) + + elif req["request_type"] == "update": + _required_keys("Batched update request", ['entity_id','data'], + req) + + calls.append({ + "request_type" : "update", + "type" : req["entity_type"], + "id" : req["entity_id"], + "fields" : self._dict_to_list(req["data"]), + }) + + elif req["request_type"] == "delete": + _required_keys("Batched delete request", ['entity_id'], req) + + calls.append({ + "request_type" : "delete", + "type" : req["entity_type"], + "id" : req["entity_id"], + }) + + else: + raise ShotgunError("Invalid request_type '%s' for batch" % ( + req["request_type"])) + + records = self._call_rpc("batch", calls) + return self._parse_records(records) + + def schema_entity_read(self): + """Gets all active entities defined in the schema. + + :returns: dict of Entity Type to dict containing the display name. """ - Upload a file as an attachment/thumbnail to the entity_type and entity_id - - @param entity_type: the entity type - @param entity_id: id for given entity to attach to - @param path: path to file on disk - @param field_name: the field on the entity to upload to (ignored if thumbnail) - @param display_name: the display name to use for the file in the ui (ignored if thumbnail) - @param tag_list: comma-separated string of tags to assign to the file + + return self._call_rpc("schema_entity_read", None) + + def schema_read(self): + """Gets the schema for all fields in all entities. + + :returns: nested dicts """ - is_thumbnail = (field_name == "thumb_image") - params = {} - params["entity_type"] = entity_type - params["entity_id"] = entity_id + return self._call_rpc("schema_read", None) + + def schema_field_read(self, entity_type, field_name=None): + """Gets all schema for fields in the specified entity_type or one + field. - # send auth, so server knows which - # script uploaded the file - params["script_name"] = self.script_name - params["script_key"] = self.api_key + :param entity_type: Required, entity type (string) to get the schema + for. - if not os.path.isfile(path): - raise ShotgunError("Path must be a valid file.") + :param field_name: Optional, name of the field to get the schema + definition for. If not supplied all fields for the entity type are + returned. - url = "%s/upload/upload_file" % (self.base_url) - if is_thumbnail: - url = "%s/upload/publish_thumbnail" % (self.base_url) - params["thumb_image"] = open(path, "rb") - else: - if display_name is None: - display_name = os.path.basename(path) - # we allow linking to nothing for generic reference use cases - if field_name is not None: - params["field_name"] = field_name - params["display_name"] = display_name - params["tag_list"] = tag_list - params["file"] = open(path, "rb") + :returns: dict of field name to nested dicts which describe the field + """ - # Create opener with extended form post support - opener = urllib2.build_opener(FormPostHandler) + params = { + "type" : entity_type, + } + if field_name: + params["field_name"] = field_name + + return self._call_rpc("schema_field_read", params) + + def schema_field_create(self, entity_type, data_type, display_name, + properties=None): + """Creates a field for the specified entity type. - # Perform the request - try: - result = opener.open(url, params).read() + :param entity_type: Required, entity type (string) to add the field to + + :param data_type: Required, Shotgun data type for the new field. + + :param display_name: Required, display name for the new field. + + :param properties: Optional, dict of properties for the new field. + + :returns: The Shotgun name (string) for the new field, this is + different to the display_name passed in. + """ + + params = { + "type" : entity_type, + "data_type" : data_type, + "properties" : [ + {'property_name': 'name', 'value': display_name} + ] + } + params["properties"].extend(self._dict_to_list(properties, + key_name="property_name", value_name="value")) + + return self._call_rpc("schema_field_create", params) + + def schema_field_update(self, entity_type, field_name, properties): + """Updates the specified field definition with the supplied + properties. + + :param entity_type: Required, entity type (string) to add the field to + + :param field_name: Required, Shotgun name of the field to update. + + :param properties: Required, dict of updated properties for the field. + + :returns: True if the field was updated, False otherwise. + """ + + params = { + "type" : entity_type, + "field_name" : field_name, + "properties": [ + {"property_name" : k, "value" : v} + for k,v in (properties or {}).iteritems() + ] + } + + return self._call_rpc("schema_field_update", params) + + def schema_field_delete(self, entity_type, field_name): + """Deletes the specified field definition from the entity_type. + + :param entity_type: Required, entity type (string) to delete the field + from. + + :param field_name: Required, Shotgun name of the field to delete. + + :param properties: Required, dict of updated properties for the field. + + :returns: True if the field was updated, False otherwise. + """ + + params = { + "type" : entity_type, + "field_name" : field_name + } + + return self._call_rpc("schema_field_delete", params) + + def set_session_uuid(self, session_uuid): + """Sets the browser session_uuid for this API session. + + Once set events generated by this API session will include the + session_uuid in their EventLogEntries. + + :param session_uuid: Session UUID to set. + """ + + self.config.session_uuid = session_uuid + return + + + def upload_thumbnail(self, entity_type, entity_id, path, **kwargs): + """Convenience function for uploading thumbnails, see upload. + """ + return self.upload(entity_type, entity_id, path, + field_name="thumb_image", **kwargs) + + def upload(self, entity_type, entity_id, path, field_name=None, + display_name=None, tag_list=None): + """Upload a file as an attachment/thumbnail to the specified + entity_type and entity_id. + + :param entity_type: Required, entity type (string) to revive. + + :param entity_id: Required, id of the entity to revive. + + :param path: path to file on disk + + :param field_name: the field on the entity to upload to + (ignored if thumbnail) + + :param display_name: the display name to use for the file in the ui + (ignored if thumbnail) + + :param tag_list: comma-separated string of tags to assign to the file + + :returns: Id of the new attachment. + """ + path = os.path.abspath(os.path.expanduser(path or "")) + if not os.path.isfile(path): + raise ShotgunError("Path must be a valid file, got '%s'" % path) + + is_thumbnail = (field_name == "thumb_image") + + params = { + "entity_type" : entity_type, + "entity_id" : entity_id, + "script_name" : self.config.script_name, + "script_key" : self.config.api_key, + } + if self.config.session_uuid: + params["session_uuid"] = self.config.session_uuid + + if is_thumbnail: + url = urlparse.urlunparse((self.config.scheme, self.config.server, + "/upload/publish_thumbnail", None, None, None)) + params["thumb_image"] = open(path, "rb") + + else: + url = urlparse.urlunparse((self.config.scheme, self.config.server, + "/upload/upload_file", None, None, None)) + if display_name is None: + display_name = os.path.basename(path) + # we allow linking to nothing for generic reference use cases + if field_name is not None: + params["field_name"] = field_name + params["display_name"] = display_name + params["tag_list"] = tag_list + params["file"] = open(path, "rb") + + # Create opener with extended form post support + opener = urllib2.build_opener(FormPostHandler) + + # Perform the request + try: + result = opener.open(url, params).read() except urllib2.HTTPError, e: if e.code == 500: - raise ShotgunError("Server encountered an internal error. \n%s\n(%s)\n%s\n\n" % (url, params, e)) + raise ShotgunError("Server encountered an internal error. " + "\n%s\n(%s)\n%s\n\n" % (url, params, e)) else: - raise ShotgunError("Unanticipated error occurred uploading %s: %s" % (path, e)) + raise ShotgunError("Unanticipated error occurred uploading " + "%s: %s" % (path, e)) else: if not str(result).startswith("1"): - raise ShotgunError("Could not upload file successfully, but not sure why.\nPath: %s\nUrl: %s\nError: %s" % (path, url, str(result))) + raise ShotgunError("Could not upload file successfully, but "\ + "not sure why.\nPath: %s\nUrl: %s\nError: %s" % ( + path, url, str(result))) # we changed the result string in the middle of 1.8 to return the id # remove once everyone is > 1.8.3 @@ -628,29 +791,31 @@ def upload(self, entity_type, entity_id, path, field_name=None, display_name=Non id = int(str(result).split(":")[1].split("\n")[0]) return id - def upload_thumbnail(self, entity_type, entity_id, path, **kwargs): - """ - Convenience function for thumbnail uploads. - """ - result = self.upload(entity_type, entity_id, path, field_name="thumb_image", **kwargs) - return result - - def download_attachment(self, entity_id): - """ - Gets session authentication and returns binary content of Attachment data + def download_attachment(self, attachment_id): + """Gets the returns binary content of the specified attachment. + + :param attachment_id: id of the attachment to get. + + :returns: binary data as a string """ + sid = self._get_session_token() - domain = urlparse(self.base_url)[1].split(':',1)[0] cj = cookielib.LWPCookieJar() - c = cookielib.Cookie('0', '_session_id', sid, None, False, domain, False, False, "/", True, False, None, True, None, None, {}) + c = cookielib.Cookie('0', '_session_id', sid, None, False, + self.config.server, False, False, "/", True, False, None, True, + None, None, {}) cj.set_cookie(c) cookie_handler = urllib2.HTTPCookieProcessor(cj) urllib2.install_opener(urllib2.build_opener(cookie_handler)) - url = '%s/file_serve/attachment/%s' % (self.base_url, entity_id) + url = urlparse.urlunparse((self.config.scheme, self.config.server, + "/file_serve/%s" % urllib.quote(str(attachment_id)), + None, None, None)) try: request = urllib2.Request(url) - request.add_header('User-agent','Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en-US; rv:1.9.0.7) Gecko/2009021906 Firefox/3.0.7') + request.add_header('User-agent', + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en-US; "\ + "rv:1.9.0.7) Gecko/2009021906 Firefox/3.0.7") attachment = urllib2.urlopen(request).read() except IOError, e: @@ -658,84 +823,454 @@ def download_attachment(self, entity_id): if hasattr(e, 'code'): err += "\nWe failed with error code - %s." % e.code elif hasattr(e, 'reason'): - err += "\nThe error object has the following 'reason' attribute :", e.reason - err += "\nThis usually means the server doesn't exist, is down, or we don't have an internet connection." + err += "\nThe error object has the following 'reason' "\ + "attribute :", e.reason + err += "\nThis usually means the server doesn't exist, is "\ + "down, or we don't have an internet connection." raise ShotgunError(err) else: if attachment.lstrip().startswith('= 300: + raise RuntimeError("HTTP error from server %s %s" % status) + return + + def _decode_response(self, headers, body): + """Decodes the response from the server from the wire format to + a python data structure. + + :param headers: Headers from the server. + + :param body: Raw response body from the server. + + :returns: If the content-type starts with application/json or + text/javascript the body is json decoded. Otherwise the raw body is + returned. + """ + if not body: + return body + + ct = (headers.get("content-type") or "application/json").lower() + + if ct.startswith("application/json") or \ + ct.startswith("text/javascript"): + return json.loads(body) + + return body + + def _response_errors(self, sg_response): + """Raises any API errors specified in the response. + + :raises ShotgunError: If the server response contains an exception. + """ + + if isinstance(sg_response, dict) and sg_response.get("exception"): + raise Fault(sg_response.get("message", + "Unknown Error")) + return + + def _visit_data(self, data, visitor): + """Walk the data (simple python types) and call the visitor.""" + + if not data: + return data + + recursive = self._visit_data + if isinstance(data, list): + return [recursive(i, visitor) for i in data] + + if isinstance(data, tuple): + return tuple(recursive(i, visitor) for i in data) + + if isinstance(data, dict): + return dict( + (k, recursive(v, visitor)) + for k,v in data.iteritems() + ) -class ShotgunCRUD(object): - def __init__(self, options): - self.__sg_url = options['server_url'] - self.__auth_args = {'script_name': options['script_name'], 'script_key': options['script_key']} + return visitor(data) + + def _transform_outbound(self, data): + """Transforms data types or values before they are sent by the + client. - if 'session_uuid' in options: - self.__auth_args['session_uuid'] = options['session_uuid'] + - changes timezones + - converts dates and times to strings + """ - if 'convert_datetimes_to_utc' in options: - convert_datetimes_to_utc = options['convert_datetimes_to_utc'] + if self.config.convert_datetimes_to_utc: + def _change_tz(value): + if value.tzinfo == None: + value = value.replace(tzinfo=sg_timezone.local) + return value.astimezone(sg_timezone.utc) else: - convert_datetimes_to_utc = 1 - if 'error_stream' in options: - self.__err_stream = options['error_stream'] + _change_tz = None + + local_now = datetime.datetime.now() + + def _outbound_visitor(value): + + if isinstance(value, datetime.datetime): + if _change_tz: + value = _change_tz(value) + + return value.strftime("%Y-%m-%dT%H:%M:%SZ") + + if isinstance(value, datetime.date): + #existing code did not tz transform dates. + return value.strftime("%Y-%m-%d") + + if isinstance(value, datetime.time): + value = local_now.replace(hour=value.hour, + minute=value.minute, second=value.second, + microsecond=value.microsecond) + if _change_tz: + value = _change_tz(value) + return value.strftime("%Y-%m-%dT%H:%M:%SZ") + + return value + + return self._visit_data(data, _outbound_visitor) + + def _transform_inbound(self, data): + """Transforms data types or values after they are received from the + server.""" + + #NOTE: The time zone is removed from the time after it is transformed + #to the local time, otherwise it will fail to compare to datetimes + #that do not have a time zone. + if self.config.convert_datetimes_to_utc: + _change_tz = lambda x: x.replace(tzinfo=sg_timezone.utc)\ + .astimezone(sg_timezone.local)\ + .replace(tzinfo=None) else: - self.__err_stream = 'sys.stderr' - if 'http_proxy' in options and options['http_proxy']: - p = ProxiedTransport() - p.set_proxy( options['http_proxy'] ) - self.__sg = ServerProxy(self.__sg_url, convert_datetimes_to_utc = convert_datetimes_to_utc, transport=p) + _change_tz = None + + def _inbound_visitor(value): + if isinstance(value, basestring): + if len(value) >= 19 and self._DATE_TIME_PATTERN.match(value): + try: + # strptime was not on datetime in python2.4 + value = datetime.datetime( + *time.strptime(value, "%Y-%m-%dT%H:%M:%SZ")[:6]) + except ValueError: + return value + if _change_tz: + return _change_tz(value) + return value + + #check for dates, no utz transform + if len(value) >= 10 and self._DATE_PATTERN.match(value): + try: + # strptime was not on datetime in python2.4 + return datetime.datetime( + *time.strptime(value, "%Y-%m-%d")[:6]).date() + except ValueError: + return value + return value + + return self._visit_data(data, _inbound_visitor) + + # ======================================================================== + # Connection Functions + + def _get_connection(self): + """Returns the current connection or creates a new connection to the + current server. + """ + if self._connection is not None: + return self._connection + + if self.config.proxy_server: + pi = ProxyInfo(PROXY_TYPE_HTTP, self.config.proxy_server, + self.config.proxy_port) + self._connection = Http(timeout=self.config.timeout_secs, + proxy_info=pi) else: - self.__sg = ServerProxy(self.__sg_url, convert_datetimes_to_utc = convert_datetimes_to_utc) - - def __getattr__(self, attr): - def callable(*args, **kwargs): - return self.meta_caller(attr, *args, **kwargs) - return callable - - def info(self): - try: - server_info = self.__sg.info() - except: - server_info = {} + self._connection = Http(timeout=self.config.timeout_secs) + + return self._connection + + def _close_connection(self): + """Closes the current connection.""" + if self._connection is None: + return + + for conn in self._connection.connections.values(): + try: + conn.close() + except Exception: + pass + self._connection.connections.clear() + self._connection = None + return + # ======================================================================== + # Utility + + def _parse_records(self, records): + """Parses 'records' returned from the api to insert thumbnail urls + or local file paths. + + :param records: List of records (dicts) to process or a single record. + + :returns: A list of the records processed. + """ + + if not records: + return [] + + if not isinstance(records, (list, tuple)): + records=[records,] + + for rec in records: + # skip results that aren't entity dictionaries + if not isinstance(rec, dict): + continue + + # iterate over each item and check each field for possible injection + for k, v in rec.iteritems(): + if not v: + continue + + # check for thumbnail + if k == 'image': + rec['image'] = self._build_thumb_url(rec['type'], + rec['id']) + continue + + if isinstance(v, dict) and v.get('link_type') == 'local' \ + and self.client_caps.local_path_field in v: + local_path = v[self.client_caps.local_path_field] + v['local_path'] = local_path + v['url'] = "file://%s" % (local_path or "",) - return server_info + return records - def meta_caller(self, attr, *args, **kwargs): - try: - return eval( - 'self._%s__sg.%s(self._%s__auth_args, *args, **kwargs)' % - (self.__class__.__name__, attr, self.__class__.__name__) - ) - except Fault, e: - if self.__err_stream: - eval('%s.write("\\n" + "-"*80 + "\\n")' % self.__err_stream) - eval('%s.write("XMLRPC Fault %s:\\n")' % (self.__err_stream, e.faultCode)) - eval('%s.write(e.faultString)' % self.__err_stream) - eval('%s.write("\\n" + "-"*80 + "\\n")' % self.__err_stream) - raise + def _build_thumb_url(self, entity_type, entity_id): + """Returns the URL for the thumbnail of an entity given the + entity type and the entity id. + + Note: This makes a call to the server for every thumbnail. + + :param entity_type: Entity type the id is for. + + :param entity_id: id of the entity to get the thumbnail for. + + :returns: Fully qualified url to the thumbnail. + """ + # Example response from the end point + # curl "https://foo.com/upload/get_thumbnail_url?entity_type=Version&entity_id=1" + # 1 + # /files/0000/0000/0012/232/shot_thumb.jpg.jpg + + url = "/upload/get_thumbnail_url?entity_type=%s&entity_id=%s" % ( + urllib.quote(entity_type), urllib.quote(str(entity_id))) + + _, _, body = self._make_call("GET", url, None, None) + + code, thumb_url = body.splitlines() + code = int(code) + + #code of 0 means error, second line is the error code + if code == 0: + raise ShotgunError(thumb_url) + + if code == 1: + return urlparse.urlunparse((self.config.scheme, + self.config.server, thumb_url.strip(), None, None, None)) + + # Comments in prev version said we can get this sometimes. + raise RuntimeError("Unknown code %s %s" % (code, thumb_url)) + + def _dict_to_list(self, d, key_name="field_name", value_name="value"): + """Utility function to convert a dict into a list dicts using the + key_name and value_name keys. + + e.g. d {'foo' : 'bar'} changed to [{'field_name':'foo, 'value':'bar'}] + """ + + return [ + {key_name : k, value_name : v } + for k,v in (d or {}).iteritems() + ] + + + + + + + +# ---------------------------------------------------------------------------- +# Helpers from the previous API, left as is. # Based on http://code.activestate.com/recipes/146306/ class FormPostHandler(urllib2.BaseHandler): @@ -791,59 +1326,59 @@ def https_request(self, request): - -# --------------------------------------------------------------------------------------------- +# ---------------------------------------------------------------------------- # SG_TIMEZONE module -# this is rolled into the this shotgun api file to avoid having to require current users of -# api2 to install new modules and modify PYTHONPATH info. -# --------------------------------------------------------------------------------------------- -from datetime import tzinfo, timedelta, datetime -import time as _time - -ZERO = timedelta(0) -STDOFFSET = timedelta(seconds = -_time.timezone) -if _time.daylight: - DSTOFFSET = timedelta(seconds = -_time.altzone) -else: - DSTOFFSET = STDOFFSET -DSTDIFF = DSTOFFSET - STDOFFSET +# this is rolled into the this shotgun api file to avoid having to require +# current users of api2 to install new modules and modify PYTHONPATH info. +# ---------------------------------------------------------------------------- class SgTimezone(object): + from datetime import tzinfo, timedelta, datetime + import time as _time + + ZERO = timedelta(0) + STDOFFSET = timedelta(seconds = -_time.timezone) + if _time.daylight: + DSTOFFSET = timedelta(seconds = -_time.altzone) + else: + DSTOFFSET = STDOFFSET + DSTDIFF = DSTOFFSET - STDOFFSET - def __init__(self): + def __init__(self): self.utc = self.UTC() self.local = self.LocalTimezone() class UTC(tzinfo): def utcoffset(self, dt): - return ZERO + return SgTimezone.ZERO def tzname(self, dt): return "UTC" def dst(self, dt): - return ZERO + return SgTimezone.ZERO class LocalTimezone(tzinfo): def utcoffset(self, dt): if self._isdst(dt): - return DSTOFFSET + return SgTimezone.DSTOFFSET else: - return STDOFFSET + return SgTimezone.STDOFFSET def dst(self, dt): if self._isdst(dt): - return DSTDIFF + return SgTimezone.DSTDIFF else: - return ZERO + return SgTimezone.ZERO def tzname(self, dt): return _time.tzname[self._isdst(dt)] def _isdst(self, dt): tt = (dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second, dt.weekday(), 0, -1) + import time as _time stamp = _time.mktime(tt) tt = _time.localtime(stamp) return tt.tm_isdst > 0 @@ -854,1622 +1389,1736 @@ def _isdst(self, dt): -# -# XML-RPC CLIENT LIBRARY -# $Id: xmlrpclib.py 41594 2005-12-04 19:11:17Z andrew.kuchling $ -# -# an XML-RPC client interface for Python. -# -# the marshalling and response parser code can also be used to -# implement XML-RPC servers. -# -# Notes: -# this version is designed to work with Python 2.1 or newer. -# -# History: -# 1999-01-14 fl Created -# 1999-01-15 fl Changed dateTime to use localtime -# 1999-01-16 fl Added Binary/base64 element, default to RPC2 service -# 1999-01-19 fl Fixed array data element (from Skip Montanaro) -# 1999-01-21 fl Fixed dateTime constructor, etc. -# 1999-02-02 fl Added fault handling, handle empty sequences, etc. -# 1999-02-10 fl Fixed problem with empty responses (from Skip Montanaro) -# 1999-06-20 fl Speed improvements, pluggable parsers/transports (0.9.8) -# 2000-11-28 fl Changed boolean to check the truth value of its argument -# 2001-02-24 fl Added encoding/Unicode/SafeTransport patches -# 2001-02-26 fl Added compare support to wrappers (0.9.9/1.0b1) -# 2001-03-28 fl Make sure response tuple is a singleton -# 2001-03-29 fl Don't require empty params element (from Nicholas Riley) -# 2001-06-10 fl Folded in _xmlrpclib accelerator support (1.0b2) -# 2001-08-20 fl Base xmlrpclib.Error on built-in Exception (from Paul Prescod) -# 2001-09-03 fl Allow Transport subclass to override getparser -# 2001-09-10 fl Lazy import of urllib, cgi, xmllib (20x import speedup) -# 2001-10-01 fl Remove containers from memo cache when done with them -# 2001-10-01 fl Use faster escape method (80% dumps speedup) -# 2001-10-02 fl More dumps microtuning -# 2001-10-04 fl Make sure import expat gets a parser (from Guido van Rossum) -# 2001-10-10 sm Allow long ints to be passed as ints if they don't overflow -# 2001-10-17 sm Test for int and long overflow (allows use on 64-bit systems) -# 2001-11-12 fl Use repr() to marshal doubles (from Paul Felix) -# 2002-03-17 fl Avoid buffered read when possible (from James Rucker) -# 2002-04-07 fl Added pythondoc comments -# 2002-04-16 fl Added __str__ methods to datetime/binary wrappers -# 2002-05-15 fl Added error constants (from Andrew Kuchling) -# 2002-06-27 fl Merged with Python CVS version -# 2002-10-22 fl Added basic authentication (based on code from Phillip Eby) -# 2003-01-22 sm Add support for the bool type -# 2003-02-27 gvr Remove apply calls -# 2003-04-24 sm Use cStringIO if available -# 2003-04-25 ak Add support for nil -# 2003-06-15 gn Add support for time.struct_time -# 2003-07-12 gp Correct marshalling of Faults -# 2003-10-31 mvl Add multicall support -# 2004-08-20 mvl Bump minimum supported Python version to 2.1 -# -# Copyright (c) 1999-2002 by Secret Labs AB. -# Copyright (c) 1999-2002 by Fredrik Lundh. -# -# info@pythonware.com -# http://www.pythonware.com -# -# -------------------------------------------------------------------- -# The XML-RPC client interface is -# -# Copyright (c) 1999-2002 by Secret Labs AB -# Copyright (c) 1999-2002 by Fredrik Lundh -# -# By obtaining, using, and/or copying this software and/or its -# associated documentation, you agree that you have read, understood, -# and will comply with the following terms and conditions: -# -# Permission to use, copy, modify, and distribute this software and -# its associated documentation for any purpose and without fee is -# hereby granted, provided that the above copyright notice appears in -# all copies, and that both that copyright notice and this permission -# notice appear in supporting documentation, and that the name of -# Secret Labs AB or the author not be used in advertising or publicity -# pertaining to distribution of the software without specific, written -# prior permission. -# -# SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD -# TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANT- -# ABILITY AND FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR -# BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY -# DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, -# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS -# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE -# OF THIS SOFTWARE. -# -------------------------------------------------------------------- -# -# things to look into some day: -# TODO: sort out True/False/boolean issues for Python 2.3 -""" -An XML-RPC client interface for Python. - -The marshalling and response parser code can also be used to -implement XML-RPC servers. - -Exported exceptions: - - Error Base class for client errors - ProtocolError Indicates an HTTP protocol error - ResponseError Indicates a broken response package - Fault Indicates an XML-RPC fault package - -Exported classes: - - ServerProxy Represents a logical connection to an XML-RPC server - - MultiCall Executor of boxcared xmlrpc requests - Boolean boolean wrapper to generate a "boolean" XML-RPC value - DateTime dateTime wrapper for an ISO 8601 string or time tuple or - localtime integer value to generate a "dateTime.iso8601" - XML-RPC value - Binary binary data wrapper - - SlowParser Slow but safe standard parser (based on xmllib) - Marshaller Generate an XML-RPC params chunk from a Python data structure - Unmarshaller Unmarshal an XML-RPC response from incoming XML event message - Transport Handles an HTTP transaction to an XML-RPC server - SafeTransport Handles an HTTPS transaction to an XML-RPC server - -Exported constants: - - True - False - -Exported functions: - - boolean Convert any Python value to an XML-RPC boolean - getparser Create instance of the fastest available parser & attach - to an unmarshalling object - dumps Convert an argument tuple or a Fault instance to an XML-RPC - request (or response, if the methodresponse option is used). - loads Convert an XML-RPC packet to unmarshalled data plus a method - name (None if not present). -""" -import re, string, time, operator -from types import * -import socket -import errno -import httplib -# -------------------------------------------------------------------- -# Internal stuff -try: - unicode -except NameError: - unicode = None # unicode support not available -try: - import datetime - #import sg_timezone -except ImportError: - datetime = None -try: - _bool_is_builtin = False.__class__.__name__ == "bool" -except NameError: - _bool_is_builtin = 0 - -def _decode(data, encoding, is8bit=re.compile("[\x80-\xff]").search): - # decode non-ascii string (if possible) - if unicode and encoding and is8bit(data): - data = unicode(data, encoding) - return data - -def escape(s, replace=string.replace): - s = replace(s, "&", "&") - s = replace(s, "<", "<") - return replace(s, ">", ">",) - -if unicode: - def _stringify(string): - # convert to 7-bit ascii if possible - try: - return string.encode("ascii") - except UnicodeError: - return string -else: - def _stringify(string): - return string - -#__version__ = "1.0.1" - -# xmlrpc integer limits -MAXINT = 2L**31-1 -MININT = -2L**31 - -# -------------------------------------------------------------------- -# Error constants (from Dan Libby's specification at -# http://xmlrpc-epi.sourceforge.net/specs/rfc.fault_codes.php) - -# Ranges of errors -PARSE_ERROR = -32700 -SERVER_ERROR = -32600 -APPLICATION_ERROR = -32500 -SYSTEM_ERROR = -32400 -TRANSPORT_ERROR = -32300 - -# Specific errors -NOT_WELLFORMED_ERROR = -32700 -UNSUPPORTED_ENCODING = -32701 -INVALID_ENCODING_CHAR = -32702 -INVALID_XMLRPC = -32600 -METHOD_NOT_FOUND = -32601 -INVALID_METHOD_PARAMS = -32602 -INTERNAL_ERROR = -32603 - -# -------------------------------------------------------------------- -# Exceptions - -## -# Base class for all kinds of client-side errors. - -class Error(Exception): - """Base class for client errors.""" - def __str__(self): - return repr(self) -## -# Indicates an HTTP-level protocol error. This is raised by the HTTP -# transport layer, if the server returns an error code other than 200 -# (OK). +# ---------------------------------------------------------------------------- +# Included external modules, left as is or otherwise noted here an in the code +# search for "amorton" +# +# - httplib2: contents from __init__.py and iri2uri.py +# - removed reference to the socks module, it is included directly. +# - set TCP_NODELAY on the sockets see +# http://code.google.com/p/httplib2/issues/detail?id=28 +# - patch to handle socket not opening applied +# see http://code.google.com/p/httplib2/source/detail?r=0cff83696d +# - socks module from http://socksipy.sourceforge.net/ # -# @param url The target URL. -# @param errcode The HTTP error code. -# @param errmsg The HTTP error message. -# @param headers The HTTP header dictionary. - -class ProtocolError(Error): - """Indicates an HTTP protocol error.""" - def __init__(self, url, errcode, errmsg, headers): - Error.__init__(self) - self.url = url - self.errcode = errcode - self.errmsg = errmsg - self.headers = headers - def __repr__(self): - return ( - "" % - (self.url, self.errcode, self.errmsg) - ) -## -# Indicates a broken XML-RPC response package. This exception is -# raised by the unmarshalling layer, if the XML-RPC response is -# malformed. +# ---------------------------------------------------------------------------- +# httplib2/iri2uri.py + +import urlparse -class ResponseError(Error): - """Indicates a broken response package.""" - pass -## -# Indicates an XML-RPC fault response package. This exception is -# raised by the unmarshalling layer, if the XML-RPC response contains -# a fault string. This exception can also used as a class, to -# generate a fault XML-RPC message. +# Convert an IRI to a URI following the rules in RFC 3987 +# +# The characters we need to enocde and escape are defined in the spec: # -# @param faultCode The XML-RPC fault code. -# @param faultString The XML-RPC fault string. - -class Fault(Error): - """Indicates an XML-RPC fault package.""" - def __init__(self, faultCode, faultString, **extra): - Error.__init__(self) - self.faultCode = faultCode - self.faultString = faultString - def __repr__(self): - return ( - "" % - (self.faultCode, repr(self.faultString)) - ) +# iprivate = %xE000-F8FF / %xF0000-FFFFD / %x100000-10FFFD +# ucschar = %xA0-D7FF / %xF900-FDCF / %xFDF0-FFEF +# / %x10000-1FFFD / %x20000-2FFFD / %x30000-3FFFD +# / %x40000-4FFFD / %x50000-5FFFD / %x60000-6FFFD +# / %x70000-7FFFD / %x80000-8FFFD / %x90000-9FFFD +# / %xA0000-AFFFD / %xB0000-BFFFD / %xC0000-CFFFD +# / %xD0000-DFFFD / %xE1000-EFFFD + +escape_range = [ + (0xA0, 0xD7FF ), + (0xE000, 0xF8FF ), + (0xF900, 0xFDCF ), + (0xFDF0, 0xFFEF), + (0x10000, 0x1FFFD ), + (0x20000, 0x2FFFD ), + (0x30000, 0x3FFFD), + (0x40000, 0x4FFFD ), + (0x50000, 0x5FFFD ), + (0x60000, 0x6FFFD), + (0x70000, 0x7FFFD ), + (0x80000, 0x8FFFD ), + (0x90000, 0x9FFFD), + (0xA0000, 0xAFFFD ), + (0xB0000, 0xBFFFD ), + (0xC0000, 0xCFFFD), + (0xD0000, 0xDFFFD ), + (0xE1000, 0xEFFFD), + (0xF0000, 0xFFFFD ), + (0x100000, 0x10FFFD) +] + +def encode(c): + retval = c + i = ord(c) + for low, high in escape_range: + if i < low: + break + if i >= low and i <= high: + retval = "".join(["%%%2X" % ord(o) for o in c.encode('utf-8')]) + break + return retval + + +def iri2uri(uri): + """Convert an IRI to a URI. Note that IRIs must be + passed in a unicode strings. That is, do not utf-8 encode + the IRI before passing it into the function.""" + if isinstance(uri ,unicode): + (scheme, authority, path, query, fragment) = urlparse.urlsplit(uri) + authority = authority.encode('idna') + # For each character in 'ucschar' or 'iprivate' + # 1. encode as utf-8 + # 2. then %-encode each octet of that utf-8 + uri = urlparse.urlunsplit((scheme, authority, path, query, fragment)) + uri = "".join([encode(c) for c in uri]) + return uri + +# ---------------------------------------------------------------------------- +# httplib2/__init__.py + + +# HACK: Amorton: future import moved to top of file +""" +httplib2 -# -------------------------------------------------------------------- -# Special values +A caching http interface that supports ETags and gzip +to conserve bandwidth. -## -# Wrapper for XML-RPC boolean values. Use the xmlrpclib.True and -# xmlrpclib.False constants, or the xmlrpclib.boolean() function, to -# generate boolean XML-RPC values. -# -# @param value A boolean value. Any true value is interpreted as True, -# all other values are interpreted as False. +Requires Python 2.3 or later -if _bool_is_builtin: - boolean = Boolean = bool - # to avoid breaking code which references xmlrpclib.{True,False} - True, False = True, False -else: - class Boolean: - """Boolean-value wrapper. - - Use True or False to generate a "boolean" XML-RPC value. - """ - - def __init__(self, value = 0): - self.value = operator.truth(value) - - def encode(self, out): - out.write("%d\n" % self.value) - - def __cmp__(self, other): - if isinstance(other, Boolean): - other = other.value - return cmp(self.value, other) - - def __repr__(self): - if self.value: - return "" % id(self) - else: - return "" % id(self) - - def __int__(self): - return self.value - - def __nonzero__(self): - return self.value - - True, False = Boolean(1), Boolean(0) - - ## - # Map true or false value to XML-RPC boolean values. - # - # @def boolean(value) - # @param value A boolean value. Any true value is mapped to True, - # all other values are mapped to False. - # @return xmlrpclib.True or xmlrpclib.False. - # @see Boolean - # @see True - # @see False - - def boolean(value, _truefalse=(False, True)): - """Convert any Python value to XML-RPC 'boolean'.""" - return _truefalse[operator.truth(value)] - -## -# Wrapper for XML-RPC DateTime values. This converts a time value to -# the format used by XML-RPC. -#

-# The value can be given as a string in the format -# "yyyymmddThh:mm:ss", as a 9-item time tuple (as returned by -# time.localtime()), or an integer value (as returned by time.time()). -# The wrapper uses time.localtime() to convert an integer to a time -# tuple. -# -# @param value The time, given as an ISO 8601 string, a time -# tuple, or a integer time value. +Changelog: +2007-08-18, Rick: Modified so it's able to use a socks proxy if needed. -class DateTime: - """DateTime wrapper for an ISO 8601 string or time tuple or - localtime integer value to generate 'dateTime.iso8601' XML-RPC - value. - """ - - def __init__(self, value=0): - if not isinstance(value, StringType): - if datetime and isinstance(value, datetime.datetime): - self.value = value.strftime("%Y%m%dT%H:%M:%S") - return - if datetime and isinstance(value, datetime.date): - self.value = value.strftime("%Y%m%dT%H:%M:%S") - return - if datetime and isinstance(value, datetime.time): - today = datetime.datetime.now().strftime("%Y%m%d") - self.value = value.strftime(today+"T%H:%M:%S") - return - if not isinstance(value, (TupleType, time.struct_time)): - if value == 0: - value = time.time() - value = time.localtime(value) - value = time.strftime("%Y%m%dT%H:%M:%S", value) - self.value = value - - def __cmp__(self, other): - if isinstance(other, DateTime): - other = other.value - return cmp(self.value, other) - - ## - # Get date/time value. - # - # @return Date/time value, as an ISO 8601 string. - - def __str__(self): - return self.value - - def __repr__(self): - return "" % (repr(self.value), id(self)) - - def decode(self, data): - data = str(data) - self.value = string.strip(data) - - def encode(self, out): - out.write("") - out.write(self.value) - out.write("\n") - -def _datetime(data): - # decode xml element contents into a DateTime structure. - value = DateTime() - value.decode(data) - return value - -def _datetime_type(data): - t = time.strptime(data, "%Y%m%dT%H:%M:%S") - return datetime.datetime(*tuple(t)[:6]) - -## -# Wrapper for binary data. This can be used to transport any kind -# of binary data over XML-RPC, using BASE64 encoding. -# -# @param data An 8-bit string containing arbitrary data. +""" +__author__ = "Joe Gregorio (joe@bitworking.org)" +__copyright__ = "Copyright 2006, Joe Gregorio" +__contributors__ = ["Thomas Broyer (t.broyer@ltgt.net)", + "James Antill", + "Xavier Verges Farrero", + "Jonathan Feinberg", + "Blair Zajac", + "Sam Ruby", + "Louis Nyffenegger"] +__license__ = "MIT" +__version__ = "$Rev$" + +import re +import sys +import email +import email.Utils +import email.Message +import email.FeedParser +import StringIO +import gzip +import zlib +import httplib +import urlparse import base64 +import os +import copy +import calendar +import time +import random +# remove depracated warning in python2.6 try: - import cStringIO as StringIO + from hashlib import sha1 as _sha, md5 as _md5 except ImportError: - import StringIO + import sha + import md5 + _sha = sha.new + _md5 = md5.new +import hmac +from gettext import gettext as _ +import socket -class Binary: - """Wrapper for binary data.""" - - def __init__(self, data=None): - self.data = data - - ## - # Get buffer contents. - # - # @return Buffer contents, as an 8-bit string. - - def __str__(self): - return self.data or "" - - def __cmp__(self, other): - if isinstance(other, Binary): - other = other.data - return cmp(self.data, other) - - def decode(self, data): - self.data = base64.decodestring(data) - - def encode(self, out): - out.write("\n") - base64.encode(StringIO.StringIO(self.data), out) - out.write("\n") +# HACK: amorton sock module is included in this file now +# try: +# import socks +# except ImportError: +# socks = None -def _binary(data): - # decode xml element contents into a Binary structure - value = Binary() - value.decode(data) - return value +# Build the appropriate socket wrapper for ssl +try: + import ssl # python 2.6 + _ssl_wrap_socket = ssl.wrap_socket +except ImportError: + def _ssl_wrap_socket(sock, key_file, cert_file): + ssl_sock = socket.ssl(sock, key_file, cert_file) + return httplib.FakeSocket(sock, ssl_sock) -WRAPPERS = (DateTime, Binary) -if not _bool_is_builtin: - WRAPPERS = WRAPPERS + (Boolean,) -# -------------------------------------------------------------------- -# XML parsers +if sys.version_info >= (2,3): + # amorton: code pulled into this file, see above + #from iri2uri import iri2uri + pass +else: + def iri2uri(uri): + return uri -try: - # optional xmlrpclib accelerator - import _xmlrpclib - FastParser = _xmlrpclib.Parser - FastUnmarshaller = _xmlrpclib.Unmarshaller -except (AttributeError, ImportError): - FastParser = FastUnmarshaller = None +def has_timeout(timeout): # python 2.6 + if hasattr(socket, '_GLOBAL_DEFAULT_TIMEOUT'): + return (timeout is not None and timeout is not socket._GLOBAL_DEFAULT_TIMEOUT) + return (timeout is not None) -try: - import _xmlrpclib - FastMarshaller = _xmlrpclib.Marshaller -except (AttributeError, ImportError): - FastMarshaller = None +__all__ = ['Http', 'Response', 'ProxyInfo', 'HttpLib2Error', + 'RedirectMissingLocation', 'RedirectLimit', 'FailedToDecompressContent', + 'UnimplementedDigestAuthOptionError', 'UnimplementedHmacDigestAuthOptionError', + 'debuglevel'] -# -# the SGMLOP parser is about 15x faster than Python's builtin -# XML parser. SGMLOP sources can be downloaded from: -# -# http://www.pythonware.com/products/xml/sgmlop.htm -# -try: - import sgmlop - if not hasattr(sgmlop, "XMLParser"): - raise ImportError -except ImportError: - SgmlopParser = None # sgmlop accelerator not available -else: - class SgmlopParser: - def __init__(self, target): - - # setup callbacks - self.finish_starttag = target.start - self.finish_endtag = target.end - self.handle_data = target.data - self.handle_xml = target.xml - - # activate parser - self.parser = sgmlop.XMLParser() - self.parser.register(self) - self.feed = self.parser.feed - self.entity = { - "amp": "&", "gt": ">", "lt": "<", - "apos": "'", "quot": '"' - } - - def close(self): - try: - self.parser.close() - finally: - self.parser = self.feed = None # nuke circular reference - - def handle_proc(self, tag, attr): - m = re.search("encoding\s*=\s*['\"]([^\"']+)[\"']", attr) - if m: - self.handle_xml(m.group(1), 1) - - def handle_entityref(self, entity): - # entity - try: - self.handle_data(self.entity[entity]) - except KeyError: - self.handle_data("&%s;" % entity) +# The httplib debug level, set to a non-zero value to get debug output +debuglevel = 0 -try: - from xml.parsers import expat - if not hasattr(expat, "ParserCreate"): - raise ImportError -except ImportError: - ExpatParser = None # expat not available -else: - class ExpatParser: - # fast expat parser for Python 2.0 and later. this is about - # 50% slower than sgmlop, on roundtrip testing - def __init__(self, target): - self._parser = parser = expat.ParserCreate(None, None) - self._target = target - parser.StartElementHandler = target.start - parser.EndElementHandler = target.end - parser.CharacterDataHandler = target.data - encoding = None - if not parser.returns_unicode: - encoding = "utf-8" - target.xml(encoding, None) - - def feed(self, data): - self._parser.Parse(data, 0) - - def close(self): - self._parser.Parse("", 1) # end of data - del self._target, self._parser # get rid of circular references - -class SlowParser: - """Default XML parser (based on xmllib.XMLParser).""" - # this is about 10 times slower than sgmlop, on roundtrip - # testing. - def __init__(self, target): - import xmllib # lazy subclassing (!) - if xmllib.XMLParser not in SlowParser.__bases__: - SlowParser.__bases__ = (xmllib.XMLParser,) - self.handle_xml = target.xml - self.unknown_starttag = target.start - self.handle_data = target.data - self.handle_cdata = target.data - self.unknown_endtag = target.end - try: - xmllib.XMLParser.__init__(self, accept_utf8=1) - except TypeError: - xmllib.XMLParser.__init__(self) # pre-2.0 -# -------------------------------------------------------------------- -# XML-RPC marshalling and unmarshalling code +# Python 2.3 support +if sys.version_info < (2,4): + def sorted(seq): + seq.sort() + return seq -## -# XML-RPC marshaller. -# -# @param encoding Default encoding for 8-bit strings. The default -# value is None (interpreted as UTF-8). -# @see dumps +# Python 2.3 support +def HTTPResponse__getheaders(self): + """Return list of (header, value) tuples.""" + if self.msg is None: + raise httplib.ResponseNotReady() + return self.msg.items() -class Marshaller: - """Generate an XML-RPC params chunk from a Python data structure. - - Create a Marshaller instance for each set of parameters, and use - the "dumps" method to convert your data (represented as a tuple) - to an XML-RPC params chunk. To write a fault response, pass a - Fault instance instead. You may prefer to use the "dumps" module - function for this purpose. - """ - - # by the way, if you don't understand what's going on in here, - # that's perfectly ok. - - def __init__(self, encoding=None, allow_none=1, convert_datetimes_to_utc=1): - self.memo = {} - self.data = None - self.encoding = encoding - self.allow_none = allow_none - self.convert_datetimes_to_utc = convert_datetimes_to_utc - - dispatch = {} - - def dumps(self, values): - out = [] - write = out.append - dump = self.__dump - if isinstance(values, Fault): - # fault instance - write("\n") - dump({'faultCode': values.faultCode, - 'faultString': values.faultString}, - write) - write("\n") - else: - # parameter block - # FIXME: the xml-rpc specification allows us to leave out - # the entire block if there are no parameters. - # however, changing this may break older code (including - # old versions of xmlrpclib.py), so this is better left as - # is for now. See @XMLRPC3 for more information. /F - write("\n") - for v in values: - write("\n") - dump(v, write) - write("\n") - write("\n") - result = string.join(out, "") - return result - - def __dump(self, value, write): - try: - f = self.dispatch[type(value)] - except KeyError: - raise TypeError, "cannot marshal %s objects" % type(value) - else: - f(self, value, write) - - def dump_nil (self, value, write): - if not self.allow_none: - raise TypeError, "cannot marshal None unless allow_none is enabled" - write("") - dispatch[NoneType] = dump_nil - - def dump_int(self, value, write): - # in case ints are > 32 bits - if value > MAXINT or value < MININT: - raise OverflowError, "int exceeds XML-RPC limits" - write("") - write(str(value)) - write("\n") - dispatch[IntType] = dump_int - - if _bool_is_builtin: - def dump_bool(self, value, write): - write("") - write(value and "1" or "0") - write("\n") - dispatch[bool] = dump_bool - - def dump_long(self, value, write): - if value > MAXINT or value < MININT: - raise OverflowError, "long int exceeds XML-RPC limits" - write("") - write(str(int(value))) - write("\n") - dispatch[LongType] = dump_long - - def dump_double(self, value, write): - write("") - write(repr(value)) - write("\n") - dispatch[FloatType] = dump_double - - def dump_string(self, value, write, escape=escape): - write("") - write(escape(value)) - write("\n") - dispatch[StringType] = dump_string - - if unicode: - def dump_unicode(self, value, write, escape=escape): - value = value.encode(self.encoding) - write("") - write(escape(value)) - write("\n") - dispatch[UnicodeType] = dump_unicode - - def dump_array(self, value, write): - i = id(value) - if self.memo.has_key(i): - raise TypeError, "cannot marshal recursive sequences" - self.memo[i] = None - dump = self.__dump - write("\n") - for v in value: - dump(v, write) - write("\n") - del self.memo[i] - dispatch[TupleType] = dump_array - dispatch[ListType] = dump_array - - def dump_struct(self, value, write, escape=escape): - i = id(value) - if self.memo.has_key(i): - raise TypeError, "cannot marshal recursive dictionaries" - self.memo[i] = None - dump = self.__dump - write("\n") - for k, v in value.items(): - write("\n") - if type(k) is not StringType: - if unicode and type(k) is UnicodeType: - k = k.encode(self.encoding) - else: - raise TypeError, "dictionary key must be string" - write("%s\n" % escape(k)) - dump(v, write) - write("\n") - write("\n") - del self.memo[i] - dispatch[DictType] = dump_struct - - if datetime: - def dump_datetime(self, value, write): - if self.convert_datetimes_to_utc: - if value.tzinfo == None: - value = value.replace(tzinfo = sg_timezone.local) - value = value.astimezone(sg_timezone.utc) - write("") - write(value.strftime("%Y%m%dT%H:%M:%S")) - write("\n") - dispatch[datetime.datetime] = dump_datetime - - def dump_date(self, value, write): - write("") - write(value.strftime("%Y%m%dT00:00:00")) - write("\n") - dispatch[datetime.date] = dump_date - - def dump_time(self, value, write): - if self.convert_datetimes_to_utc: - if value.tzinfo == None: - value = value.replace(tzinfo = sg_timezone.local) - value = value.astimezone(sg_timezone.utc) - write("") - write(datetime.datetime.now().date().strftime("%Y%m%dT")) - write(value.strftime("%H:%M:%S")) - write("\n") - dispatch[datetime.time] = dump_time - - def dump_instance(self, value, write): - # check for special wrappers - if value.__class__ in WRAPPERS: - self.write = write - value.encode(self) - del self.write - else: - # store instance attributes as a struct (really?) - self.dump_struct(value.__dict__, write) - dispatch[InstanceType] = dump_instance +if not hasattr(httplib.HTTPResponse, 'getheaders'): + httplib.HTTPResponse.getheaders = HTTPResponse__getheaders -## -# XML-RPC unmarshaller. -# -# @see loads +# All exceptions raised here derive from HttpLib2Error +class HttpLib2Error(Exception): pass -class Unmarshaller: - """Unmarshal an XML-RPC response, based on incoming XML event - messages (start, data, end). Call close() to get the resulting - data structure. - - Note that this reader is fairly tolerant, and gladly accepts bogus - XML-RPC data without complaining (but not bogus XML). +# Some exceptions can be caught and optionally +# be turned back into responses. +class HttpLib2ErrorWithResponse(HttpLib2Error): + def __init__(self, desc, response, content): + self.response = response + self.content = content + HttpLib2Error.__init__(self, desc) + +class RedirectMissingLocation(HttpLib2ErrorWithResponse): pass +class RedirectLimit(HttpLib2ErrorWithResponse): pass +class FailedToDecompressContent(HttpLib2ErrorWithResponse): pass +class UnimplementedDigestAuthOptionError(HttpLib2ErrorWithResponse): pass +class UnimplementedHmacDigestAuthOptionError(HttpLib2ErrorWithResponse): pass + +class RelativeURIError(HttpLib2Error): pass +class ServerNotFoundError(HttpLib2Error): pass + +# Open Items: +# ----------- +# Proxy support + +# Are we removing the cached content too soon on PUT (only delete on 200 Maybe?) + +# Pluggable cache storage (supports storing the cache in +# flat files by default. We need a plug-in architecture +# that can support Berkeley DB and Squid) + +# == Known Issues == +# Does not handle a resource that uses conneg and Last-Modified but no ETag as a cache validator. +# Does not handle Cache-Control: max-stale +# Does not use Age: headers when calculating cache freshness. + + +# The number of redirections to follow before giving up. +# Note that only GET redirects are automatically followed. +# Will also honor 301 requests by saving that info and never +# requesting that URI again. +DEFAULT_MAX_REDIRECTS = 5 + +# Which headers are hop-by-hop headers by default +HOP_BY_HOP = ['connection', 'keep-alive', 'proxy-authenticate', 'proxy-authorization', 'te', 'trailers', 'transfer-encoding', 'upgrade'] + +def _get_end2end_headers(response): + hopbyhop = list(HOP_BY_HOP) + hopbyhop.extend([x.strip() for x in response.get('connection', '').split(',')]) + return [header for header in response.keys() if header not in hopbyhop] + +URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?") + +def parse_uri(uri): + """Parses a URI using the regex given in Appendix B of RFC 3986. + + (scheme, authority, path, query, fragment) = parse_uri(uri) + """ + groups = URI.match(uri).groups() + return (groups[1], groups[3], groups[4], groups[6], groups[8]) + +def urlnorm(uri): + (scheme, authority, path, query, fragment) = parse_uri(uri) + if not scheme or not authority: + raise RelativeURIError("Only absolute URIs are allowed. uri = %s" % uri) + authority = authority.lower() + scheme = scheme.lower() + if not path: + path = "/" + # Could do syntax based normalization of the URI before + # computing the digest. See Section 6.2.2 of Std 66. + request_uri = query and "?".join([path, query]) or path + scheme = scheme.lower() + defrag_uri = scheme + "://" + authority + request_uri + return scheme, authority, request_uri, defrag_uri + + +# Cache filename construction (original borrowed from Venus http://intertwingly.net/code/venus/) +re_url_scheme = re.compile(r'^\w+://') +re_slash = re.compile(r'[?/:|]+') + +def safename(filename): + """Return a filename suitable for the cache. + + Strips dangerous and common characters to create a filename we + can use to store the cache in. + """ + + try: + if re_url_scheme.match(filename): + if isinstance(filename,str): + filename = filename.decode('utf-8') + filename = filename.encode('idna') + else: + filename = filename.encode('idna') + except UnicodeError: + pass + if isinstance(filename,unicode): + filename=filename.encode('utf-8') + filemd5 = _md5(filename).hexdigest() + filename = re_url_scheme.sub("", filename) + filename = re_slash.sub(",", filename) + + # limit length of filename + if len(filename)>200: + filename=filename[:200] + return ",".join((filename, filemd5)) + +NORMALIZE_SPACE = re.compile(r'(?:\r\n)?[ \t]+') +def _normalize_headers(headers): + return dict([ (key.lower(), NORMALIZE_SPACE.sub(value, ' ').strip()) for (key, value) in headers.iteritems()]) + +def _parse_cache_control(headers): + retval = {} + if headers.has_key('cache-control'): + parts = headers['cache-control'].split(',') + parts_with_args = [tuple([x.strip().lower() for x in part.split("=", 1)]) for part in parts if -1 != part.find("=")] + parts_wo_args = [(name.strip().lower(), 1) for name in parts if -1 == name.find("=")] + retval = dict(parts_with_args + parts_wo_args) + return retval + +# Whether to use a strict mode to parse WWW-Authenticate headers +# Might lead to bad results in case of ill-formed header value, +# so disabled by default, falling back to relaxed parsing. +# Set to true to turn on, usefull for testing servers. +USE_WWW_AUTH_STRICT_PARSING = 0 + +# In regex below: +# [^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+ matches a "token" as defined by HTTP +# "(?:[^\0-\x08\x0A-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?" matches a "quoted-string" as defined by HTTP, when LWS have already been replaced by a single space +# Actually, as an auth-param value can be either a token or a quoted-string, they are combined in a single pattern which matches both: +# \"?((?<=\")(?:[^\0-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?(?=\")|(?@,;:\\\"/[\]?={} \t]+(?!\"))\"? +WWW_AUTH_STRICT = re.compile(r"^(?:\s*(?:,\s*)?([^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+)\s*=\s*\"?((?<=\")(?:[^\0-\x08\x0A-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?(?=\")|(?@,;:\\\"/[\]?={} \t]+(?!\"))\"?)(.*)$") +WWW_AUTH_RELAXED = re.compile(r"^(?:\s*(?:,\s*)?([^ \t\r\n=]+)\s*=\s*\"?((?<=\")(?:[^\\\"]|\\.)*?(?=\")|(? current_age: + retval = "FRESH" + return retval + +def _decompressContent(response, new_content): + content = new_content + try: + encoding = response.get('content-encoding', None) + if encoding in ['gzip', 'deflate']: + if encoding == 'gzip': + content = gzip.GzipFile(fileobj=StringIO.StringIO(new_content)).read() + if encoding == 'deflate': + content = zlib.decompress(content) + response['content-length'] = str(len(content)) + # Record the historical presence of the encoding in a way the won't interfere. + response['-content-encoding'] = response['content-encoding'] + del response['content-encoding'] + except IOError: + content = "" + raise FailedToDecompressContent(_("Content purported to be compressed with %s but failed to decompress.") % response.get('content-encoding'), response, content) + return content + +def _updateCache(request_headers, response_headers, content, cache, cachekey): + if cachekey: + cc = _parse_cache_control(request_headers) + cc_response = _parse_cache_control(response_headers) + if cc.has_key('no-store') or cc_response.has_key('no-store'): + cache.delete(cachekey) else: - return f(self, data) - - # - # element decoders - - dispatch = {} - - def end_nil (self, data): - self.append(None) - self._value = 0 - dispatch["nil"] = end_nil - - def end_boolean(self, data): - if data == "0": - self.append(False) - elif data == "1": - self.append(True) + info = email.Message.Message() + for key, value in response_headers.iteritems(): + if key not in ['status','content-encoding','transfer-encoding']: + info[key] = value + + # Add annotations to the cache to indicate what headers + # are variant for this request. + vary = response_headers.get('vary', None) + if vary: + vary_headers = vary.lower().replace(' ', '').split(',') + for header in vary_headers: + key = '-varied-%s' % header + try: + info[key] = request_headers[header] + except KeyError: + pass + + status = response_headers.status + if status == 304: + status = 200 + + status_header = 'status: %d\r\n' % response_headers.status + + header_str = info.as_string() + + header_str = re.sub("\r(?!\n)|(? 0: + service = "cl" + # No point in guessing Base or Spreadsheet + #elif request_uri.find("spreadsheets") > 0: + # service = "wise" + + auth = dict(Email=credentials[0], Passwd=credentials[1], service=service, source=headers['user-agent']) + resp, content = self.http.request("https://www.google.com/accounts/ClientLogin", method="POST", body=urlencode(auth), headers={'Content-Type': 'application/x-www-form-urlencoded'}) + lines = content.split('\n') + d = dict([tuple(line.split("=", 1)) for line in lines if line]) + if resp.status == 403: + self.Auth = "" else: - raise ValueError,\ - "unexpected type in multicall result" + self.Auth = d['Auth'] -class MultiCall: - """server -> a object used to boxcar method calls - - server should be a ServerProxy object. - - Methods can be added to the MultiCall using normal - method call syntax e.g.: - - multicall = MultiCall(server_proxy) - multicall.add(2,3) - multicall.get_address("Guido") - - To execute the multicall, call the MultiCall object e.g.: - - add_result, address = multicall() + def request(self, method, request_uri, headers, content): + """Modify the request headers to add the appropriate + Authorization header.""" + headers['authorization'] = 'GoogleLogin Auth=' + self.Auth + + +AUTH_SCHEME_CLASSES = { + "basic": BasicAuthentication, + "wsse": WsseAuthentication, + "digest": DigestAuthentication, + "hmacdigest": HmacDigestAuthentication, + "googlelogin": GoogleLoginAuthentication +} + +AUTH_SCHEME_ORDER = ["hmacdigest", "googlelogin", "digest", "wsse", "basic"] + +class FileCache(object): + """Uses a local directory as a store for cached files. + Not really safe to use if multiple threads or processes are going to + be running on the same cache. """ - - def __init__(self, server): - self.__server = server - self.__call_list = [] - - def __repr__(self): - return "" % id(self) - - __str__ = __repr__ - - def __getattr__(self, name): - return _MultiCallMethod(self.__call_list, name) - - def __call__(self): - marshalled_list = [] - for name, args in self.__call_list: - marshalled_list.append({'methodName' : name, 'params' : args}) + def __init__(self, cache, safe=safename): # use safe=lambda x: md5.new(x).hexdigest() for the old behavior + self.cache = cache + self.safe = safe + if not os.path.exists(cache): + os.makedirs(self.cache) + + def get(self, key): + retval = None + cacheFullPath = os.path.join(self.cache, self.safe(key)) + try: + f = file(cacheFullPath, "rb") + retval = f.read() + f.close() + except IOError: + pass + return retval + + def set(self, key, value): + cacheFullPath = os.path.join(self.cache, self.safe(key)) + f = file(cacheFullPath, "wb") + f.write(value) + f.close() + + def delete(self, key): + cacheFullPath = os.path.join(self.cache, self.safe(key)) + if os.path.exists(cacheFullPath): + os.remove(cacheFullPath) + +class Credentials(object): + def __init__(self): + self.credentials = [] + + def add(self, name, password, domain=""): + self.credentials.append((domain.lower(), name, password)) + + def clear(self): + self.credentials = [] + + def iter(self, domain): + for (cdomain, name, password) in self.credentials: + if cdomain == "" or domain == cdomain: + yield (name, password) + +class KeyCerts(Credentials): + """Identical to Credentials except that + name/password are mapped to key/cert.""" + pass + + +class ProxyInfo(object): + """Collect information required to use a proxy.""" + def __init__(self, proxy_type, proxy_host, proxy_port, proxy_rdns=None, proxy_user=None, proxy_pass=None): + """The parameter proxy_type must be set to one of socks.PROXY_TYPE_XXX + constants. For example: + +p = ProxyInfo(proxy_type=socks.PROXY_TYPE_HTTP, proxy_host='localhost', proxy_port=8000) + """ + self.proxy_type, self.proxy_host, self.proxy_port, self.proxy_rdns, self.proxy_user, self.proxy_pass = proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass + + def astuple(self): + return (self.proxy_type, self.proxy_host, self.proxy_port, self.proxy_rdns, + self.proxy_user, self.proxy_pass) + + def isgood(self): + # HACK: amorton socks module is included in this file now + #return socks and (self.proxy_host != None) and (self.proxy_port != None) + return (self.proxy_host != None) and (self.proxy_port != None) + +class HTTPConnectionWithTimeout(httplib.HTTPConnection): + """HTTPConnection subclass that supports timeouts""" + + def __init__(self, host, port=None, strict=None, timeout=None, proxy_info=None): + httplib.HTTPConnection.__init__(self, host, port, strict) + self.timeout = timeout + self.proxy_info = proxy_info + + def connect(self): + """Connect to the host and port specified in __init__.""" + # Mostly verbatim from httplib.py. + msg = "getaddrinfo returns an empty list" + for res in socket.getaddrinfo(self.host, self.port, 0, + socket.SOCK_STREAM): + af, socktype, proto, canonname, sa = res + try: + if self.proxy_info and self.proxy_info.isgood(): + # HACK: amorton socks module is included in the file + # self.sock = socks.socksocket(af, socktype, proto) + self.sock = socksocket(af, socktype, proto) + # HACK: amorton enabled TCP_NODLEAY on socket + self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + self.sock.setproxy(*self.proxy_info.astuple()) + else: + self.sock = socket.socket(af, socktype, proto) + # HACK: amorton enabled TCP_NODLEAY on socket + self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + # Different from httplib: support timeouts. + if has_timeout(self.timeout): + self.sock.settimeout(self.timeout) + # End of difference from httplib. + if self.debuglevel > 0: + print "connect: (%s, %s)" % (self.host, self.port) + + self.sock.connect(sa) + except socket.error, msg: + if self.debuglevel > 0: + print 'connect fail:', (self.host, self.port) + if self.sock: + self.sock.close() + self.sock = None + continue + break + if not self.sock: + raise socket.error, msg + +class HTTPSConnectionWithTimeout(httplib.HTTPSConnection): + "This class allows communication via SSL." + + def __init__(self, host, port=None, key_file=None, cert_file=None, + strict=None, timeout=None, proxy_info=None): + httplib.HTTPSConnection.__init__(self, host, port=port, key_file=key_file, + cert_file=cert_file, strict=strict) + self.timeout = timeout + self.proxy_info = proxy_info + + def connect(self): + "Connect to a host on a given (SSL) port." + + if self.proxy_info and self.proxy_info.isgood(): + # HACK: amorton sock module is included in this file now + #sock = socks.socksocket(socket.AF_INET, socket.SOCK_STREAM) + sock = socksocket(socket.AF_INET, socket.SOCK_STREAM) + # HACK: amorton enabled TCP_NODLEAY on socket + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + sock.setproxy(*self.proxy_info.astuple()) + else: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + # HACK: amorton enabled TCP_NODLEAY on socket + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - return MultiCallIterator(self.__server.system.multicall(marshalled_list)) + if has_timeout(self.timeout): + sock.settimeout(self.timeout) + sock.connect((self.host, self.port)) + self.sock =_ssl_wrap_socket(sock, self.key_file, self.cert_file) -# -------------------------------------------------------------------- -# convenience functions -## -# Create a parser object, and connect it to an unmarshalling instance. -# This function picks the fastest available XML parser. -# -# return A (parser, unmarshaller) tuple. -def getparser(use_datetime=1, convert_datetimes_to_utc=1): - """getparser() -> parser, unmarshaller - - Create an instance of the fastest available parser, and attach it - to an unmarshalling object. Return both objects. +class Http(object): + """An HTTP client that handles: +- all methods +- caching +- ETags +- compression, +- HTTPS +- Basic +- Digest +- WSSE + +and more. """ - if use_datetime and not datetime: - raise ValueError, "the datetime module is not available" - if FastParser and FastUnmarshaller: - if use_datetime: - mkdatetime = _datetime_type + def __init__(self, cache=None, timeout=None, proxy_info=None): + """The value of proxy_info is a ProxyInfo instance. + +If 'cache' is a string then it is used as a directory name +for a disk cache. Otherwise it must be an object that supports +the same interface as FileCache.""" + self.proxy_info = proxy_info + # Map domain name to an httplib connection + self.connections = {} + # The location of the cache, for now a directory + # where cached responses are held. + if cache and isinstance(cache, str): + self.cache = FileCache(cache) else: - mkdatetime = _datetime - target = FastUnmarshaller(True, False, _binary, mkdatetime, Fault) - parser = FastParser(target) - else: - target = Unmarshaller(use_datetime=use_datetime, convert_datetimes_to_utc=convert_datetimes_to_utc) - if FastParser: - parser = FastParser(target) - elif SgmlopParser: - parser = SgmlopParser(target) - elif ExpatParser: - parser = ExpatParser(target) - else: - parser = SlowParser(target) - return parser, target + self.cache = cache -## -# Convert a Python tuple or a Fault instance to an XML-RPC packet. -# -# @def dumps(params, **options) -# @param params A tuple or Fault instance. -# @keyparam methodname If given, create a methodCall request for -# this method name. -# @keyparam methodresponse If given, create a methodResponse packet. -# If used with a tuple, the tuple must be a singleton (that is, -# it must contain exactly one element). -# @keyparam encoding The packet encoding. -# @return A string containing marshalled data. - -def dumps(params, methodname=None, methodresponse=None, encoding=None, - allow_none=1, convert_datetimes_to_utc=1): - """data [,options] -> marshalled data - - Convert an argument tuple or a Fault instance to an XML-RPC - request (or response, if the methodresponse option is used). - - In addition to the data object, the following options can be given - as keyword arguments: - - methodname: the method name for a methodCall packet - - methodresponse: true to create a methodResponse packet. - If this option is used with a tuple, the tuple must be - a singleton (i.e. it can contain only one element). + # Name/password + self.credentials = Credentials() + + # Key/cert + self.certificates = KeyCerts() + + # authorization objects + self.authorizations = [] + + # If set to False then no redirects are followed, even safe ones. + self.follow_redirects = True - encoding: the packet encoding (default is UTF-8) - - All 8-bit strings in the data structure are assumed to use the - packet encoding. Unicode strings are automatically converted, - where necessary. - """ - - assert isinstance(params, TupleType) or isinstance(params, Fault),\ - "argument must be tuple or Fault instance" - - if isinstance(params, Fault): - methodresponse = 1 - elif methodresponse and isinstance(params, TupleType): - assert len(params) == 1, "response tuple must be a singleton" - - if not encoding: - encoding = "utf-8" - - if FastMarshaller: - m = FastMarshaller(encoding) - else: - m = Marshaller(encoding, allow_none, convert_datetimes_to_utc) - - data = m.dumps(params) - - if encoding != "utf-8": - xmlheader = "\n" % str(encoding) - else: - xmlheader = "\n" # utf-8 is default - - # standard XML-RPC wrappings - if methodname: - # a method call - if not isinstance(methodname, StringType): - methodname = methodname.encode(encoding) - data = ( - xmlheader, - "\n" - "", methodname, "\n", - data, - "\n" - ) - elif methodresponse: - # a method response, or a fault structure - data = ( - xmlheader, - "\n", - data, - "\n" - ) - else: - return data # return as is - return string.join(data, "") + # Which HTTP methods do we apply optimistic concurrency to, i.e. + # which methods get an "if-match:" etag header added to them. + self.optimistic_concurrency_methods = ["PUT"] -## -# Convert an XML-RPC packet to a Python object. If the XML-RPC packet -# represents a fault condition, this function raises a Fault exception. -# -# @param data An XML-RPC packet, given as an 8-bit string. -# @return A tuple containing the unpacked data, and the method name -# (None if not present). -# @see Fault + # If 'follow_redirects' is True, and this is set to True then + # all redirecs are followed, including unsafe ones. + self.follow_all_redirects = False -def loads(data, use_datetime=1, convert_datetimes_to_utc=1): - """data -> unmarshalled data, method name - - Convert an XML-RPC packet to unmarshalled data plus a method - name (None if not present). - - If the XML-RPC packet represents a fault condition, this function - raises a Fault exception. - """ - p, u = getparser(use_datetime=use_datetime, convert_datetimes_to_utc=convert_datetimes_to_utc) - p.feed(data) - p.close() - return u.close(), u.getmethodname() + self.ignore_etag = False + self.force_exception_to_status_code = False -# -------------------------------------------------------------------- -# request dispatcher + self.timeout = timeout -class _Method: - # some magic to bind an XML-RPC method to an RPC server. - # supports "nested" methods (e.g. examples.getStateName) - def __init__(self, send, name): - self.__send = send - self.__name = name - def __getattr__(self, name): - return _Method(self.__send, "%s.%s" % (self.__name, name)) - def __call__(self, *args): - return self.__send(self.__name, args) - -## -# Standard transport class for XML-RPC over HTTP. -#

-# You can create custom transports by subclassing this method, and -# overriding selected methods. - -class Transport: - """Handles an HTTP transaction to an XML-RPC server.""" - - # client identifier (may be overridden) - user_agent = "xmlrpclib.py/%s (by www.pythonware.com)" % __version__ - - def __init__(self, use_datetime=1, convert_datetimes_to_utc=1): - self._use_datetime = use_datetime - self._connection = (None, None) - self._extra_headers = [] - self._convert_datetimes_to_utc = convert_datetimes_to_utc - - ## - # Send a complete request, and parse the response. - # Retry request if a cached connection has disconnected. - # - # @param host Target host. - # @param handler Target PRC handler. - # @param request_body XML-RPC request body. - # @param verbose Debugging flag. - # @return Parsed response. - - def request(self, host, handler, request_body, verbose=0): - #retry request once if cached connection has gone cold - for i in range(10): + def _auth_from_challenge(self, host, request_uri, headers, response, content): + """A generator that creates Authorization objects + that can be applied to requests. + """ + challenges = _parse_www_authenticate(response, 'www-authenticate') + for cred in self.credentials.iter(host): + for scheme in AUTH_SCHEME_ORDER: + if challenges.has_key(scheme): + yield AUTH_SCHEME_CLASSES[scheme](cred, host, request_uri, headers, response, content, self) + + def add_credentials(self, name, password, domain=""): + """Add a name and password that will be used + any time a request requires authentication.""" + self.credentials.add(name, password, domain) + + def add_certificate(self, key, cert, domain): + """Add a key and cert that will be used + any time a request requires authentication.""" + self.certificates.add(key, cert, domain) + + def clear_credentials(self): + """Remove all the names and passwords + that are used for authentication""" + self.credentials.clear() + self.authorizations = [] + + def _conn_request(self, conn, request_uri, method, body, headers): + for i in range(2): try: - return self.single_request(host, handler, request_body, verbose) - except socket.error, e: - if i >= 10 or e.errno not in (errno.ECONNRESET, errno.ECONNABORTED, errno.EPIPE): - raise - except httplib.BadStatusLine: #close after we sent request - if i >= 10: + conn.request(method, request_uri, body, headers) + except socket.gaierror: + conn.close() + raise ServerNotFoundError("Unable to find the server at %s" % conn.host) + except (socket.error, httplib.HTTPException): + # Just because the server closed the connection doesn't apparently mean + # that the server didn't send a response. + # amorton: patch from http://code.google.com/p/httplib2/source/detail?r=0cff83696d + if conn.sock is None: + if i == 0: + conn.close() + conn.connect() + continue + else: + conn.close() + raise + if i == 0: + conn.close() + conn.connect() + continue + pass + try: + response = conn.getresponse() + except (socket.error, httplib.HTTPException): + if i == 0: + conn.close() + conn.connect() + continue + else: raise - - ## - # Send a complete request, and parse the response. - # - # @param host Target host. - # @param handler Target PRC handler. - # @param request_body XML-RPC request body. - # @param verbose Debugging flag. - # @return Parsed response. - - def single_request(self, host, handler, request_body, verbose=0): - # issue XML-RPC request - h = self.make_connection(host) - if verbose: - h.set_debuglevel(1) - - try: - self.send_request(h, handler, request_body) - self.send_host(h, host) - self.send_user_agent(h) - self.send_content(h, request_body) - - response = h.getresponse() - if response.status == 200: - self.verbose = verbose - return self.parse_response(response) - except Fault: - raise - except Exception: - # All unexpected errors leave connection in - # a strange state, so we clear it. - self.close() - raise - - #discard any response data and raise exception - if (response.getheader("content-length", 0)): - response.read() - raise ProtocolError( - host + handler, - response.status, response.reason, - response.msg, - ) - - ## - # Create parser. - # - # @return A 2-tuple containing a parser and a unmarshaller. - - def getparser(self): - # get parser and unmarshaller - return getparser(use_datetime=self._use_datetime, convert_datetimes_to_utc=self._convert_datetimes_to_utc) - - ## - # Get authorization info from host parameter - # Host may be a string, or a (host, x509-dict) tuple; if a string, - # it is checked for a "user:pw@host" format, and a "Basic - # Authentication" header is added if appropriate. - # - # @param host Host descriptor (URL or (URL, x509 info) tuple). - # @return A 3-tuple containing (actual host, extra headers, - # x509 info). The header and x509 fields may be None. - - def get_host_info(self, host): - - x509 = {} - if isinstance(host, TupleType): - host, x509 = host - - import urllib - auth, host = urllib.splituser(host) - - if auth: - import base64 - auth = base64.encodestring(urllib.unquote(auth)) - auth = string.join(string.split(auth), "") # get rid of whitespace - extra_headers = [ - ("Authorization", "Basic " + auth) - ] - else: - extra_headers = None - - return host, extra_headers, x509 - - ## - # Connect to server. - # - # @param host Target host. - # @return A connection handle. - - def make_connection(self, host): - #return an existing connection if possible. This allows - #HTTP/1.1 keep-alive. - if self._connection and host == self._connection[0]: - return self._connection[1] - # create a HTTP connection object from a host descriptor - chost, self._extra_headers, x509 = self.get_host_info(host) - #store the host argument along with the connection object - self._connection = host, httplib.HTTPConnection(chost) - return self._connection[1] - - ## - # Clear any cached connection object. - # Used in the event of socket errors. - # - def close(self): - if self._connection[1]: - self._connection[1].close() - self._connection = (None, None) - - ## - # Send request header. - # - # @param connection Connection handle. - # @param handler Target RPC handler. - # @param request_body XML-RPC body. - - def send_request(self, connection, handler, request_body): - connection.putrequest("POST", handler) - - ## - # Send host name. - # - # @param connection Connection handle. - # @param host Host name. - # - # Note: This function doesn't actually add the "Host" - # header anymore, it is done as part of the connection.putrequest() in - # send_request() above. - - def send_host(self, connection, host): - extra_headers = self._extra_headers - if extra_headers: - if isinstance(extra_headers, DictType): - extra_headers = extra_headers.items() - for key, value in extra_headers: - connection.putheader(key, value) - - ## - # Send user-agent identifier. - # - # @param connection Connection handle. - - def send_user_agent(self, connection): - connection.putheader("User-Agent", self.user_agent) - - ## - # Send request body. - # - # @param connection Connection handle. - # @param request_body XML-RPC request body. - - def send_content(self, connection, request_body): - connection.putheader("Content-Type", "text/xml") - connection.putheader("Content-Length", str(len(request_body))) - connection.endheaders() - if request_body: - connection.send(request_body) - - ## - # Parse response. - # - # @param file Stream. - # @return Response tuple and target method. - - def parse_response(self, file): - # compatibility interface - return self._parse_response(file, None) - - ## - # Parse response (alternate interface). This is similar to the - # parse_response method, but also provides direct access to the - # underlying socket object (where available). - # - # @param file Stream. - # @param sock Socket handle (or None, if the socket object - # could not be accessed). - # @return Response tuple and target method. - - def _parse_response(self, file, sock): - # read response from input file/socket, and parse it - - p, u = self.getparser() - - while 1: - if sock: - response = sock.recv(1024) else: - response = file.read(1024) - if not response: - break - if self.verbose: - print "body:", repr(response) - p.feed(response) - - file.close() - p.close() - - return u.close() + content = "" + if method == "HEAD": + response.close() + else: + content = response.read() + response = Response(response) + if method != "HEAD": + content = _decompressContent(response, content) + break + return (response, content) + + + def _request(self, conn, host, absolute_uri, request_uri, method, body, headers, redirections, cachekey): + """Do the actual request using the connection object + and also follow one level of redirects if necessary""" + + auths = [(auth.depth(request_uri), auth) for auth in self.authorizations if auth.inscope(host, request_uri)] + auth = auths and sorted(auths)[0][1] or None + if auth: + auth.request(method, request_uri, headers, body) + + (response, content) = self._conn_request(conn, request_uri, method, body, headers) + + if auth: + if auth.response(response, body): + auth.request(method, request_uri, headers, body) + (response, content) = self._conn_request(conn, request_uri, method, body, headers ) + response._stale_digest = 1 + + if response.status == 401: + for authorization in self._auth_from_challenge(host, request_uri, headers, response, content): + authorization.request(method, request_uri, headers, body) + (response, content) = self._conn_request(conn, request_uri, method, body, headers, ) + if response.status != 401: + self.authorizations.append(authorization) + authorization.response(response, body) + break + + if (self.follow_all_redirects or (method in ["GET", "HEAD"]) or response.status == 303): + if self.follow_redirects and response.status in [300, 301, 302, 303, 307]: + # Pick out the location header and basically start from the beginning + # remembering first to strip the ETag header and decrement our 'depth' + if redirections: + if not response.has_key('location') and response.status != 300: + raise RedirectMissingLocation( _("Redirected but the response is missing a Location: header."), response, content) + # Fix-up relative redirects (which violate an RFC 2616 MUST) + if response.has_key('location'): + location = response['location'] + (scheme, authority, path, query, fragment) = parse_uri(location) + if authority == None: + response['location'] = urlparse.urljoin(absolute_uri, location) + if response.status == 301 and method in ["GET", "HEAD"]: + response['-x-permanent-redirect-url'] = response['location'] + if not response.has_key('content-location'): + response['content-location'] = absolute_uri + _updateCache(headers, response, content, self.cache, cachekey) + if headers.has_key('if-none-match'): + del headers['if-none-match'] + if headers.has_key('if-modified-since'): + del headers['if-modified-since'] + if response.has_key('location'): + location = response['location'] + old_response = copy.deepcopy(response) + if not old_response.has_key('content-location'): + old_response['content-location'] = absolute_uri + redirect_method = ((response.status == 303) and (method not in ["GET", "HEAD"])) and "GET" or method + (response, content) = self.request(location, redirect_method, body=body, headers = headers, redirections = redirections - 1) + response.previous = old_response + else: + raise RedirectLimit( _("Redirected more times than rediection_limit allows."), response, content) + elif response.status in [200, 203] and method == "GET": + # Don't cache 206's since we aren't going to handle byte range requests + if not response.has_key('content-location'): + response['content-location'] = absolute_uri + _updateCache(headers, response, content, self.cache, cachekey) -## -# Standard transport class for XML-RPC over HTTPS. + return (response, content) -class SafeTransport(Transport): - """Handles an HTTPS transaction to an XML-RPC server.""" - - # FIXME: mostly untested - - def make_connection(self, host): - #return an existing connection if possible. This allows - #HTTP/1.1 keep-alive. - if self._connection and host == self._connection[0]: - return self._connection[1] - # create a HTTPS connection object from a host descriptor - # host may be a string, or a (host, x509-dict) tuple + def _normalize_headers(self, headers): + return _normalize_headers(headers) + +# Need to catch and rebrand some exceptions +# Then need to optionally turn all exceptions into status codes +# including all socket.* and httplib.* exceptions. + + + def request(self, uri, method="GET", body=None, headers=None, redirections=DEFAULT_MAX_REDIRECTS, connection_type=None): + """ Performs a single HTTP request. +The 'uri' is the URI of the HTTP resource and can begin +with either 'http' or 'https'. The value of 'uri' must be an absolute URI. + +The 'method' is the HTTP method to perform, such as GET, POST, DELETE, etc. +There is no restriction on the methods allowed. + +The 'body' is the entity body to be sent with the request. It is a string +object. + +Any extra headers that are to be sent with the request should be provided in the +'headers' dictionary. + +The maximum number of redirect to follow before raising an +exception is 'redirections. The default is 5. + +The return value is a tuple of (response, content), the first +being and instance of the 'Response' class, the second being +a string that contains the response entity body. + """ try: - HTTPS = httplib.HTTPSConnection - except AttributeError: - raise NotImplementedError( - "your version of httplib doesn't support HTTPS" - ) - else: - chost, self._extra_headers, x509 = self.get_host_info(host) - self._connection = host, HTTPS(chost, None, **(x509 or {})) - return self._connection[1] - -# From example here, modified for keepalive changes: http://docs.python.org/library/xmlrpxlib.html -class ProxiedTransport(Transport): - """Handles an HTTP transaction via proxy server to an XML-RPC server. - Can also handle authentication. However, cannot handle an SSL - connection via proxy server""" - - def set_proxy(self, proxy): - self.proxy = proxy - - def make_connection(self, host): - self.realhost = host - proxy_host = self.proxy - proxy_user_pass = None - if '@' in proxy_host: - user_pass, proxy_host = proxy_host.split('@', 1) - if ':' in user_pass: - user, password = user_pass.split(':', 1) - proxy_user_pass = base64.encodestring('%s:%s' % (urllib.unquote(user), - urllib.unquote(password))).strip() - if not proxy_user_pass: - self._extra_headers += [('User-agent', self.user_agent)] + if headers is None: + headers = {} else: - self._extra_headers += [('User-agent', self.user_agent), - ('Proxy-authorization', 'Basic ' + proxy_user_pass) ] - #return an existing connection if possible. This allows - #HTTP/1.1 keep-alive. - if self._connection and host == self._connection[0]: - return self._connection[1] - # create a HTTP connection object from a host descriptor - chost, extra_headers, x509 = self.get_host_info(proxy_host) - if extra_headers: - self._extra_headers += extra_headers - #store the host argument along with the connection object - self._connection = host, httplib.HTTPConnection(chost) - return self._connection[1] - - def send_request(self, connection, handler, request_body): - connection.putrequest("POST", 'http://%s%s' % (self.realhost, handler)) + headers = self._normalize_headers(headers) + if not headers.has_key('user-agent'): + headers['user-agent'] = "Python-httplib2/%s" % __version__ -## -# Standard server proxy. This class establishes a virtual connection -# to an XML-RPC server. -#

-# This class is available as ServerProxy and Server. New code should -# use ServerProxy, to avoid confusion. -# -# @def ServerProxy(uri, **options) -# @param uri The connection point on the server. -# @keyparam transport A transport factory, compatible with the -# standard transport class. -# @keyparam encoding The default encoding used for 8-bit strings -# (default is UTF-8). -# @keyparam verbose Use a true value to enable debugging output. -# (printed to standard output). -# @see Transport - -class ServerProxy: - """uri [,options] -> a logical connection to an XML-RPC server - - uri is the connection point on the server, given as - scheme://host/target. - - The standard implementation always supports the "http" scheme. If - SSL socket support is available (Python 2.0), it also supports - "https". - - If the target part and the slash preceding it are both omitted, - "/RPC2" is assumed. - - The following options can be given as keyword arguments: - - transport: a transport factory - encoding: the request encoding (default is UTF-8) - - All 8-bit strings passed to the server proxy are assumed to use - the given encoding. - """ - - def __init__(self, uri, transport=None, encoding=None, verbose=0, - allow_none=1, use_datetime=1, convert_datetimes_to_utc=1): - # establish a "logical" server connection - - # get the url - import urllib - type, uri = urllib.splittype(uri) - if type not in ("http", "https"): - raise IOError, "unsupported XML-RPC protocol" - self.__host, self.__handler = urllib.splithost(uri) - if not self.__handler: - self.__handler = "/RPC2" - - if transport is None: - if type == "https": - transport = SafeTransport(use_datetime=use_datetime, convert_datetimes_to_utc=convert_datetimes_to_utc) + uri = iri2uri(uri) + + (scheme, authority, request_uri, defrag_uri) = urlnorm(uri) + domain_port = authority.split(":")[0:2] + if len(domain_port) == 2 and domain_port[1] == '443' and scheme == 'http': + scheme = 'https' + authority = domain_port[0] + + conn_key = scheme+":"+authority + if conn_key in self.connections: + conn = self.connections[conn_key] else: - transport = Transport(use_datetime=use_datetime, convert_datetimes_to_utc=convert_datetimes_to_utc) - self.__transport = transport - - self.__encoding = encoding - self.__verbose = verbose - self.__allow_none = allow_none - self.__convert_datetimes_to_utc = convert_datetimes_to_utc - - def __close(self): - self.__transport.close() - - def __request(self, methodname, params): - # call a method on the remote server - - request = dumps(params, methodname, encoding=self.__encoding, - allow_none=self.__allow_none, convert_datetimes_to_utc=self.__convert_datetimes_to_utc) - - response = self.__transport.request( - self.__host, - self.__handler, - request, - verbose=self.__verbose - ) - - if len(response) == 1: - response = response[0] - - return response - - def __repr__(self): - return ( - "" % - (self.__host, self.__handler) - ) - - __str__ = __repr__ - + if not connection_type: + connection_type = (scheme == 'https') and HTTPSConnectionWithTimeout or HTTPConnectionWithTimeout + certs = list(self.certificates.iter(authority)) + if scheme == 'https' and certs: + conn = self.connections[conn_key] = connection_type(authority, key_file=certs[0][0], + cert_file=certs[0][1], timeout=self.timeout, proxy_info=self.proxy_info) + else: + conn = self.connections[conn_key] = connection_type(authority, timeout=self.timeout, proxy_info=self.proxy_info) + conn.set_debuglevel(debuglevel) + + if method in ["GET", "HEAD"] and 'range' not in headers and 'accept-encoding' not in headers: + headers['accept-encoding'] = 'gzip, deflate' + + info = email.Message.Message() + cached_value = None + if self.cache: + cachekey = defrag_uri + cached_value = self.cache.get(cachekey) + if cached_value: + # info = email.message_from_string(cached_value) + # + # Need to replace the line above with the kludge below + # to fix the non-existent bug not fixed in this + # bug report: http://mail.python.org/pipermail/python-bugs-list/2005-September/030289.html + try: + info, content = cached_value.split('\r\n\r\n', 1) + feedparser = email.FeedParser.FeedParser() + feedparser.feed(info) + info = feedparser.close() + feedparser._parse = None + except IndexError: + self.cache.delete(cachekey) + cachekey = None + cached_value = None + else: + cachekey = None + + if method in self.optimistic_concurrency_methods and self.cache and info.has_key('etag') and not self.ignore_etag and 'if-match' not in headers: + # http://www.w3.org/1999/04/Editing/ + headers['if-match'] = info['etag'] + + if method not in ["GET", "HEAD"] and self.cache and cachekey: + # RFC 2616 Section 13.10 + self.cache.delete(cachekey) + + # Check the vary header in the cache to see if this request + # matches what varies in the cache. + if method in ['GET', 'HEAD'] and 'vary' in info: + vary = info['vary'] + vary_headers = vary.lower().replace(' ', '').split(',') + for header in vary_headers: + key = '-varied-%s' % header + value = info[key] + if headers.get(header, '') != value: + cached_value = None + break + + if cached_value and method in ["GET", "HEAD"] and self.cache and 'range' not in headers: + if info.has_key('-x-permanent-redirect-url'): + # Should cached permanent redirects be counted in our redirection count? For now, yes. + (response, new_content) = self.request(info['-x-permanent-redirect-url'], "GET", headers = headers, redirections = redirections - 1) + response.previous = Response(info) + response.previous.fromcache = True + else: + # Determine our course of action: + # Is the cached entry fresh or stale? + # Has the client requested a non-cached response? + # + # There seems to be three possible answers: + # 1. [FRESH] Return the cache entry w/o doing a GET + # 2. [STALE] Do the GET (but add in cache validators if available) + # 3. [TRANSPARENT] Do a GET w/o any cache validators (Cache-Control: no-cache) on the request + entry_disposition = _entry_disposition(info, headers) + + if entry_disposition == "FRESH": + if not cached_value: + info['status'] = '504' + content = "" + response = Response(info) + if cached_value: + response.fromcache = True + return (response, content) + + if entry_disposition == "STALE": + if info.has_key('etag') and not self.ignore_etag and not 'if-none-match' in headers: + headers['if-none-match'] = info['etag'] + if info.has_key('last-modified') and not 'last-modified' in headers: + headers['if-modified-since'] = info['last-modified'] + elif entry_disposition == "TRANSPARENT": + pass + + (response, new_content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey) + + if response.status == 304 and method == "GET": + # Rewrite the cache entry with the new end-to-end headers + # Take all headers that are in response + # and overwrite their values in info. + # unless they are hop-by-hop, or are listed in the connection header. + + for key in _get_end2end_headers(response): + info[key] = response[key] + merged_response = Response(info) + if hasattr(response, "_stale_digest"): + merged_response._stale_digest = response._stale_digest + _updateCache(headers, merged_response, content, self.cache, cachekey) + response = merged_response + response.status = 200 + response.fromcache = True + + elif response.status == 200: + content = new_content + else: + self.cache.delete(cachekey) + content = new_content + else: + cc = _parse_cache_control(headers) + if cc.has_key('only-if-cached'): + info['status'] = '504' + response = Response(info) + content = "" + else: + (response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey) + except Exception, e: + if self.force_exception_to_status_code: + if isinstance(e, HttpLib2ErrorWithResponse): + response = e.response + content = e.content + response.status = 500 + response.reason = str(e) + elif isinstance(e, socket.timeout): + content = "Request Timeout" + response = Response( { + "content-type": "text/plain", + "status": "408", + "content-length": len(content) + }) + response.reason = "Request Timeout" + else: + content = str(e) + response = Response( { + "content-type": "text/plain", + "status": "400", + "content-length": len(content) + }) + response.reason = "Bad Request" + else: + raise + + + return (response, content) + + + +class Response(dict): + """An object more like email.Message than httplib.HTTPResponse.""" + + """Is this response from our local cache""" + fromcache = False + + """HTTP protocol version used by server. 10 for HTTP/1.0, 11 for HTTP/1.1. """ + version = 11 + + "Status code returned by server. " + status = 200 + + """Reason phrase returned by server.""" + reason = "Ok" + + previous = None + + def __init__(self, info): + # info is either an email.Message or + # an httplib.HTTPResponse object. + if isinstance(info, httplib.HTTPResponse): + for key, value in info.getheaders(): + self[key.lower()] = value + self.status = info.status + self['status'] = str(self.status) + self.reason = info.reason + self.version = info.version + elif isinstance(info, email.Message.Message): + for key, value in info.items(): + self[key] = value + self.status = int(self['status']) + else: + for key, value in info.iteritems(): + self[key] = value + self.status = int(self.get('status', self.status)) + + def __getattr__(self, name): - # magic method dispatcher - return _Method(self.__request, name) - - # note: to call a remote object with an non-standard name, use - # result getattr(server, "strange-python-name")(args) - - def __call__(self, attr): - """A workaround to get special attributes on the ServerProxy - without interfering with the magic __getattr__ - """ - if attr == "close": - return self.__close - elif attr == "transport": - return self.__transport - raise AttributeError("Attribute %r not found" % (attr,)) + if name == 'dict': + return self + else: + raise AttributeError, name -# compatibility -Server = ServerProxy -# -------------------------------------------------------------------- -# test code -# if __name__ == "__main__": -# -# # simple test program (from the XML-RPC specification) -# -# # server = ServerProxy("http://localhost:8000") # local server -# server = ServerProxy("http://time.xmlrpc.com/RPC2") -# -# print server -# -# try: -# print server.currentTime.getCurrentTime() -# except Error, v: -# print "ERROR", v -# -# multi = MultiCall(server) -# multi.currentTime.getCurrentTime() -# multi.currentTime.getCurrentTime() -# try: -# for response in multi(): -# print response -# except Error, v: -# print "ERROR", v - -if __name__ == "__main__": - from pprint import pprint - - sg = Shotgun('http://localhost:3000', 'wrapper_script', 'ca8e878c9c7f6d8ab3bf1d92fd1a624361cf4e6e') - - # for i in range(1001,5000): - # pprint(sg.create("Asset",{"code":"Asset %d"%i,"project_names":"Test Project"})) - # - # pprint(sg.find("Asset",filters=[], filter_operator='any', fields=['code','image'])) - # sg.upload("Asset", 1, "/path/to/file.png", display_name="My File", field_name="sg_attachment") + + + + + + + + + + + + + + +# ---------------------------------------------------------------------------- +# SocksiPy/socks.py + +"""SocksiPy - Python SOCKS module. +Version 1.00 + +Copyright 2006 Dan-Haim. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +3. Neither the name of Dan Haim nor the names of his contributors may be used + to endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY DAN HAIM "AS IS" AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL DAN HAIM OR HIS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMANGE. + + +This module provides a standard socket-like interface for Python +for tunneling connections through SOCKS proxies. + +""" + +import socket +import struct + +PROXY_TYPE_SOCKS4 = 1 +PROXY_TYPE_SOCKS5 = 2 +PROXY_TYPE_HTTP = 3 + +_defaultproxy = None +_orgsocket = socket.socket + +class ProxyError(Exception): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +class GeneralProxyError(ProxyError): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +class Socks5AuthError(ProxyError): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +class Socks5Error(ProxyError): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +class Socks4Error(ProxyError): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +class HTTPError(ProxyError): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +_generalerrors = ("success", + "invalid data", + "not connected", + "not available", + "bad proxy type", + "bad input") + +_socks5errors = ("succeeded", + "general SOCKS server failure", + "connection not allowed by ruleset", + "Network unreachable", + "Host unreachable", + "Connection refused", + "TTL expired", + "Command not supported", + "Address type not supported", + "Unknown error") + +_socks5autherrors = ("succeeded", + "authentication is required", + "all offered authentication methods were rejected", + "unknown username or invalid password", + "unknown error") + +_socks4errors = ("request granted", + "request rejected or failed", + "request rejected because SOCKS server cannot connect to identd on the client", + "request rejected because the client program and identd report different user-ids", + "unknown error") + +def setdefaultproxy(proxytype=None,addr=None,port=None,rdns=True,username=None,password=None): + """setdefaultproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) + Sets a default proxy which all further socksocket objects will use, + unless explicitly changed. + """ + global _defaultproxy + _defaultproxy = (proxytype,addr,port,rdns,username,password) + +class socksocket(socket.socket): + """socksocket([family[, type[, proto]]]) -> socket object + + Open a SOCKS enabled socket. The parameters are the same as + those of the standard socket init. In order for SOCKS to work, + you must specify family=AF_INET, type=SOCK_STREAM and proto=0. + """ + + def __init__(self, family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0, _sock=None): + _orgsocket.__init__(self,family,type,proto,_sock) + if _defaultproxy != None: + self.__proxy = _defaultproxy + else: + self.__proxy = (None, None, None, None, None, None) + self.__proxysockname = None + self.__proxypeername = None + + def __recvall(self, bytes): + """__recvall(bytes) -> data + Receive EXACTLY the number of bytes requested from the socket. + Blocks until the required number of bytes have been received. + """ + data = "" + while len(data) < bytes: + data = data + self.recv(bytes-len(data)) + return data + + def setproxy(self,proxytype=None,addr=None,port=None,rdns=True,username=None,password=None): + """setproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) + Sets the proxy to be used. + proxytype - The type of the proxy to be used. Three types + are supported: PROXY_TYPE_SOCKS4 (including socks4a), + PROXY_TYPE_SOCKS5 and PROXY_TYPE_HTTP + addr - The address of the server (IP or DNS). + port - The port of the server. Defaults to 1080 for SOCKS + servers and 8080 for HTTP proxy servers. + rdns - Should DNS queries be preformed on the remote side + (rather than the local side). The default is True. + Note: This has no effect with SOCKS4 servers. + username - Username to authenticate with to the server. + The default is no authentication. + password - Password to authenticate with to the server. + Only relevant when username is also provided. + """ + self.__proxy = (proxytype,addr,port,rdns,username,password) + + def __negotiatesocks5(self,destaddr,destport): + """__negotiatesocks5(self,destaddr,destport) + Negotiates a connection through a SOCKS5 server. + """ + # First we'll send the authentication packages we support. + if (self.__proxy[4]!=None) and (self.__proxy[5]!=None): + # The username/password details were supplied to the + # setproxy method so we support the USERNAME/PASSWORD + # authentication (in addition to the standard none). + self.sendall("\x05\x02\x00\x02") + else: + # No username/password were entered, therefore we + # only support connections with no authentication. + self.sendall("\x05\x01\x00") + # We'll receive the server's response to determine which + # method was selected + chosenauth = self.__recvall(2) + if chosenauth[0] != "\x05": + self.close() + raise GeneralProxyError((1,_generalerrors[1])) + # Check the chosen authentication method + if chosenauth[1] == "\x00": + # No authentication is required + pass + elif chosenauth[1] == "\x02": + # Okay, we need to perform a basic username/password + # authentication. + self.sendall("\x01" + chr(len(self.__proxy[4])) + self.__proxy[4] + chr(len(self.proxy[5])) + self.__proxy[5]) + authstat = self.__recvall(2) + if authstat[0] != "\x01": + # Bad response + self.close() + raise GeneralProxyError((1,_generalerrors[1])) + if authstat[1] != "\x00": + # Authentication failed + self.close() + raise Socks5AuthError,((3,_socks5autherrors[3])) + # Authentication succeeded + else: + # Reaching here is always bad + self.close() + if chosenauth[1] == "\xFF": + raise Socks5AuthError((2,_socks5autherrors[2])) + else: + raise GeneralProxyError((1,_generalerrors[1])) + # Now we can request the actual connection + req = "\x05\x01\x00" + # If the given destination address is an IP address, we'll + # use the IPv4 address request even if remote resolving was specified. + try: + ipaddr = socket.inet_aton(destaddr) + req = req + "\x01" + ipaddr + except socket.error: + # Well it's not an IP number, so it's probably a DNS name. + if self.__proxy[3]==True: + # Resolve remotely + ipaddr = None + req = req + "\x03" + chr(len(destaddr)) + destaddr + else: + # Resolve locally + ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) + req = req + "\x01" + ipaddr + req = req + struct.pack(">H",destport) + self.sendall(req) + # Get the response + resp = self.__recvall(4) + if resp[0] != "\x05": + self.close() + raise GeneralProxyError((1,_generalerrors[1])) + elif resp[1] != "\x00": + # Connection failed + self.close() + if ord(resp[1])<=8: + raise Socks5Error(ord(resp[1]),_generalerrors[ord(resp[1])]) + else: + raise Socks5Error(9,_generalerrors[9]) + # Get the bound address/port + elif resp[3] == "\x01": + boundaddr = self.__recvall(4) + elif resp[3] == "\x03": + resp = resp + self.recv(1) + boundaddr = self.__recvall(resp[4]) + else: + self.close() + raise GeneralProxyError((1,_generalerrors[1])) + boundport = struct.unpack(">H",self.__recvall(2))[0] + self.__proxysockname = (boundaddr,boundport) + if ipaddr != None: + self.__proxypeername = (socket.inet_ntoa(ipaddr),destport) + else: + self.__proxypeername = (destaddr,destport) + + def getproxysockname(self): + """getsockname() -> address info + Returns the bound IP address and port number at the proxy. + """ + return self.__proxysockname + + def getproxypeername(self): + """getproxypeername() -> address info + Returns the IP and port number of the proxy. + """ + return _orgsocket.getpeername(self) + + def getpeername(self): + """getpeername() -> address info + Returns the IP address and port number of the destination + machine (note: getproxypeername returns the proxy) + """ + return self.__proxypeername + + def __negotiatesocks4(self,destaddr,destport): + """__negotiatesocks4(self,destaddr,destport) + Negotiates a connection through a SOCKS4 server. + """ + # Check if the destination address provided is an IP address + rmtrslv = False + try: + ipaddr = socket.inet_aton(destaddr) + except socket.error: + # It's a DNS name. Check where it should be resolved. + if self.__proxy[3]==True: + ipaddr = "\x00\x00\x00\x01" + rmtrslv = True + else: + ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) + # Construct the request packet + req = "\x04\x01" + struct.pack(">H",destport) + ipaddr + # The username parameter is considered userid for SOCKS4 + if self.__proxy[4] != None: + req = req + self.__proxy[4] + req = req + "\x00" + # DNS name if remote resolving is required + # NOTE: This is actually an extension to the SOCKS4 protocol + # called SOCKS4A and may not be supported in all cases. + if rmtrslv==True: + req = req + destaddr + "\x00" + self.sendall(req) + # Get the response from the server + resp = self.__recvall(8) + if resp[0] != "\x00": + # Bad data + self.close() + raise GeneralProxyError((1,_generalerrors[1])) + if resp[1] != "\x5A": + # Server returned an error + self.close() + if ord(resp[1]) in (91,92,93): + self.close() + raise Socks4Error((ord(resp[1]),_socks4errors[ord(resp[1])-90])) + else: + raise Socks4Error((94,_socks4errors[4])) + # Get the bound address/port + self.__proxysockname = (socket.inet_ntoa(resp[4:]),struct.unpack(">H",resp[2:4])[0]) + if rmtrslv != None: + self.__proxypeername = (socket.inet_ntoa(ipaddr),destport) + else: + self.__proxypeername = (destaddr,destport) + + def __negotiatehttp(self,destaddr,destport): + """__negotiatehttp(self,destaddr,destport) + Negotiates a connection through an HTTP server. + """ + # If we need to resolve locally, we do this now + if self.__proxy[3] == False: + addr = socket.gethostbyname(destaddr) + else: + addr = destaddr + self.sendall("CONNECT " + addr + ":" + str(destport) + " HTTP/1.1\r\n" + "Host: " + destaddr + "\r\n\r\n") + # We read the response until we get the string "\r\n\r\n" + resp = self.recv(1) + while resp.find("\r\n\r\n")==-1: + resp = resp + self.recv(1) + # We just need the first line to check if the connection + # was successful + statusline = resp.splitlines()[0].split(" ",2) + if statusline[0] not in ("HTTP/1.0","HTTP/1.1"): + self.close() + raise GeneralProxyError((1,_generalerrors[1])) + try: + statuscode = int(statusline[1]) + except ValueError: + self.close() + raise GeneralProxyError((1,_generalerrors[1])) + if statuscode != 200: + self.close() + raise HTTPError((statuscode,statusline[2])) + self.__proxysockname = ("0.0.0.0",0) + self.__proxypeername = (addr,destport) + + def connect(self,destpair): + """connect(self,despair) + Connects to the specified destination through a proxy. + destpar - A tuple of the IP/DNS address and the port number. + (identical to socket's connect). + To select the proxy server use setproxy(). + """ + # Do a minimal input check first + if (type(destpair) in (list,tuple)==False) or (len(destpair)<2) or (type(destpair[0])!=str) or (type(destpair[1])!=int): + raise GeneralProxyError((5,_generalerrors[5])) + if self.__proxy[0] == PROXY_TYPE_SOCKS5: + if self.__proxy[2] != None: + portnum = self.__proxy[2] + else: + portnum = 1080 + _orgsocket.connect(self,(self.__proxy[1],portnum)) + self.__negotiatesocks5(destpair[0],destpair[1]) + elif self.__proxy[0] == PROXY_TYPE_SOCKS4: + if self.__proxy[2] != None: + portnum = self.__proxy[2] + else: + portnum = 1080 + _orgsocket.connect(self,(self.__proxy[1],portnum)) + self.__negotiatesocks4(destpair[0],destpair[1]) + elif self.__proxy[0] == PROXY_TYPE_HTTP: + if self.__proxy[2] != None: + portnum = self.__proxy[2] + else: + portnum = 8080 + _orgsocket.connect(self,(self.__proxy[1],portnum)) + self.__negotiatehttp(destpair[0],destpair[1]) + elif self.__proxy[0] == None: + _orgsocket.connect(self,(destpair[0],destpair[1])) + else: + raise GeneralProxyError((4,_generalerrors[4])) From 478e48f9943a3dca994f035179a817b763f8f7be Mon Sep 17 00:00:00 2001 From: Matt Daw Date: Tue, 7 Jun 2011 10:31:28 -0700 Subject: [PATCH 002/570] Updated README --- README.mdown | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.mdown b/README.mdown index 742ae2770..f85bc2ec9 100644 --- a/README.mdown +++ b/README.mdown @@ -1,3 +1,17 @@ +# Shotgun JSON API - Alpha + +This is an alpha release of a new version of the Python API that uses JSON as a transport instead of XML-RPC. The intent is for this to be drop-in compatible with the previous version. If you find any incompatibilities, please contact us through support or on the dev list. + +## New requirements + +Requires 'simplejson' for Python versions older than 2.6. + +## Known Issues + +Does not yet support the new summarize method from the summary_grouping branch. + + + # Shotgun Python API Shotgun provides a simple Python-based API for accessing Shotgun and integrating with other tools. This is the official API that is maintained by Shotgun Software (support@shotgunsoftware.com) From 03c87790900db63ee148aed0a71dd4072a99bb9d Mon Sep 17 00:00:00 2001 From: Matt Daw Date: Tue, 7 Jun 2011 10:34:02 -0700 Subject: [PATCH 003/570] Updated README --- README.mdown | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.mdown b/README.mdown index f85bc2ec9..e69d384ca 100644 --- a/README.mdown +++ b/README.mdown @@ -4,7 +4,10 @@ This is an alpha release of a new version of the Python API that uses JSON as a ## New requirements -Requires 'simplejson' for Python versions older than 2.6. +Shotgun 2.4 or greater. + +'simplejson' for Python versions older than 2.6. + ## Known Issues From 3bad66e331e923ce7a2c6526a826d340879fd240 Mon Sep 17 00:00:00 2001 From: Matt Daw Date: Tue, 7 Jun 2011 11:06:58 -0700 Subject: [PATCH 004/570] Don't parse dates into native objects for compatibility with XML. --- shotgun_api3.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/shotgun_api3.py b/shotgun_api3.py index e50774f4a..ab5765e7f 100755 --- a/shotgun_api3.py +++ b/shotgun_api3.py @@ -1127,15 +1127,7 @@ def _inbound_visitor(value): if _change_tz: return _change_tz(value) return value - - #check for dates, no utz transform - if len(value) >= 10 and self._DATE_PATTERN.match(value): - try: - # strptime was not on datetime in python2.4 - return datetime.datetime( - *time.strptime(value, "%Y-%m-%d")[:6]).date() - except ValueError: - return value + return value return self._visit_data(data, _inbound_visitor) From 75962e958bcff77554c68f307d1783901f5a64eb Mon Sep 17 00:00:00 2001 From: Matt Daw Date: Tue, 7 Jun 2011 11:22:25 -0700 Subject: [PATCH 005/570] Be more selective about datetime parsing --- shotgun_api3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shotgun_api3.py b/shotgun_api3.py index ab5765e7f..bdbea1433 100755 --- a/shotgun_api3.py +++ b/shotgun_api3.py @@ -1117,7 +1117,7 @@ def _transform_inbound(self, data): def _inbound_visitor(value): if isinstance(value, basestring): - if len(value) >= 19 and self._DATE_TIME_PATTERN.match(value): + if len(value) == 20 and self._DATE_TIME_PATTERN.match(value): try: # strptime was not on datetime in python2.4 value = datetime.datetime( From 5b22a82e16cd6c937f865d4d0b81a8e93f4977cd Mon Sep 17 00:00:00 2001 From: Matt Daw Date: Tue, 7 Jun 2011 11:28:45 -0700 Subject: [PATCH 006/570] Update README with simplejson info --- README.mdown | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.mdown b/README.mdown index e69d384ca..14b24b2ff 100644 --- a/README.mdown +++ b/README.mdown @@ -6,7 +6,9 @@ This is an alpha release of a new version of the Python API that uses JSON as a Shotgun 2.4 or greater. -'simplejson' for Python versions older than 2.6. +For Python 2.4 and 2.5, install simplejson 2.0.9: http://pypi.python.org/pypi/simplejson/2.0.9 + +For Python 2.6 and higher, we will use the built-in json module... but installing the latest simplejson will give the best performance. Currently, that's version 2.1.6: http://pypi.python.org/pypi/simplejson/2.1.6 ## Known Issues From b30eee032a7ac765fd9c599607d8b2cb955a45fb Mon Sep 17 00:00:00 2001 From: KP Date: Wed, 8 Jun 2011 15:50:51 -0700 Subject: [PATCH 007/570] Edited README.mdown via GitHub --- README.mdown | 1 + 1 file changed, 1 insertion(+) diff --git a/README.mdown b/README.mdown index 14b24b2ff..bdd498077 100644 --- a/README.mdown +++ b/README.mdown @@ -26,6 +26,7 @@ Shotgun provides a simple Python-based API for accessing Shotgun and integrating [GitHub is the new official location](http://github.com/shotgunsoftware/python-api) for the latest Shotgun Python API. We will no longer be including it in our code tree that is deployed to your servers or posting it for download from our support site. So update your bookmarks! ## Minimum Requirements +**SEE NEW REQUIREMENTS ABOVE** - Shotgun server v1.8.0 or higher. Note that some API features are only available in later Shotgun server versions. - Python v2.4 - v2.7. (We do have plans to eventually support Python 3) From f9dc6c2f08dff5a935940881989aa2461940b829 Mon Sep 17 00:00:00 2001 From: KP Date: Wed, 8 Jun 2011 15:52:56 -0700 Subject: [PATCH 008/570] Edited README.mdown via GitHub --- README.mdown | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.mdown b/README.mdown index bdd498077..7d6ee42ac 100644 --- a/README.mdown +++ b/README.mdown @@ -1,4 +1,4 @@ -# Shotgun JSON API - Alpha +# Shotgun Python JSON API - Alpha This is an alpha release of a new version of the Python API that uses JSON as a transport instead of XML-RPC. The intent is for this to be drop-in compatible with the previous version. If you find any incompatibilities, please contact us through support or on the dev list. @@ -26,7 +26,7 @@ Shotgun provides a simple Python-based API for accessing Shotgun and integrating [GitHub is the new official location](http://github.com/shotgunsoftware/python-api) for the latest Shotgun Python API. We will no longer be including it in our code tree that is deployed to your servers or posting it for download from our support site. So update your bookmarks! ## Minimum Requirements -**SEE NEW REQUIREMENTS ABOVE** +**SEE NEW REQUIREMENTS ABOVE** - Shotgun server v1.8.0 or higher. Note that some API features are only available in later Shotgun server versions. - Python v2.4 - v2.7. (We do have plans to eventually support Python 3) @@ -43,6 +43,9 @@ Some useful direct links within this section are below: * [Reference: Filter Syntax](https://github.com/shotgunsoftware/python-api/wiki/Reference%3A-Filter-Syntax) ## Changelog +(Note that many of these items may be specific to the XML-RPC version of the API and may not apply to the Python JSON API) + + **v3.0.6 - 2010 Jan 25** + optimization: don't request paging_info unless required (and server support is available) From 50d1fb8d7d4512f9a8adda1afd2c01e2ca262d38 Mon Sep 17 00:00:00 2001 From: Matt Daw Date: Thu, 9 Jun 2011 10:07:50 -0700 Subject: [PATCH 009/570] Add version string (3.1a1) --- shotgun_api3.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shotgun_api3.py b/shotgun_api3.py index bdbea1433..edcfc1bcc 100755 --- a/shotgun_api3.py +++ b/shotgun_api3.py @@ -27,6 +27,8 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +__version__ = "3.1a1" + # needed for httplib2, future imports must be first from __future__ import generators From 25a0b3d9a4716a25a3247c4032a21f2d31fce6fa Mon Sep 17 00:00:00 2001 From: KP Date: Mon, 20 Jun 2011 16:21:02 -0700 Subject: [PATCH 010/570] move version string definition below __future__ imports so Python 2.5+ doesn't complain --- shotgun_api3.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/shotgun_api3.py b/shotgun_api3.py index edcfc1bcc..4943ca956 100755 --- a/shotgun_api3.py +++ b/shotgun_api3.py @@ -27,7 +27,6 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -__version__ = "3.1a1" # needed for httplib2, future imports must be first from __future__ import generators @@ -57,6 +56,10 @@ log.debug("simplejson not found, dropping back to json") import json as json +# ---------------------------------------------------------------------------- +# Version +__version__ = "3.1a1" + # ---------------------------------------------------------------------------- # Errors From 22e9499ed77dc6b37bb9fc3e51922d1e46bfd3b4 Mon Sep 17 00:00:00 2001 From: kennedy behrman Date: Fri, 8 Jul 2011 08:16:46 -0700 Subject: [PATCH 011/570] added modified version of amorton's tests --- run-tests | 1 + run-tests-2.4 | 1 + run-tests-2.5 | 1 + tests/__init__.py | 0 tests/base.py | 227 ++++++++++ tests/dummy_data.py | 876 +++++++++++++++++++++++++++++++++++++ tests/mock.py | 958 +++++++++++++++++++++++++++++++++++++++++ tests/sg_logo.jpg | Bin 0 -> 6414 bytes tests/test_api.py | 278 ++++++++++++ tests/test_api_long.py | 105 +++++ tests/test_client.py | 491 +++++++++++++++++++++ 11 files changed, 2938 insertions(+) create mode 100755 run-tests create mode 100755 run-tests-2.4 create mode 100755 run-tests-2.5 create mode 100644 tests/__init__.py create mode 100644 tests/base.py create mode 100644 tests/dummy_data.py create mode 100644 tests/mock.py create mode 100644 tests/sg_logo.jpg create mode 100644 tests/test_api.py create mode 100644 tests/test_api_long.py create mode 100644 tests/test_client.py diff --git a/run-tests b/run-tests new file mode 100755 index 000000000..6ce09db3d --- /dev/null +++ b/run-tests @@ -0,0 +1 @@ +clear && find ./ -name ".coverage" -delete && find ./ -name "*.pyc" -delete && nosetests -vd --stop --with-cover --cover-package=shotgun_json \ No newline at end of file diff --git a/run-tests-2.4 b/run-tests-2.4 new file mode 100755 index 000000000..b670d87ab --- /dev/null +++ b/run-tests-2.4 @@ -0,0 +1 @@ +clear && find ./ -name ".coverage" -delete && find ./ -name "*.pyc" -delete && nosetests-2.4 -vd --stop --with-cover --cover-package=shotgun_json \ No newline at end of file diff --git a/run-tests-2.5 b/run-tests-2.5 new file mode 100755 index 000000000..5ed4104e9 --- /dev/null +++ b/run-tests-2.5 @@ -0,0 +1 @@ +clear && find ./ -name ".coverage" -delete && find ./ -name "*.pyc" -delete && nosetests-2.5 -vd --stop --with-cover --cover-package=shotgun_json \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/base.py b/tests/base.py new file mode 100644 index 000000000..da3b652c8 --- /dev/null +++ b/tests/base.py @@ -0,0 +1,227 @@ +"""Base class for Shotgun API tests.""" +import unittest +from ConfigParser import ConfigParser + +try: + import simplejson as json +except ImportError: + import json as json + +import mock +import shotgun_api3 as api + +CONFIG_PATH = 'tests/config' + +class TestBase(unittest.TestCase): + '''Base class for tests. + + Sets up mocking and database test data.''' + def __init__(self, *args, **kws): + unittest.TestCase.__init__(self, *args, **kws) + self.is_mock = False + self.human_user = None + self.project = None + self.shot = None + self.asset = None + self.version = None + self.human_password = None + self.server_url = None + + + def setUp(self): + config = SgTestConfig() + config.read_config(CONFIG_PATH) + self.human_password = config.human_password + self.server_url = config.server_url + self.script_name = config.script_name + self.api_key = config.api_key + self.http_proxy = config.http_proxy + self.session_uuid = config.session_uuid + + self.sg = api.Shotgun(config.server_url, config.script_name, + config.api_key, http_proxy=config.http_proxy) + + if config.session_uuid: + self.sg.set_session_uuid(config.session_uuid) + + if config.mock: + self._setup_mock() + self._setup_mock_data(config) + else: + self._setup_db(config) + + + def tearDown(self): + self.sg = None + return + + def _setup_mock(self): + """Setup mocking on the ShotgunClient to stop it calling a live server + """ + #Replace the function used to make the final call to the server + #eaiser than mocking the http connection + response + self.sg._http_request = mock.Mock(spec=api.Shotgun._http_request, + return_value=((200, "OK"), {}, None)) + + #also replace the function that is called to get the http connection + #to avoid calling the server. OK to return a mock as we will not use + #it + self.mock_conn = mock.Mock(spec=api.Http) + #The Http objects connection property is a dict of connections + #it is holding + self.mock_conn.connections = dict() + self.sg._connection = self.mock_conn + self.sg._get_connection = mock.Mock(return_value=self.mock_conn) + + #create the server caps directly to say we have the correct version + self.sg._server_caps = api.ServerCapabilities(self.sg.config.server, + {"version" : [2,4,0]}) + self.is_mock = True + return + + def _mock_http(self, data, headers=None, status=None): + """Setup a mock response from the SG server. + + Only has an affect if the server has been mocked. + """ + #test for a mock object rather than config.mock as some tests + #force the mock to be created + if not isinstance(self.sg._http_request, mock.Mock): + return + + if not isinstance(data, basestring): + data = json.dumps(data, ensure_ascii=False, encoding="utf-8") + + resp_headers = { + 'cache-control': 'no-cache', + 'connection': 'close', + 'content-length': (data and str(len(data))) or 0 , + 'content-type': 'application/json; charset=utf-8', + 'date': 'Wed, 13 Apr 2011 04:18:58 GMT', + 'server': 'Apache/2.2.3 (CentOS)', + 'status': '200 OK' + } + if headers: + resp_headers.update(headers) + + if not status: + status = (200, "OK") + #create a new mock to reset call list etc. + self._setup_mock() + self.sg._http_request.return_value = (status, resp_headers, data) + + self.is_mock = True + return + + def _assert_http_method(self, method, params, check_auth=True): + """Asserts _http_request is called with the method and params.""" + + args, _ = self.sg._http_request.call_args + arg_body = args[2] + assert isinstance(arg_body, basestring) + arg_body = json.loads(arg_body) + + arg_params = arg_body.get("params") + + self.assertEqual(method, arg_body["method_name"]) + if check_auth: + auth = arg_params[0] + self.assertEqual(self.script_name, auth["script_name"]) + self.assertEqual(self.api_key, auth["script_key"]) + + if params: + rpc_args = arg_params[len(arg_params)-1] + self.assertEqual(params, rpc_args) + + return + + def _setup_mock_data(self, config): + self.human_user = { 'id':1, + 'login':config.human_login, + 'type':'HumanUser' } + self.project = { 'id':2, + 'name':config.project_name, + 'type':'Project' } + self.shot = { 'id':3, + 'code':config.shot_code, + 'type':'Shot' } + self.asset = { 'id':4, + 'code':config.asset_code, + 'type':'Asset' } + self.version = { 'id':5, + 'code':config.version_code, + 'type':'Version' } + + def _setup_db(self, config): + data = {'name':config.project_name} + self.project = _find_or_create_entity(self.sg, 'Project', data) + + data = {'name':config.human_name, + 'login':config.human_login, + 'password_proxy':config.human_password} + self.human_user = _find_or_create_entity(self.sg, 'HumanUser', data) + + data = {'code':config.asset_code, + 'project':self.project} + keys = ['code'] + self.asset = _find_or_create_entity(self.sg, 'Asset', data, keys) + + data = {'project':self.project, + 'code':config.version_code, + 'entity':self.asset, + 'user':self.human_user} + keys = ['code','project'] + self.version = _find_or_create_entity(self.sg, 'Version', data, keys) + + keys = ['code','project'] + data = {'code':config.shot_code, + 'project':self.project} + self.shot = _find_or_create_entity(self.sg, 'Shot', data, keys) + + + +def _find_or_create_entity(sg, entity_type, data, identifyiers=None): + '''Finds or creates entities. + @params: + sg - shogun_json.Shotgun instance + entity_type - entity type + data - dictionary of data for the entity + identifyiers -list of subset of keys from data which should be used to + uniquely identity the entity + @returns dicitonary of the entity values + ''' + identifyiers = identifyiers or ['name'] + fields = data.keys() + filters = [[key, 'is', data[key]] for key in identifyiers] + entity = sg.find_one(entity_type, filters, fields=fields) + entity = entity or sg.create(entity_type, data, return_fields=fields) + assert(entity) + return entity + +class SgTestConfig(object): + '''Reads test config and holds values''' + def __init__(self): + self.mock = True + self.server_url = None + self.script_name = None + self.api_key = None + self.http_proxy = None + self.session_uuid = None + self.project_name = None + self.human_name = None + self.human_login = None + self.human_password = None + self.asset_code = None + self.version_code = None + self.shot_code = None + + def read_config(self, config_path): + config_parser = ConfigParser() + config_parser.read(config_path) + for section in config_parser.sections(): + for option in config_parser.options(section): + value = config_parser.get(section, option) + setattr(self, option, value) + # cast non-sting attributes + self.mock = 'True' == str(self.mock) + diff --git a/tests/dummy_data.py b/tests/dummy_data.py new file mode 100644 index 000000000..5cd5d1634 --- /dev/null +++ b/tests/dummy_data.py @@ -0,0 +1,876 @@ +"""Dummy data returned for schema functions when mocking the server. + +NOTE: Mostly abbreviated version of real data returned from the server. +""" + +schema_entity_read = {u'Version': {u'name': {u'editable': False, u'value': u'Version'}}} + +schema_read = { + u'Version' : {u'code': {u'data_type': {u'editable': False, u'value': u'text'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': True}, + u'name': {u'editable': True, u'value': u'Version Name'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'created_at': {u'data_type': {u'editable': False, u'value': u'date_time'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': False}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Date Created'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'created_by': {u'data_type': {u'editable': False, u'value': u'entity'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': False}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Created by'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}, + u'valid_types': {u'editable': True, + u'value': [u'HumanUser', + u'ApiUser']}}}, + u'description': {u'data_type': {u'editable': False, u'value': u'text'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Description'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'entity': {u'data_type': {u'editable': False, u'value': u'entity'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Link'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}, + u'valid_types': {u'editable': True, + u'value': [u'Asset', + u'Scene', + u'Sequence', + u'Shot']}}}, + u'frame_count': {u'data_type': {u'editable': False, u'value': u'number'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Frame Count'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'frame_range': {u'data_type': {u'editable': False, u'value': u'text'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Frame Range'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'id': {u'data_type': {u'editable': False, u'value': u'number'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': False}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Id'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'image': {u'data_type': {u'editable': False, u'value': u'image'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Thumbnail'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'notes': {u'data_type': {u'editable': False, u'value': u'multi_entity'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Notes'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}, + u'valid_types': {u'editable': True, + u'value': [u'Note']}}}, + u'open_notes': {u'data_type': {u'editable': False, + u'value': u'multi_entity'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': False}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Open Notes'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}, + u'valid_types': {u'editable': True, + u'value': [u'Note']}}}, + u'open_notes_count': {u'data_type': {u'editable': False, u'value': u'text'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': False}, + u'entity_type': {u'editable': False, + u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, + u'value': u'Open Notes Count'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'playlists': {u'data_type': {u'editable': False, u'value': u'multi_entity'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Playlists'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}, + u'valid_types': {u'editable': True, + u'value': [u'Playlist']}}}, + u'project': {u'data_type': {u'editable': False, u'value': u'entity'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Project'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}, + u'valid_types': {u'editable': True, + u'value': [u'Project']}}}, + u'sg_department': {u'data_type': {u'editable': False, u'value': u'text'}, + u'description': {u'editable': True, + u'value': u'The department the Version was submitted from. This is used to find the latest Version from the same department.'}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, + u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Department'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'sg_first_frame': {u'data_type': {u'editable': False, u'value': u'number'}, + u'description': {u'editable': True, + u'value': u'The first frame number contained in the Version. Used in playback of the movie or frames to calculate the first frame available in the Version.'}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, + u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'First Frame'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'sg_frames_aspect_ratio': {u'data_type': {u'editable': False, + u'value': u'float'}, + u'description': {u'editable': True, + u'value': u'Aspect ratio of the high res frames. Used to format the image correctly for viewing.'}, + u'editable': {u'editable': False, + u'value': True}, + u'entity_type': {u'editable': False, + u'value': u'Version'}, + u'mandatory': {u'editable': False, + u'value': False}, + u'name': {u'editable': True, + u'value': u'Frames Aspect Ratio'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'sg_frames_have_slate': {u'data_type': {u'editable': False, + u'value': u'checkbox'}, + u'description': {u'editable': True, + u'value': u'Indicates whether the frames have a slate or not. This is used to include or omit the slate from playback.'}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, + u'value': u'Version'}, + u'mandatory': {u'editable': False, + u'value': False}, + u'name': {u'editable': True, + u'value': u'Frames Have Slate'}, + u'properties': {u'default_value': {u'editable': False, + u'value': False}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'sg_last_frame': {u'data_type': {u'editable': False, u'value': u'number'}, + u'description': {u'editable': True, + u'value': u'The last frame number contained in the Version. Used in playback of the movie or frames to calculate the last frame available in the Version.'}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, + u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Last Frame'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'sg_movie_aspect_ratio': {u'data_type': {u'editable': False, + u'value': u'float'}, + u'description': {u'editable': True, + u'value': u'Aspect ratio of the the movie. Used to format the image correctly for viewing.'}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, + u'value': u'Version'}, + u'mandatory': {u'editable': False, + u'value': False}, + u'name': {u'editable': True, + u'value': u'Movie Aspect Ratio'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'sg_movie_has_slate': {u'data_type': {u'editable': False, + u'value': u'checkbox'}, + u'description': {u'editable': True, + u'value': u'Indicates whether the movie file has a slate or not. This is used to include or omit the slate from playback.'}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, + u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, + u'value': u'Movie Has Slate'}, + u'properties': {u'default_value': {u'editable': False, + u'value': False}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'sg_path_to_frames': {u'data_type': {u'editable': False, u'value': u'text'}, + u'description': {u'editable': True, + u'value': u'Location of the high res frames on your local filesystem. Used for playback of high resolution frames.'}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, + u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, + u'value': u'Path to Frames'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'sg_path_to_movie': {u'data_type': {u'editable': False, u'value': u'text'}, + u'description': {u'editable': True, + u'value': u'Location of the movie on your local filesystem (not uploaded). Used for playback of lower resolution movie media stored locally.'}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, + u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, + u'value': u'Path to Movie'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'sg_status_list': {u'data_type': {u'editable': False, + u'value': u'status_list'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, + u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Status'}, + u'properties': {u'default_value': {u'editable': True, + u'value': u'rev'}, + u'summary_default': {u'editable': True, + u'value': u'status_list'}, + u'valid_values': {u'editable': True, + u'value': [u'na', + u'rev', + u'vwd']}}}, + u'sg_task': {u'data_type': {u'editable': False, u'value': u'entity'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Task'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}, + u'valid_types': {u'editable': True, + u'value': [u'Task']}}}, + u'sg_uploaded_movie': {u'data_type': {u'editable': False, u'value': u'url'}, + u'description': {u'editable': True, + u'value': u'File field to contain the uploaded movie file. Used for playback of lower resolution movie media stored in Shotgun.'}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, + u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, + u'value': u'Uploaded Movie'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'open_in_new_window': {u'editable': True, + u'value': True}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'sg_version_type': {u'data_type': {u'editable': False, u'value': u'list'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, + u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Type'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}, + u'valid_values': {u'editable': True, + u'value': []}}}, + u'step_0': {u'data_type': {u'editable': False, u'value': u'pivot_column'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': False}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'ALL TASKS'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': False, + u'value': u'none'}}}, + u'tag_list': {u'data_type': {u'editable': False, u'value': u'tag_list'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Tags'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}, + u'valid_types': {u'editable': True, + u'value': [u'Tag']}}}, + u'task_template': {u'data_type': {u'editable': False, u'value': u'entity'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, + u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Task Template'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}, + u'valid_types': {u'editable': True, + u'value': [u'TaskTemplate']}}}, + u'tasks': {u'data_type': {u'editable': False, u'value': u'multi_entity'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Tasks'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}, + u'valid_types': {u'editable': True, + u'value': [u'Task']}}}, + u'updated_at': {u'data_type': {u'editable': False, u'value': u'date_time'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': False}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Date Updated'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'updated_by': {u'data_type': {u'editable': False, u'value': u'entity'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': False}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Updated by'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}, + u'valid_types': {u'editable': True, + u'value': [u'HumanUser', + u'ApiUser']}}}, + u'user': {u'data_type': {u'editable': False, u'value': u'entity'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Artist'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}, + u'valid_types': {u'editable': True, + u'value': [u'HumanUser', + u'ApiUser']}}} +}} + + + + + + + + +schema_field_read_version = {u'code': {u'data_type': {u'editable': False, u'value': u'text'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': True}, + u'name': {u'editable': True, u'value': u'Version Name'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'created_at': {u'data_type': {u'editable': False, u'value': u'date_time'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': False}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Date Created'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'created_by': {u'data_type': {u'editable': False, u'value': u'entity'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': False}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Created by'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}, + u'valid_types': {u'editable': True, + u'value': [u'HumanUser', + u'ApiUser']}}}, + u'description': {u'data_type': {u'editable': False, u'value': u'text'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Description'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'entity': {u'data_type': {u'editable': False, u'value': u'entity'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Link'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}, + u'valid_types': {u'editable': True, + u'value': [u'Asset', + u'Scene', + u'Sequence', + u'Shot']}}}, + u'frame_count': {u'data_type': {u'editable': False, u'value': u'number'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Frame Count'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'frame_range': {u'data_type': {u'editable': False, u'value': u'text'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Frame Range'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'id': {u'data_type': {u'editable': False, u'value': u'number'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': False}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Id'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'image': {u'data_type': {u'editable': False, u'value': u'image'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Thumbnail'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'notes': {u'data_type': {u'editable': False, u'value': u'multi_entity'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Notes'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}, + u'valid_types': {u'editable': True, + u'value': [u'Note']}}}, + u'open_notes': {u'data_type': {u'editable': False, + u'value': u'multi_entity'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': False}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Open Notes'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}, + u'valid_types': {u'editable': True, + u'value': [u'Note']}}}, + u'open_notes_count': {u'data_type': {u'editable': False, u'value': u'text'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': False}, + u'entity_type': {u'editable': False, + u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, + u'value': u'Open Notes Count'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'playlists': {u'data_type': {u'editable': False, u'value': u'multi_entity'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Playlists'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}, + u'valid_types': {u'editable': True, + u'value': [u'Playlist']}}}, + u'project': {u'data_type': {u'editable': False, u'value': u'entity'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Project'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}, + u'valid_types': {u'editable': True, + u'value': [u'Project']}}}, + u'sg_department': {u'data_type': {u'editable': False, u'value': u'text'}, + u'description': {u'editable': True, + u'value': u'The department the Version was submitted from. This is used to find the latest Version from the same department.'}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, + u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Department'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'sg_first_frame': {u'data_type': {u'editable': False, u'value': u'number'}, + u'description': {u'editable': True, + u'value': u'The first frame number contained in the Version. Used in playback of the movie or frames to calculate the first frame available in the Version.'}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, + u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'First Frame'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'sg_frames_aspect_ratio': {u'data_type': {u'editable': False, + u'value': u'float'}, + u'description': {u'editable': True, + u'value': u'Aspect ratio of the high res frames. Used to format the image correctly for viewing.'}, + u'editable': {u'editable': False, + u'value': True}, + u'entity_type': {u'editable': False, + u'value': u'Version'}, + u'mandatory': {u'editable': False, + u'value': False}, + u'name': {u'editable': True, + u'value': u'Frames Aspect Ratio'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'sg_frames_have_slate': {u'data_type': {u'editable': False, + u'value': u'checkbox'}, + u'description': {u'editable': True, + u'value': u'Indicates whether the frames have a slate or not. This is used to include or omit the slate from playback.'}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, + u'value': u'Version'}, + u'mandatory': {u'editable': False, + u'value': False}, + u'name': {u'editable': True, + u'value': u'Frames Have Slate'}, + u'properties': {u'default_value': {u'editable': False, + u'value': False}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'sg_last_frame': {u'data_type': {u'editable': False, u'value': u'number'}, + u'description': {u'editable': True, + u'value': u'The last frame number contained in the Version. Used in playback of the movie or frames to calculate the last frame available in the Version.'}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, + u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Last Frame'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'sg_movie_aspect_ratio': {u'data_type': {u'editable': False, + u'value': u'float'}, + u'description': {u'editable': True, + u'value': u'Aspect ratio of the the movie. Used to format the image correctly for viewing.'}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, + u'value': u'Version'}, + u'mandatory': {u'editable': False, + u'value': False}, + u'name': {u'editable': True, + u'value': u'Movie Aspect Ratio'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'sg_movie_has_slate': {u'data_type': {u'editable': False, + u'value': u'checkbox'}, + u'description': {u'editable': True, + u'value': u'Indicates whether the movie file has a slate or not. This is used to include or omit the slate from playback.'}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, + u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, + u'value': u'Movie Has Slate'}, + u'properties': {u'default_value': {u'editable': False, + u'value': False}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'sg_path_to_frames': {u'data_type': {u'editable': False, u'value': u'text'}, + u'description': {u'editable': True, + u'value': u'Location of the high res frames on your local filesystem. Used for playback of high resolution frames.'}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, + u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, + u'value': u'Path to Frames'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'sg_path_to_movie': {u'data_type': {u'editable': False, u'value': u'text'}, + u'description': {u'editable': True, + u'value': u'Location of the movie on your local filesystem (not uploaded). Used for playback of lower resolution movie media stored locally.'}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, + u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, + u'value': u'Path to Movie'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'sg_status_list': {u'data_type': {u'editable': False, + u'value': u'status_list'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, + u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Status'}, + u'properties': {u'default_value': {u'editable': True, + u'value': u'rev'}, + u'summary_default': {u'editable': True, + u'value': u'status_list'}, + u'valid_values': {u'editable': True, + u'value': [u'na', + u'rev', + u'vwd']}}}, + u'sg_task': {u'data_type': {u'editable': False, u'value': u'entity'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Task'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}, + u'valid_types': {u'editable': True, + u'value': [u'Task']}}}, + u'sg_uploaded_movie': {u'data_type': {u'editable': False, u'value': u'url'}, + u'description': {u'editable': True, + u'value': u'File field to contain the uploaded movie file. Used for playback of lower resolution movie media stored in Shotgun.'}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, + u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, + u'value': u'Uploaded Movie'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'open_in_new_window': {u'editable': True, + u'value': True}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'sg_version_type': {u'data_type': {u'editable': False, u'value': u'list'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, + u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Type'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}, + u'valid_values': {u'editable': True, + u'value': []}}}, + u'step_0': {u'data_type': {u'editable': False, u'value': u'pivot_column'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': False}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'ALL TASKS'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': False, + u'value': u'none'}}}, + u'tag_list': {u'data_type': {u'editable': False, u'value': u'tag_list'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Tags'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}, + u'valid_types': {u'editable': True, + u'value': [u'Tag']}}}, + u'task_template': {u'data_type': {u'editable': False, u'value': u'entity'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, + u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Task Template'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}, + u'valid_types': {u'editable': True, + u'value': [u'TaskTemplate']}}}, + u'tasks': {u'data_type': {u'editable': False, u'value': u'multi_entity'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Tasks'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}, + u'valid_types': {u'editable': True, + u'value': [u'Task']}}}, + u'updated_at': {u'data_type': {u'editable': False, u'value': u'date_time'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': False}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Date Updated'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}}}, + u'updated_by': {u'data_type': {u'editable': False, u'value': u'entity'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': False}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Updated by'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}, + u'valid_types': {u'editable': True, + u'value': [u'HumanUser', + u'ApiUser']}}}, + u'user': {u'data_type': {u'editable': False, u'value': u'entity'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Artist'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}, + u'valid_types': {u'editable': True, + u'value': [u'HumanUser', + u'ApiUser']}}}} + + + + + +schema_field_read_version_user = {u'user': {u'data_type': {u'editable': False, u'value': u'entity'}, + u'description': {u'editable': True, u'value': u''}, + u'editable': {u'editable': False, u'value': True}, + u'entity_type': {u'editable': False, u'value': u'Version'}, + u'mandatory': {u'editable': False, u'value': False}, + u'name': {u'editable': True, u'value': u'Artist'}, + u'properties': {u'default_value': {u'editable': False, + u'value': None}, + u'summary_default': {u'editable': True, + u'value': u'none'}, + u'valid_types': {u'editable': True, + u'value': [u'HumanUser', + u'ApiUser']}}}} + \ No newline at end of file diff --git a/tests/mock.py b/tests/mock.py new file mode 100644 index 000000000..b07172a3c --- /dev/null +++ b/tests/mock.py @@ -0,0 +1,958 @@ +# mock.py +# Test tools for mocking and patching. +# Copyright (C) 2007-2011 Michael Foord & the mock team +# E-mail: fuzzyman AT voidspace DOT org DOT uk + +# mock 0.7.0 +# http://www.voidspace.org.uk/python/mock/ + +# Released subject to the BSD License +# Please see http://www.voidspace.org.uk/python/license.shtml + +# Scripts maintained at http://www.voidspace.org.uk/python/index.shtml +# Comments, suggestions and bug reports welcome. + + +__all__ = ( + 'Mock', + 'MagicMock', + 'mocksignature', + 'patch', + 'patch_object', + 'sentinel', + 'DEFAULT' +) + +__version__ = '0.7.0' + +__unittest = True + + +import sys +import warnings + +try: + import inspect +except ImportError: + # for alternative platforms that + # may not have inspect + inspect = None + +try: + BaseException +except NameError: + # Python 2.4 compatibility + BaseException = Exception + +try: + from functools import wraps +except ImportError: + # Python 2.4 compatibility + def wraps(original): + def inner(f): + f.__name__ = original.__name__ + f.__doc__ = original.__doc__ + f.__module__ = original.__module__ + return f + return inner + +try: + unicode +except NameError: + # Python 3 + basestring = unicode = str + +try: + long +except NameError: + # Python 3 + long = int + +inPy3k = sys.version_info[0] == 3 + +if inPy3k: + self = '__self__' +else: + self = 'im_self' + + +# getsignature and mocksignature heavily "inspired" by +# the decorator module: http://pypi.python.org/pypi/decorator/ +# by Michele Simionato + +def _getsignature(func, skipfirst): + if inspect is None: + raise ImportError('inspect module not available') + + if inspect.isclass(func): + func = func.__init__ + # will have a self arg + skipfirst = True + elif not (inspect.ismethod(func) or inspect.isfunction(func)): + func = func.__call__ + + regargs, varargs, varkwargs, defaults = inspect.getargspec(func) + + # instance methods need to lose the self argument + if getattr(func, self, None) is not None: + regargs = regargs[1:] + + _msg = "_mock_ is a reserved argument name, can't mock signatures using _mock_" + assert '_mock_' not in regargs, _msg + if varargs is not None: + assert '_mock_' not in varargs, _msg + if varkwargs is not None: + assert '_mock_' not in varkwargs, _msg + if skipfirst: + regargs = regargs[1:] + signature = inspect.formatargspec(regargs, varargs, varkwargs, defaults, + formatvalue=lambda value: "") + return signature[1:-1], func + + +def _copy_func_details(func, funcopy): + funcopy.__name__ = func.__name__ + funcopy.__doc__ = func.__doc__ + funcopy.__dict__.update(func.__dict__) + funcopy.__module__ = func.__module__ + if not inPy3k: + funcopy.func_defaults = func.func_defaults + else: + funcopy.__defaults__ = func.__defaults__ + funcopy.__kwdefaults__ = func.__kwdefaults__ + + +def mocksignature(func, mock=None, skipfirst=False): + """ + mocksignature(func, mock=None, skipfirst=False) + + Create a new function with the same signature as `func` that delegates + to `mock`. If `skipfirst` is True the first argument is skipped, useful + for methods where `self` needs to be omitted from the new function. + + If you don't pass in a `mock` then one will be created for you. + + The mock is set as the `mock` attribute of the returned function for easy + access. + + `mocksignature` can also be used with classes. It copies the signature of + the `__init__` method. + + When used with callable objects (instances) it copies the signature of the + `__call__` method. + """ + if mock is None: + mock = Mock() + signature, func = _getsignature(func, skipfirst) + src = "lambda %(signature)s: _mock_(%(signature)s)" % { + 'signature': signature + } + + funcopy = eval(src, dict(_mock_=mock)) + _copy_func_details(func, funcopy) + funcopy.mock = mock + return funcopy + + +def _is_magic(name): + return '__%s__' % name[2:-2] == name + + +class SentinelObject(object): + "A unique, named, sentinel object." + def __init__(self, name): + self.name = name + + def __repr__(self): + return '' % self.name + + +class Sentinel(object): + """Access attributes to return a named object, usable as a sentinel.""" + def __init__(self): + self._sentinels = {} + + def __getattr__(self, name): + if name == '__bases__': + # Without this help(mock) raises an exception + raise AttributeError + return self._sentinels.setdefault(name, SentinelObject(name)) + + +sentinel = Sentinel() + +DEFAULT = sentinel.DEFAULT + + +class OldStyleClass: + pass +ClassType = type(OldStyleClass) + + +def _copy(value): + if type(value) in (dict, list, tuple, set): + return type(value)(value) + return value + + +if inPy3k: + class_types = type +else: + class_types = (type, ClassType) + + +class Mock(object): + """ + Create a new ``Mock`` object. ``Mock`` takes several optional arguments + that specify the behaviour of the Mock object: + + * ``spec``: This can be either a list of strings or an existing object (a + class or instance) that acts as the specification for the mock object. If + you pass in an object then a list of strings is formed by calling dir on + the object (excluding unsupported magic attributes and methods). Accessing + any attribute not in this list will raise an ``AttributeError``. + + If ``spec`` is an object (rather than a list of strings) then + `mock.__class__` returns the class of the spec object. This allows mocks + to pass `isinstance` tests. + + * ``spec_set``: A stricter variant of ``spec``. If used, attempting to *set* + or get an attribute on the mock that isn't on the object passed as + ``spec_set`` will raise an ``AttributeError``. + + * ``side_effect``: A function to be called whenever the Mock is called. See + the :attr:`Mock.side_effect` attribute. Useful for raising exceptions or + dynamically changing return values. The function is called with the same + arguments as the mock, and unless it returns :data:`DEFAULT`, the return + value of this function is used as the return value. + + Alternatively ``side_effect`` can be an exception class or instance. In + this case the exception will be raised when the mock is called. + + * ``return_value``: The value returned when the mock is called. By default + this is a new Mock (created on first access). See the + :attr:`Mock.return_value` attribute. + + * ``wraps``: Item for the mock object to wrap. If ``wraps`` is not None + then calling the Mock will pass the call through to the wrapped object + (returning the real result and ignoring ``return_value``). Attribute + access on the mock will return a Mock object that wraps the corresponding + attribute of the wrapped object (so attempting to access an attribute that + doesn't exist will raise an ``AttributeError``). + + If the mock has an explicit ``return_value`` set then calls are not passed + to the wrapped object and the ``return_value`` is returned instead. + + * ``name``: If the mock has a name then it will be used in the repr of the + mock. This can be useful for debugging. The name is propagated to child + mocks. + """ + def __new__(cls, *args, **kw): + # every instance has its own class + # so we can create magic methods on the + # class without stomping on other mocks + new = type(cls.__name__, (cls,), {'__doc__': cls.__doc__}) + return object.__new__(new) + + + def __init__(self, spec=None, side_effect=None, return_value=DEFAULT, + wraps=None, name=None, spec_set=None, parent=None): + self._parent = parent + self._name = name + _spec_class = None + if spec_set is not None: + spec = spec_set + spec_set = True + + if spec is not None and not isinstance(spec, list): + if isinstance(spec, class_types): + _spec_class = spec + else: + _spec_class = spec.__class__ + spec = dir(spec) + + self._spec_class = _spec_class + self._spec_set = spec_set + self._methods = spec + self._children = {} + self._return_value = return_value + self.side_effect = side_effect + self._wraps = wraps + + self.reset_mock() + + + @property + def __class__(self): + if self._spec_class is None: + return type(self) + return self._spec_class + + + def reset_mock(self): + "Restore the mock object to its initial state." + self.called = False + self.call_args = None + self.call_count = 0 + self.call_args_list = [] + self.method_calls = [] + for child in self._children.values(): + child.reset_mock() + if isinstance(self._return_value, Mock): + if not self._return_value is self: + self._return_value.reset_mock() + + + def __get_return_value(self): + if self._return_value is DEFAULT: + self._return_value = self._get_child_mock() + return self._return_value + + def __set_return_value(self, value): + self._return_value = value + + __return_value_doc = "The value to be returned when the mock is called." + return_value = property(__get_return_value, __set_return_value, + __return_value_doc) + + + def __call__(self, *args, **kwargs): + self.called = True + self.call_count += 1 + self.call_args = callargs((args, kwargs)) + self.call_args_list.append(callargs((args, kwargs))) + + parent = self._parent + name = self._name + while parent is not None: + parent.method_calls.append(callargs((name, args, kwargs))) + if parent._parent is None: + break + name = parent._name + '.' + name + parent = parent._parent + + ret_val = DEFAULT + if self.side_effect is not None: + if (isinstance(self.side_effect, BaseException) or + isinstance(self.side_effect, class_types) and + issubclass(self.side_effect, BaseException)): + raise self.side_effect + + ret_val = self.side_effect(*args, **kwargs) + if ret_val is DEFAULT: + ret_val = self.return_value + + if self._wraps is not None and self._return_value is DEFAULT: + return self._wraps(*args, **kwargs) + if ret_val is DEFAULT: + ret_val = self.return_value + return ret_val + + + def __getattr__(self, name): + if name == '_methods': + raise AttributeError(name) + elif self._methods is not None: + if name not in self._methods or name in _all_magics: + raise AttributeError("Mock object has no attribute '%s'" % name) + elif _is_magic(name): + raise AttributeError(name) + + if name not in self._children: + wraps = None + if self._wraps is not None: + wraps = getattr(self._wraps, name) + self._children[name] = self._get_child_mock(parent=self, name=name, wraps=wraps) + + return self._children[name] + + + def __repr__(self): + if self._name is None and self._spec_class is None: + return object.__repr__(self) + + name_string = '' + spec_string = '' + if self._name is not None: + def get_name(name): + if name is None: + return 'mock' + return name + parent = self._parent + name = self._name + while parent is not None: + name = get_name(parent._name) + '.' + name + parent = parent._parent + name_string = ' name=%r' % name + if self._spec_class is not None: + spec_string = ' spec=%r' + if self._spec_set: + spec_string = ' spec_set=%r' + spec_string = spec_string % self._spec_class.__name__ + return "<%s%s%s id='%s'>" % (type(self).__name__, + name_string, + spec_string, + id(self)) + + + def __setattr__(self, name, value): + if not 'method_calls' in self.__dict__: + # allow all attribute setting until initialisation is complete + return object.__setattr__(self, name, value) + if (self._spec_set and self._methods is not None and name not in + self._methods and name not in self.__dict__ and + name != 'return_value'): + raise AttributeError("Mock object has no attribute '%s'" % name) + if name in _unsupported_magics: + msg = 'Attempting to set unsupported magic method %r.' % name + raise AttributeError(msg) + elif name in _all_magics: + if self._methods is not None and name not in self._methods: + raise AttributeError("Mock object has no attribute '%s'" % name) + + if not isinstance(value, Mock): + setattr(type(self), name, _get_method(name, value)) + original = value + real = lambda *args, **kw: original(self, *args, **kw) + value = mocksignature(value, real, skipfirst=True) + else: + setattr(type(self), name, value) + return object.__setattr__(self, name, value) + + + def __delattr__(self, name): + if name in _all_magics and name in type(self).__dict__: + delattr(type(self), name) + return object.__delattr__(self, name) + + + def assert_called_with(self, *args, **kwargs): + """ + assert that the mock was called with the specified arguments. + + Raises an AssertionError if the args and keyword args passed in are + different to the last call to the mock. + """ + if self.call_args is None: + raise AssertionError('Expected: %s\nNot called' % ((args, kwargs),)) + if not self.call_args == (args, kwargs): + raise AssertionError( + 'Expected: %s\nCalled with: %s' % ((args, kwargs), self.call_args) + ) + + + def assert_called_once_with(self, *args, **kwargs): + """ + assert that the mock was called exactly once and with the specified + arguments. + """ + if not self.call_count == 1: + msg = ("Expected to be called once. Called %s times." % + self.call_count) + raise AssertionError(msg) + return self.assert_called_with(*args, **kwargs) + + + def _get_child_mock(self, **kw): + klass = type(self).__mro__[1] + return klass(**kw) + + + +class callargs(tuple): + """ + A tuple for holding the results of a call to a mock, either in the form + `(args, kwargs)` or `(name, args, kwargs)`. + + If args or kwargs are empty then a callargs tuple will compare equal to + a tuple without those values. This makes comparisons less verbose:: + + callargs('name', (), {}) == ('name',) + callargs('name', (1,), {}) == ('name', (1,)) + callargs((), {'a': 'b'}) == ({'a': 'b'},) + """ + def __eq__(self, other): + if len(self) == 3: + if other[0] != self[0]: + return False + args_kwargs = self[1:] + other_args_kwargs = other[1:] + else: + args_kwargs = tuple(self) + other_args_kwargs = other + + if len(other_args_kwargs) == 0: + other_args, other_kwargs = (), {} + elif len(other_args_kwargs) == 1: + if isinstance(other_args_kwargs[0], tuple): + other_args = other_args_kwargs[0] + other_kwargs = {} + else: + other_args = () + other_kwargs = other_args_kwargs[0] + else: + other_args, other_kwargs = other_args_kwargs + + return tuple(args_kwargs) == (other_args, other_kwargs) + + +def _dot_lookup(thing, comp, import_path): + try: + return getattr(thing, comp) + except AttributeError: + __import__(import_path) + return getattr(thing, comp) + + +def _importer(target): + components = target.split('.') + import_path = components.pop(0) + thing = __import__(import_path) + + for comp in components: + import_path += ".%s" % comp + thing = _dot_lookup(thing, comp, import_path) + return thing + + +class _patch(object): + def __init__(self, target, attribute, new, spec, create, + mocksignature, spec_set): + self.target = target + self.attribute = attribute + self.new = new + self.spec = spec + self.create = create + self.has_local = False + self.mocksignature = mocksignature + self.spec_set = spec_set + + + def copy(self): + return _patch(self.target, self.attribute, self.new, self.spec, + self.create, self.mocksignature, self.spec_set) + + + def __call__(self, func): + if isinstance(func, class_types): + return self.decorate_class(func) + else: + return self.decorate_callable(func) + + + def decorate_class(self, klass): + for attr in dir(klass): + attr_value = getattr(klass, attr) + if attr.startswith("test") and hasattr(attr_value, "__call__"): + setattr(klass, attr, self.copy()(attr_value)) + return klass + + + def decorate_callable(self, func): + if hasattr(func, 'patchings'): + func.patchings.append(self) + return func + + @wraps(func) + def patched(*args, **keywargs): + # don't use a with here (backwards compatability with 2.5) + extra_args = [] + for patching in patched.patchings: + arg = patching.__enter__() + if patching.new is DEFAULT: + extra_args.append(arg) + args += tuple(extra_args) + try: + return func(*args, **keywargs) + finally: + for patching in reversed(getattr(patched, 'patchings', [])): + patching.__exit__() + + patched.patchings = [self] + if hasattr(func, 'func_code'): + # not in Python 3 + patched.compat_co_firstlineno = getattr(func, "compat_co_firstlineno", + func.func_code.co_firstlineno) + return patched + + + def get_original(self): + target = self.target + name = self.attribute + + original = DEFAULT + local = False + + try: + original = target.__dict__[name] + except (AttributeError, KeyError): + original = getattr(target, name, DEFAULT) + else: + local = True + + if not self.create and original is DEFAULT: + raise AttributeError("%s does not have the attribute %r" % (target, name)) + return original, local + + + def __enter__(self): + """Perform the patch.""" + new, spec, spec_set = self.new, self.spec, self.spec_set + original, local = self.get_original() + if new is DEFAULT: + # XXXX what if original is DEFAULT - shouldn't use it as a spec + inherit = False + if spec_set == True: + spec_set = original + if isinstance(spec_set, class_types): + inherit = True + elif spec == True: + # set spec to the object we are replacing + spec = original + if isinstance(spec, class_types): + inherit = True + new = Mock(spec=spec, spec_set=spec_set) + if inherit: + new.return_value = Mock(spec=spec, spec_set=spec_set) + new_attr = new + if self.mocksignature: + new_attr = mocksignature(original, new) + + self.temp_original = original + self.is_local = local + setattr(self.target, self.attribute, new_attr) + return new + + + def __exit__(self, *_): + """Undo the patch.""" + if self.is_local and self.temp_original is not DEFAULT: + setattr(self.target, self.attribute, self.temp_original) + else: + delattr(self.target, self.attribute) + if not self.create and not hasattr(self.target, self.attribute): + # needed for proxy objects like django settings + setattr(self.target, self.attribute, self.temp_original) + + del self.temp_original + del self.is_local + + start = __enter__ + stop = __exit__ + + +def _patch_object(target, attribute, new=DEFAULT, spec=None, create=False, + mocksignature=False, spec_set=None): + """ + patch.object(target, attribute, new=DEFAULT, spec=None, create=False, + mocksignature=False, spec_set=None) + + patch the named member (`attribute`) on an object (`target`) with a mock + object. + + Arguments new, spec, create, mocksignature and spec_set have the same + meaning as for patch. + """ + return _patch(target, attribute, new, spec, create, mocksignature, + spec_set) + + +def patch_object(*args, **kwargs): + "A deprecated form of patch.object(...)" + warnings.warn(('Please use patch.object instead.'), DeprecationWarning, 2) + return _patch_object(*args, **kwargs) + + +def patch(target, new=DEFAULT, spec=None, create=False, + mocksignature=False, spec_set=None): + """ + ``patch`` acts as a function decorator, class decorator or a context + manager. Inside the body of the function or with statement, the ``target`` + (specified in the form `'PackageName.ModuleName.ClassName'`) is patched + with a ``new`` object. When the function/with statement exits the patch is + undone. + + The target is imported and the specified attribute patched with the new + object, so it must be importable from the environment you are calling the + decorator from. + + If ``new`` is omitted, then a new ``Mock`` is created and passed in as an + extra argument to the decorated function. + + The ``spec`` and ``spec_set`` keyword arguments are passed to the ``Mock`` + if patch is creating one for you. + + In addition you can pass ``spec=True`` or ``spec_set=True``, which causes + patch to pass in the object being mocked as the spec/spec_set object. + + If ``mocksignature`` is True then the patch will be done with a function + created by mocking the one being replaced. If the object being replaced is + a class then the signature of `__init__` will be copied. If the object + being replaced is a callable object then the signature of `__call__` will + be copied. + + By default ``patch`` will fail to replace attributes that don't exist. If + you pass in 'create=True' and the attribute doesn't exist, patch will + create the attribute for you when the patched function is called, and + delete it again afterwards. This is useful for writing tests against + attributes that your production code creates at runtime. It is off by by + default because it can be dangerous. With it switched on you can write + passing tests against APIs that don't actually exist! + + Patch can be used as a TestCase class decorator. It works by + decorating each test method in the class. This reduces the boilerplate + code when your test methods share a common patchings set. + + Patch can be used with the with statement, if this is available in your + version of Python. Here the patching applies to the indented block after + the with statement. If you use "as" then the patched object will be bound + to the name after the "as"; very useful if `patch` is creating a mock + object for you. + + `patch.dict(...)` and `patch.object(...)` are available for alternate + use-cases. + """ + try: + target, attribute = target.rsplit('.', 1) + except (TypeError, ValueError): + raise TypeError("Need a valid target to patch. You supplied: %r" % + (target,)) + target = _importer(target) + return _patch(target, attribute, new, spec, create, mocksignature, spec_set) + + +class _patch_dict(object): + """ + Patch a dictionary and restore the dictionary to its original state after + the test. + + `in_dict` can be a dictionary or a mapping like container. If it is a + mapping then it must at least support getting, setting and deleting items + plus iterating over keys. + + `in_dict` can also be a string specifying the name of the dictionary, which + will then be fetched by importing it. + + `values` can be a dictionary of values to set in the dictionary. `values` + can also be an iterable of ``(key, value)`` pairs. + + If `clear` is True then the dictionary will be cleared before the new + values are set. + """ + + def __init__(self, in_dict, values=(), clear=False): + if isinstance(in_dict, basestring): + in_dict = _importer(in_dict) + self.in_dict = in_dict + # support any argument supported by dict(...) constructor + self.values = dict(values) + self.clear = clear + self._original = None + + + def __call__(self, f): + if isinstance(f, class_types): + return self.decorate_class(f) + @wraps(f) + def _inner(*args, **kw): + self._patch_dict() + try: + return f(*args, **kw) + finally: + self._unpatch_dict() + + return _inner + + + def decorate_class(self, klass): + for attr in dir(klass): + attr_value = getattr(klass, attr) + if attr.startswith("test") and hasattr(attr_value, "__call__"): + decorator = _patch_dict(self.in_dict, self.values, self.clear) + decorated = decorator(attr_value) + setattr(klass, attr, decorated) + return klass + + + def __enter__(self): + """Patch the dict.""" + self._patch_dict() + + + def _patch_dict(self): + """Unpatch the dict.""" + values = self.values + in_dict = self.in_dict + clear = self.clear + + try: + original = in_dict.copy() + except AttributeError: + # dict like object with no copy method + # must support iteration over keys + original = {} + for key in in_dict: + original[key] = in_dict[key] + self._original = original + + if clear: + _clear_dict(in_dict) + + try: + in_dict.update(values) + except AttributeError: + # dict like object with no update method + for key in values: + in_dict[key] = values[key] + + + def _unpatch_dict(self): + in_dict = self.in_dict + original = self._original + + _clear_dict(in_dict) + + try: + in_dict.update(original) + except AttributeError: + for key in original: + in_dict[key] = original[key] + + + def __exit__(self, *args): + self._unpatch_dict() + return False + + start = __enter__ + stop = __exit__ + + +def _clear_dict(in_dict): + try: + in_dict.clear() + except AttributeError: + keys = list(in_dict) + for key in keys: + del in_dict[key] + + +patch.object = _patch_object +patch.dict = _patch_dict + + +magic_methods = ( + "lt le gt ge eq ne " + "getitem setitem delitem " + "len contains iter " + "hash str sizeof " + "enter exit " + "divmod neg pos abs invert " + "complex int float index " + "trunc floor ceil " +) + +numerics = "add sub mul div truediv floordiv mod lshift rshift and xor or pow " +inplace = ' '.join('i%s' % n for n in numerics.split()) +right = ' '.join('r%s' % n for n in numerics.split()) +extra = '' +if inPy3k: + extra = 'bool next ' +else: + extra = 'unicode long nonzero oct hex ' +# __truediv__ and __rtruediv__ not available in Python 3 either + +# not including __prepare__, __instancecheck__, __subclasscheck__ +# (as they are metaclass methods) +# __del__ is not supported at all as it causes problems if it exists + +_non_defaults = set('__%s__' % method for method in [ + 'cmp', 'getslice', 'setslice', 'coerce', 'subclasses', + 'dir', 'format', 'get', 'set', 'delete', 'reversed', + 'missing', 'reduce', 'reduce_ex', 'getinitargs', + 'getnewargs', 'getstate', 'setstate', 'getformat', + 'setformat', 'repr' +]) + + +def _get_method(name, func): + "Turns a callable object (like a mock) into a real function" + def method(self, *args, **kw): + return func(self, *args, **kw) + method.__name__ = name + return method + + +_magics = set( + '__%s__' % method for method in + ' '.join([magic_methods, numerics, inplace, right, extra]).split() +) + +_all_magics = _magics | _non_defaults + +_unsupported_magics = set([ + '__getattr__', '__setattr__', + '__init__', '__new__', '__prepare__' + '__instancecheck__', '__subclasscheck__', + '__del__' +]) + +_calculate_return_value = { + '__hash__': lambda self: object.__hash__(self), + '__str__': lambda self: object.__str__(self), + '__sizeof__': lambda self: object.__sizeof__(self), + '__unicode__': lambda self: unicode(object.__str__(self)), +} + +_return_values = { + '__int__': 1, + '__contains__': False, + '__len__': 0, + '__iter__': iter([]), + '__exit__': False, + '__complex__': 1j, + '__float__': 1.0, + '__bool__': True, + '__nonzero__': True, + '__oct__': '1', + '__hex__': '0x1', + '__long__': long(1), + '__index__': 1, +} + + +def _set_return_value(mock, method, name): + return_value = DEFAULT + if name in _return_values: + return_value = _return_values[name] + elif name in _calculate_return_value: + try: + return_value = _calculate_return_value[name](mock) + except AttributeError: + return_value = AttributeError(name) + if return_value is not DEFAULT: + method.return_value = return_value + + +class MagicMock(Mock): + """ + MagicMock is a subclass of :Mock with default implementations + of most of the magic methods. You can use MagicMock without having to + configure the magic methods yourself. + + If you use the ``spec`` or ``spec_set`` arguments then *only* magic + methods that exist in the spec will be created. + + Attributes and the return value of a `MagicMock` will also be `MagicMocks`. + """ + def __init__(self, *args, **kw): + Mock.__init__(self, *args, **kw) + + these_magics = _magics + if self._methods is not None: + these_magics = _magics.intersection(self._methods) + + for entry in these_magics: + # could specify parent? + m = Mock() + setattr(self, entry, m) + _set_return_value(self, m, entry) diff --git a/tests/sg_logo.jpg b/tests/sg_logo.jpg new file mode 100644 index 0000000000000000000000000000000000000000..dfc5fc49d08223cad31883bc1cd89be70abcd17b GIT binary patch literal 6414 zcmb7I2Ut_d7QQ!y56{JR*^rj|OnhHn_AOR^6q)I@V zM#_pn1O!B?ihvjqDS};Hjg9sj0U!th z```quj{$oC#Gip=Rgu2|^({xiK(l^mHa0Yx{m)=^0)A@)%b?lW&}{4+-|%O_zpJg& z0X`0Zj+jM)d;o$EMDl^_?Z9RL1d#A(;qm?s5P?LY**G|1t^qFq*U(?V#{N}+#pOe@ z@o&*!7x=}(RZz-fI=hbM?wH#{9oulmi~?^3IJ$FH1r+i-^DbGqww^YV@P7uE4i`FKLbsJ-^eR$;uL&W|*ia%sxSLexM*C ziA!onrf?^lGw&9kPE1S^hl>mt%LQh_qva_0>)OM{;KWZOeI3uxoyE0E{SbzmRcp;e@;-LNcP!{($Ga))xp1*oeOHONs)0_t@E;ed{q(#+t7bYq$pvb@Sd(+wPaq>2oquQ1s@p2cy=Cj4Kt`2hBDAT#e+nptdUQK3@9lPu*B#n53 zS3XU_QutUCz7vWMM%>hi0tJD%50DDLQ2;`K9WOVe0W6U4!0tKp_oD-3p_@Xj?1STD z80RHtWxadk_$tcY;9RVIZE*4hyE2i6ey2UVOrg()qEiz`#C|$e8AB> zTSJ(wQEqGIr|vfT*vT-oiWRdeEKS!FtIFGl?u6$$tEoY&wt_L14ab8hui zd8X}E^{FCZ36CqE^{)k(j{THUx0dUF@Il|L-YOMEMa3+tCCxBqf2Km!lWHgXBaUW4 z%Li!k-7h_^*3Wxl>p!O-zr^EO*gJBc)}(9eB<(iwWuW<$h{Z^NUXg6fqz%%@8L^WC=&re{((#Q1+E7J^{e#ly6^)mW3%G0n_b}{( z8)}I&5AJ%5wq)PfP2v)44WpVRY>c?O_o(Fs!f*@IO}^ab61lNS|EBGra=7=}nHz-_ zAv-INC%ZG;&U9KX_l-_RD#;A?2IeEs;@A7yiV{jnkVT+ZOMeoVl<+z0G!#7;3)GQCByzspxwz=+n zbYyO6&8HU&;US`VDVOTDhxS%Y&6@?SwBL}OINIhO>Or|#A9t>Qyl7kTf$m=_qwGcx zMN3twnK9<~#~q3g^1H3!(UyIluk>}#kP(Hhu5@uadHW^Ek~CxF7re+P;xTFV74I6X9JrxjVc z4I~_S8l}hmN$~Za_#s2DXo0}@DSKa*v5nuV5lHu(S~*n5m~4GqD>R&OM6Umu<54vs zi}JlgyN04^!_o|mpYofJzH~M9+ufa=l7Qn6GMYE<8np_vDYK4~KvJOSfyrQn)nTQ+ ztekBbjD4?8ts0DmMDy<|QmOYz&lwalS@Uss8rY)kD(~HPNnXk;cDQ5H?Yi8QF_RnD z#Y-nIPGFM?1Gv-3zcNaT7dA&86m&BPXKUGf&6#U=OnCdoGjTOdqPVH!*T>q>sxP10 z_8sK7@|bNUs)}DLJR%}MJ8;Kxgh^zy-|e6sbtk%R_&%?5vuyKiz2%ct-9M|YHGkxm z%n{cedQqaE94$5meR9AThv|pTPe)^y*8z_qYQslu?+c#}tVl4&yuxcb>ilIM_Tn0@ zbd_!RxfX3Fja`&(f2_6WIG?syP#(CnDj(Am8n{bWf_X4}_lwDU(GK$_o-1+v&$+=6U*o$+7x}e?OEo1 zI~xh^6z}Ackwx0lUA~Z-4#U?&$Pp`}52PANQE37j<80rCb984DNGF9|(WOaa z=85a44F=LX9~pU3D`EM^~Um+KE?(yf)Jf^J{e9YZ4ea`1F{t{ z#PTf_Idjd#(%pn%HFewwxj7?a+)J`9N|$UGmw*4G?BGB}JbNBLBPxcgQd)a6pZoae z+XJ+ct@M&k&0cm$vC(i2Erf^yO}h#zh(<3UhHI0(H~+{J`w&cT_^)}DlB|1Zg4mX4 zYymJF?j~)rfC?yBM8a?0Bq#_e8-C}xp{T+D!1WA|MZ6dnN2;sqfY zj1r!E7%$1gFE)IhD$bJ@szhb?Ro=C5z|^hjo|3(n*_^G+bWHv|hJnlUnbl8Rm(4AC z81>KAfwOM*=Iel=g90_)6vh*tJik^-h!Ykb7sfw{3r`omt*y2W>~YNZn{B@_vhh<#3^|w};(sttAfr$D zXPnt#^Gd4FUMr&TT{DRAs4EW>Lm!g!xdRNz`z%U|$>y877CS-xb=eMOy976++b>2H zhi>q*93~x@YM-Ghh1@Tv8lWZL*I`SM-TZ`jVBk80r0J43`z8qnSIEY${ySb0;|FbKwr959yaj9(4=5 z5BRTz*DQ|E-gy>oF*0y@>26S&ujt1#@tu5X7M}0t$vAWIt>vx^ajbsXXzfkYlM-a# zpcgInNnuT4Q^WS%Mud`X(a+QE*`dYls;!kL33dES3noqLfET4b+nbR%9&z+lU9$2v zQ%e*oU~*NJr$r=MWhA7=R_kSEmVJzfXkMhkSW$+)2mgDS!HC!WLYg7g&MkvG4ltcl zb;Z2!x4nE%kCVgBYd=ojYHqno+1f#Tcs*5Fpg_a5er_7KM<;#T_MGXBiyrw`^=B5< z#vLyqJXg$$&0`{tnq7OjyYGEgLa+KvkV?4Ha&T&yIl^ruj-M9c*t~OPA=I`+t0X}~ zz3lzD0*RjwrVZg|$1e0Mu<6n{dAS03+4Lw55jpe=%Rz{;O?116W@}7P zRKdy0Oo0|>im_ohhE!@8Hx*75m1#HU6)hZV1Pe^%1u3F+9>PU=r#ZW5cJlg~nrZ5VCbY2U%$k0F(TX(dfms1_DZ-TT^TteT z(_r)ALCvLSj^6hY!o?HJFl5rqtvP$1fiULDt#(W=>A1Jf4+7qY1_l3;RxdWM{L8hR zLMNY;s6mg-7i(t8wTBKqUb~hb-`O6pAXOy%a|V8{;GVwTw1JCcN7bPnr5`UbQ?)B{ z?x)7N9E(ZgGVwogPxzhx8%t$&6N4{eEkK8dTZC{Qcax?$f<*4tzw-_x)5Fg~#V?}H z6#5S;?I55I<3$8ncf)U+Mk*HNilEr1Rc4AttE1TSEgo++`^o`m2+!kvwN~HN! zZ_N{{5w-VCAHRF?%GLcDmd>21MN+y$Mu|bA;CvWa7pW)*( z5trlrIDpW9#UJ>x!*$w8&M|=I`-oQ1ex|uB8RygaZZS7UJH2wRJ1#P?J^Ir?_Wt4= z|Aby?>9i%~c5VVia|%sBxgm~G1)A~FVhtpF_%h&fMZ5!OfDMp@H9J8xRtl`_;c}Zp zijovopF=9ixy6!N`?}j@Rg%X#Ii)@gxW5X!p!5EwB@wKOt+6aR0-1q*Zjy)i+*w5^Nt!>kDsS) z>nVLqd(olzT55QadnUuDJL6@NkAGiBSE}(bM(Dd5M1q7wq*g?~W$iSZ@f_FnokyLl zFeh}tZh>R1Vp@x&JDaNPgoDI&3c%@K_eH+lG$4cQj=fOQ^cH+>crNShaWnr*I?`I) z%TqCu16h6VNvB#%=h&1v0fsAyTpbA-gZ2u7twc-s<;(IcSwN)XRdQnJ6aer>DOrDK z3M?jS2(tMt5#>tJ^_P|ugdXe15BG3P+MNQi2;w()8jA>25ID=rwx7<~v5cTW(!4?k zAba9gFcuBDnW?@(Xq2eol`l*>L$>hKmy;^AFQJF4&}NR}o!m_CmkyEIWM3Y zKU8@x53e+LzgwqNHQ_yYdG@Grxp$U3`?gu-bX+JhxnKF6ouszZE=|W}M$optnJ{(_ zrboH9L~am|cFwQoLhkhNXiso+J8dk%YY*iD_Z<2*gllr!D4^9zc&S)4769$QuMdY( zit(|PuR5>7E`sV`gC7Ehh(P$UA^1m<0RXkU%|siZ+?oU28v;i!=)|pNfD#45wxev| zJ!dM&*^27pq@$k9A=x20qTNOa0P5uBdDU71MP&}B4oRU@~RM0xda!Zs&FP5&*T}XbTLQg_cVaZ?(28P%J4`ih`qsWHgb*fRSoU ziUAzhnM8HUhB-c4l05}O5&XsicYYvFDhMw9$f?BE>ZXrF;jRYfTI)?A;Ow{4V0m)zuGu!oN_FM0M2i8xFic( z>OX~$x6n7yczA?Ah!}p?4}zuVKSYhB5Nv5VFZ*Bvfv1M@Kbrl=!v3v1ry)g%iv2^~ nzmfeX@y@?#{v+0ZtnQD5{x919Ql9HL1V 0) + + self._mock_http(dummy_data.schema_read) + schema = self.sg.schema_read() + self.assertTrue(schema, dict) + self.assertTrue(len(schema) > 0) + + self._mock_http(dummy_data.schema_field_read_version) + schema = self.sg.schema_field_read("Version") + self.assertTrue(schema, dict) + self.assertTrue(len(schema) > 0) + + self._mock_http(dummy_data.schema_field_read_version_user) + schema = self.sg.schema_field_read("Version", field_name="user") + self.assertTrue(schema, dict) + self.assertTrue(len(schema) > 0) + self.assertTrue("user" in schema) + + self._mock_http({"results":"sg_monkeys"}) + properties = { + "description" : "How many monkeys were needed" + } + new_field_name = self.sg.schema_field_create("Version", "number", + "Monkey Count", properties=properties) + + self._mock_http({"results":True}) + properties = { + "description" : "How many monkeys turned up" + } + ret_val = self.sg.schema_field_update("Version", new_field_name, + properties) + self.assertTrue(ret_val) + + self._mock_http({"results":True}) + ret_val = self.sg.schema_field_delete("Version", new_field_name) + self.assertTrue(ret_val) + + diff --git a/tests/test_client.py b/tests/test_client.py new file mode 100644 index 000000000..81f11e3da --- /dev/null +++ b/tests/test_client.py @@ -0,0 +1,491 @@ +"""Tests agains the client software that do not involve calling the +CRUD functions. These tests always use a mock http connection so not not +need a live server to run against.""" + +import base64 +import datetime +import re +try: + import simplejson as json +except ImportError: + import json as json +import platform +import sys +import time +import unittest + +import mock +import shotgun_api3 as api + +import base + +class TestShotgunClient(base.TestBase): + def __init__(self, *args, **kws): + super(TestShotgunClient, self).__init__(*args, **kws) + + def setUp(self): + super(TestShotgunClient, self).setUp() + #get domain and uri scheme + match = re.search('(https?://)(.*)', self.server_url) + self.uri_prefix = match.group(1) + self.domain = match.group(2) + #always want the mock on + self._setup_mock() + + def test_detect_caps(self): + """Client and server capabilities detected""" + + self.sg.connect() + self.assertEqual(1, self.sg._http_request.call_count) + + self.assertTrue(self.sg.client_caps is not None) + self.assertTrue(self.sg.client_caps.platform in ( + "windows", "linux", "mac")) + self.assertTrue(self.sg.client_caps.local_path_field.startswith( + "local_path")) + self.assertTrue(str(self.sg.client_caps).startswith( + "ClientCapabilities")) + self.assertTrue(self.sg.client_caps.py_version.startswith( + str(sys.version_info[0]))) + self.assertTrue(self.sg.client_caps.py_version.endswith( + str(sys.version_info[1]))) + + + #has paging is tested else where. + server_info = { + "version" : [9,9,9] + } + self._mock_http(server_info) + # ensrue the server caps is re-read + self.sg._server_caps = None + self.assertTrue(self.sg.server_caps is not None) + self.assertFalse(self.sg.server_caps.is_dev) + self.assertEqual((9,9,9), self.sg.server_caps.version) + self.assertTrue(self.server_url.endswith(self.sg.server_caps.host)) + self.assertTrue(self.sg.server_caps.has_paging) + self.assertTrue(str(self.sg.server_caps).startswith( + "ServerCapabilities")) + self.assertEqual(server_info, self.sg.server_info) + + + self._mock_http({ + "version" : [9,9,9, "Dev"] + }) + self.sg._server_caps = None + self.assertTrue(self.sg.server_caps.is_dev) + + return + + def test_server_version(self): + """Server supports json API""" + + sc = api.ServerCapabilities("foo", {"version" : (2,4,0)}) + + sc.version = (2,3,99) + self.assertRaises(api.ShotgunError, sc._ensure_json_supported) + self.assertRaises(api.ShotgunError, api.ServerCapabilities, "foo", + {"version" : (2,2,0)}) + + sc.version = (0,0,0) + self.assertRaises(api.ShotgunError, sc._ensure_json_supported) + + sc.version = (2,4,0) + sc._ensure_json_supported() + + sc.version = (2,5,0) + sc._ensure_json_supported() + + return + + def test_session_uuid(self): + """Session UUID is included in request""" + + #ok for the mock server to just return an error, we want to look at + #whats in the request + self._mock_http({ + "message":"Go BANG", + "exception":True + }) + + def auth_args(): + args, _ = self.sg._http_request.call_args + verb, path, body, headers = args + body = json.loads(body) + return body["params"][0] + + self.sg.set_session_uuid(None) + self.assertRaises(api.Fault, self.sg.delete, "FakeType", 1) + self.assertTrue("session_uuid" not in auth_args()) + + my_uuid = '5a1d49b0-0c69-11e0-a24c-003048d17544' + self.sg.set_session_uuid(my_uuid) + self.assertRaises(api.Fault, self.sg.delete, "FakeType", 1) + self.assertEqual(my_uuid, auth_args()["session_uuid"]) + return + + def test_config(self): + """Client config can be created""" + x = api._Config() + self.assertTrue(x is not None) + + def test_url(self): + """Server url is parsed correctly""" + login = self.human_user['login'] + password = self.human_password + + self.assertRaises(ValueError, api.Shotgun, None, None, None) + self.assertRaises(ValueError, api.Shotgun, "file://foo.com",None,None) + + self.assertEqual("/api3/json", self.sg.config.api_path) + + #support auth details in the url of the form + login_password = "%s:%s" % (login, password) + # login:password@domain + auth_url = "%s%s@%s" % (self.uri_prefix, login_password, self.domain) + sg = api.Shotgun(auth_url, None, None) + expected = "Basic " + base64.encodestring(login_password).strip() + self.assertEqual(expected, sg.config.authorization) + + return + + def test_authorization(self): + """Authorization passed to server""" + login = self.human_user['login'] + password = self.human_password + login_password = "%s:%s" % (login, password) + # login:password@domain + auth_url = "%s%s@%s" % (self.uri_prefix, login_password, self.domain) + + + self.sg = api.Shotgun(auth_url, "foo", "bar") + self._setup_mock() + self._mock_http({ + 'version': [2, 4, 0, u'Dev'] + }) + + self.sg.info() + + args, _ = self.sg._http_request.call_args + verb, path, body, headers = args + + expected = "Basic " + base64.encodestring(login_password).strip() + self.assertEqual(expected, headers.get("Authorization")) + return + + def test_connect_close(self): + """Connection is closed and opened.""" + + #The mock created an existing mock connection, + self.sg.connect() + self.assertEqual(0, self.mock_conn.request.call_count) + self.sg.close() + self.assertEqual(None, self.sg._connection) + return + + def test_has_paging(self): + """Server paging detected""" + + #tricky because we now only support version > 2.4 + sc = api.ServerCapabilities("foo", {"version" : (2,4,0)}) + + self.assertFalse(sc._is_paging((0,0,0)), + "no version has no paging") + self.assertFalse(sc._is_paging((2,3,3)), + "2,3,3, has no paging") + self.assertTrue(sc._is_paging((2,3,4)), + "2,3,4, has paging") + self.assertTrue(sc._is_paging((2,3,5)), + "2,3,5, has paging") + self.assertTrue(sc._is_paging((2,4,0)), + "any 2.4 has paging") + + def test_network_retry(self): + """Network failure is retried""" + + self.sg._http_request.side_effect = api.HttpLib2Error + + self.assertRaises(api.HttpLib2Error, self.sg.info) + self.assertTrue( + self.sg.config.max_rpc_attempts ==self.sg._http_request.call_count, + "Call is repeated") + return + + def test_http_error(self): + """HTTP error raised and not retried.""" + + self._mock_http( + "big old error string", + status=(500, "Internal Server Error") + ) + + self.assertRaises(RuntimeError, self.sg.info) + self.assertEqual(1, self.sg._http_request.call_count, + "Call is not repeated") + return + + def test_rpc_error(self): + """RPC error transformed into Python error""" + + self._mock_http({ + "message":"Go BANG", + "exception":True + }) + + self.assertRaises(api.Fault, self.sg.info) + + try: + self.sg.info() + except api.Fault, e: + self.assertEqual("Go BANG", str(e)) + + def test_call_rpc(self): + """Named rpc method is called and results handled""" + + d = { + "no-results" : "data without a results key" + } + self._mock_http(d) + rv = self.sg._call_rpc("no-results", None) + self._assert_http_method("no-results", None) + self.assertEqual(d, rv, + "rpc response without results key is returned as-is") + + d = { + "results" : {"singleton" : "result"} + } + self._mock_http(d) + rv = self.sg._call_rpc("singleton", None) + self._assert_http_method("singleton", None) + self.assertEqual(d["results"], rv, + "rpc response with singleton result") + + d = { + "results" : ["foo", "bar"] + } + a = {"some" : "args"} + self._mock_http(d) + rv = self.sg._call_rpc("list", a) + self._assert_http_method("list", a) + self.assertEqual(d["results"], rv, + "rpc response with list result") + + d = { + "results" : ["foo", "bar"] + } + a = {"some" : "args"} + self._mock_http(d) + rv = self.sg._call_rpc("list-first", a, first=True) + self._assert_http_method("list-first", a) + self.assertEqual(d["results"][0], rv, + "rpc response with list result, first item") + + def test_transform_data(self): + """Outbound data is transformed""" + + timestamp = time.time() + #microseconds will be last during transforms + now = datetime.datetime.fromtimestamp(timestamp).replace( + microsecond=0) + utc_now = datetime.datetime.utcfromtimestamp(timestamp).replace( + microsecond=0) + local = { + "date" : now.strftime('%Y-%m-%d'), + "datetime" : now, + "time" : now.time() + } + #date will still be the local date, because they are not transformed + utc = { + "date" : now.strftime('%Y-%m-%d'), + "datetime": utc_now, + "time" : utc_now.time() + } + + def _datetime(s, f): + return datetime.datetime(*time.strptime(s, f)[:6]) + + def assert_wire(wire, match): + self.assertTrue(isinstance(wire["date"], basestring)) + d = _datetime(wire["date"], "%Y-%m-%d").date() + d = wire['date'] + self.assertEqual(match["date"], d) + self.assertTrue(isinstance(wire["datetime"], basestring)) + d = _datetime(wire["datetime"], "%Y-%m-%dT%H:%M:%SZ") + self.assertEqual(match["datetime"], d) + self.assertTrue(isinstance(wire["time"], basestring)) + d = _datetime(wire["time"], "%Y-%m-%dT%H:%M:%SZ") + self.assertEqual(match["time"], d.time()) + + #leave as local + #AMORTON: tests disabled for now, always have utc over the wire + # self.sg.config.convert_datetimes_to_utc = False + # wire = self.sg._transform_outbound(local) + # print "local ", local + # print "wire ", wire + # assert_wire(wire, local) + # wire = self.sg._transform_inbound(wire) + # #times will become datetime over the wire + # wire["time"] = wire["time"].time() + # self.assertEqual(local, wire) + + self.sg.config.convert_datetimes_to_utc = True + wire = self.sg._transform_outbound(local) + assert_wire(wire, utc) + wire = self.sg._transform_inbound(wire) + #times will become datetime over the wire + wire["time"] = wire["time"].time() + self.assertEqual(local, wire) + return + + def test_encode_payload(self): + """Request body is encoded as JSON""" + + d = { + "this is " : u"my data \u00E0" + } + j = self.sg._encode_payload(d) + self.assertTrue(isinstance(j, str)) + + d = { + "this is " : u"my data" + } + j = self.sg._encode_payload(d) + self.assertTrue(isinstance(j, str)) + + def test_decode_response(self): + """HTTP Response is decoded as JSON or text""" + + headers = { + "content-type" : "application/json;charset=utf-8" + } + d = { + "this is " : u"my data \u00E0" + } + j = json.dumps(d, ensure_ascii=False, encoding="utf-8") + self.assertEqual(d, self.sg._decode_response(headers, j)) + + headers["content-type"] = "text/javascript" + self.assertEqual(d, self.sg._decode_response(headers, j)) + + headers["content-type"] = "text/foo" + self.assertEqual(j, self.sg._decode_response(headers, j)) + + def test_parse_records(self): + """Parse records to replace thumbnail and local paths""" + + system = platform.system().lower() + if system =='darwin': + local_path_field = "local_path_mac" + elif system == 'windows': + local_path_field = "local_path_windows" + elif system == 'linux': + local_path_field = "local_path_linux" + orig = { + "type" : "FakeAsset", + "id" : 1234, + "image" : "blah", + "foo" : { + "link_type" : "local", + local_path_field: "/foo/bar.jpg", + } + } + url = "http://foo/files/0000/0000/0012/232/shot_thumb.jpg" + self.sg._build_thumb_url = mock.Mock( + return_value=url) + + modified, txt = self.sg._parse_records([orig, "plain text"]) + self.assertEqual("plain text", txt, + "non dict value is left as is") + + self.sg._build_thumb_url.assert_called_once_with("FakeAsset", 1234) + + self.assertEqual(url, modified["image"], + "image path changed to url path") + self.assertEqual("/foo/bar.jpg", modified["foo"]["local_path"]) + self.assertEqual("file:///foo/bar.jpg", modified["foo"]["url"]) + + return + + def test_thumb_url(self): + """Thumbnail endpoint used to get thumbnail url""" + + #the thumbnail service returns a two line + #test response success code on line 1, data on line 2 + resp = "1\n/files/0000/0000/0012/232/shot_thumb.jpg" + self._mock_http(resp, headers={"content-type" : "text/plain"}) + self.sg.config.scheme = "http" + self.sg.config.server = "foo.com" + + url = self.sg._build_thumb_url("FakeAsset", 1234) + + self.assertEqual( + "http://foo.com/files/0000/0000/0012/232/shot_thumb.jpg", url) + self.assertTrue(self.sg._http_request.called, + "http request made to get url") + args, _ = self.sg._http_request.call_args + verb, path, body, headers = args + self.assertEqual( + "/upload/get_thumbnail_url?entity_type=FakeAsset&entity_id=1234", + path, "thumbnail url called with correct args") + + resp = "0\nSome Error" + self._mock_http(resp, headers={"content-type" : "text/plain"}) + self.assertRaises(api.ShotgunError, self.sg._build_thumb_url, + "FakeAsset", 456) + + resp = "99\nSome Error" + self._mock_http(resp, headers={"content-type" : "text/plain"}) + self.assertRaises(RuntimeError, self.sg._build_thumb_url, + "FakeAsset", 456) + return + +class TestCreateSummaryRequest(base.TestBase): + '''Test case for _create_summary_request function and parameter + validation as it exists in Shotgun.summarize. + + Does not require database connection or test data.''' + + def test_filter_operator_none(self): + expected_logical_operator = 'and' + filter_operator = None + result = api._create_summary_request('',[],None,filter_operator, None) + actual_logical_operator = result['filters']['logical_operator'] + self.assertEqual(expected_logical_operator, actual_logical_operator) + + def test_filter_operator_all(self): + expected_logical_operator = 'and' + filter_operator = 'all' + result = api._create_summary_request('',[],None,filter_operator, None) + actual_logical_operator = result['filters']['logical_operator'] + self.assertEqual(expected_logical_operator, actual_logical_operator) + + def test_filter_operator_none(self): + expected_logical_operator = 'or' + filter_operator = 'or' + result = api._create_summary_request('',[],None,filter_operator, None) + actual_logical_operator = result['filters']['logical_operator'] + self.assertEqual(expected_logical_operator, actual_logical_operator) + + def test_filters(self): + path = 'path' + relation = 'relation' + value = 'value' + expected_condition = {'path':path, 'relation':relation, 'value':value} + result = api._create_summary_request('', [[path, relation, value]], None, None, None) + actual_condition = result['filters']['conditions'][0] + + def test_grouping(self): + result = api._create_summary_request('', [], None, None, None) + self.assertFalse(result.has_key('grouping')) + grouping = ['something'] + result = api._create_summary_request('', [], None, None, grouping) + self.assertEqual(grouping, result['grouping']) + + def test_filters_type(self): + '''test_filters_type tests that filters parameter is a list''' + self.assertRaises(ValueError, self.sg.summarize, '', 'not a list', []) + + def test_grouping_type(self): + '''test_grouping_type tests that grouping parameter is a list or None''' + self.assertRaises(ValueError, self.sg.summarize, '', [], [], grouping='Not a list') +if __name__ == '__main__': + unittest.main() From 477aa5702255d878329b7b501d9b1a615bb8aea6 Mon Sep 17 00:00:00 2001 From: kennedy behrman Date: Fri, 8 Jul 2011 08:20:04 -0700 Subject: [PATCH 012/570] added summarize method to Shotgun --- shotgun_api3.py | 48 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/shotgun_api3.py b/shotgun_api3.py index 4943ca956..f4cf51214 100755 --- a/shotgun_api3.py +++ b/shotgun_api3.py @@ -122,6 +122,7 @@ def _ensure_json_supported(self): raise ShotgunError("JSON API requires server version 2.4 or "\ "higher, server is %s" % (self.version,)) return + def _is_paging(self, version): """Determines if the server version supports paging. @@ -222,7 +223,6 @@ def __init__(self, base_url, script_name, api_key, :param http_proxy: Optional, URL for the http proxy server, of the form http://proxy.com:8080 """ - self.config = _Config() self.config.api_key = api_key self.config.script_name = script_name @@ -365,10 +365,7 @@ def find(self, entity_type, filters, fields=None, order=None, else: new_filters["logical_operator"] = "or" - new_filters["conditions"] = [ - {"path" : f[0], "relation" : f[1], "values" : f[2:]} - for f in filters - ] + new_filters["conditions"] = [{"path":f[0], "relation":f[1], "values":f[2:]} for f in filters ] filters = new_filters elif filter_operator: @@ -436,6 +433,21 @@ def find(self, entity_type, filters, fields=None, order=None, return self._parse_records(records) + def summarize(self, entity_type, filters, summary_fields, filter_operator=None, grouping=None): + """ + Return group and summary information for entity_type for summary_fields + based on the given filters. + """ + if not isinstance(filters, list): + raise ValueError("summarize() 'filters' parameter must be a list") + + if not isinstance(grouping, list) and grouping != None: + raise ValueError("summarize() 'grouping' parameter must be a list or None") + + params = _create_summary_request(entity_type, filters, summary_fields, filter_operator, grouping) + records = self._call_rpc('summarize', params) + return records + def create(self, entity_type, data, return_fields=None): """Create a new entity of the specified entity_type. @@ -3119,3 +3131,29 @@ def connect(self,destpair): _orgsocket.connect(self,(destpair[0],destpair[1])) else: raise GeneralProxyError((4,_generalerrors[4])) + + + +def _create_summary_request(entity_type, filters, summary_fields, filter_operator, grouping): + '''_create_summary_request assembles a request based on input''' + new_filters = {} + if not filter_operator or filter_operator == "all": + new_filters["logical_operator"] = "and" + else: + new_filters["logical_operator"] = "or" + + new_filters["conditions"] = [] + for f in filters: + new_filters["conditions"].append( {"path":f[0],"relation":f[1],"values":f[2:]} ) + + filters = new_filters + + req = { + "type": entity_type, + "summaries": summary_fields, + "filters": filters, + } + if grouping != None: + req['grouping'] = grouping + + return req From b08d8d41bb5aac22c8dbb8cd64d32df5f6102662 Mon Sep 17 00:00:00 2001 From: Isaac Reuben Date: Mon, 11 Jul 2011 11:14:48 -0700 Subject: [PATCH 013/570] Ignore pyc (and a few other) files --- .gitignore | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..fb185af7e --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +#python specific +*.pyc + +## generic files to ignore +*~ +*.lock +*.DS_Store +*.swp +*.out +*.bak \ No newline at end of file From c763f39781811edba198756442dd1ac39a7fa6fd Mon Sep 17 00:00:00 2001 From: kennedy behrman Date: Tue, 19 Jul 2011 14:48:07 -0700 Subject: [PATCH 014/570] seperated tests using mock from those using server connection; is_paging now allows true; added unit tests --- run-tests | 2 +- shotgun_api3.py | 479 ++--------------------------------------- socks.py | 396 ++++++++++++++++++++++++++++++++++ tests/base.py | 175 +++++++-------- tests/test_api.py | 142 ++---------- tests/test_api_long.py | 44 ++-- tests/test_client.py | 204 +++++------------- tests/tests_unit.py | 214 ++++++++++++++++++ 8 files changed, 805 insertions(+), 851 deletions(-) create mode 100644 socks.py create mode 100644 tests/tests_unit.py diff --git a/run-tests b/run-tests index 6ce09db3d..045d6c199 100755 --- a/run-tests +++ b/run-tests @@ -1 +1 @@ -clear && find ./ -name ".coverage" -delete && find ./ -name "*.pyc" -delete && nosetests -vd --stop --with-cover --cover-package=shotgun_json \ No newline at end of file +clear && find ./ -name ".coverage" -delete && find ./ -name "*.pyc" -delete && nosetests -vd --with-cover --cover-package=shotgun_api3 diff --git a/shotgun_api3.py b/shotgun_api3.py index f4cf51214..d4e08a183 100755 --- a/shotgun_api3.py +++ b/shotgun_api3.py @@ -47,6 +47,7 @@ import urllib import urllib2 # used for image upload import urlparse +from socks import * log = logging.getLogger("shotgun_api3") @@ -84,32 +85,24 @@ def __init__(self, host, meta): :param meta: dict of meta data for the server returned from the info api method. """ - #Server host name self.host = host - try: - meta = dict(meta) - except ValueError: - meta = {} self.server_info = meta #Version from server is major.minor.rev or major.minor.rev."Dev" #Store version as triple and check dev flag self.version = meta.get("version", None) if not self.version: - self.version = (0,0,0) - self.is_dev = False + raise ShotgunError("Server version not specified") + + if len(self.version) > 3 and self.version[3] == "Dev": + self.is_dev = True else: - if len(self.version) > 3 and self.version[3] == "Dev": - self.is_dev = True - else: - self.is_dev = False - self.version = tuple(self.version[:3]) - + self.is_dev = False + + self.version = tuple(self.version[:3]) self._ensure_json_supported() - #Flag if this server supports paging - self.has_paging = self._is_paging(self.version) def _ensure_json_supported(self): """Checks the server version supports the JSON api, raises an @@ -117,27 +110,14 @@ def _ensure_json_supported(self): :raises ShotgunError: The current server version does not support json """ - if not self.version or self.version < (2,4,0): raise ShotgunError("JSON API requires server version 2.4 or "\ "higher, server is %s" % (self.version,)) - return - def _is_paging(self, version): - """Determines if the server version supports paging. - - :param version: Version to check. - - :returns: True if the server supports paging, False otherwise. - """ - if version >= (2, 3, 4): - return True - return False def __str__(self): - return "ServerCapabilities: host %s, version %s, is_dev %s, "\ - "has_paging %s" % (self.host, self.version, self.is_dev, - self.has_paging) + return "ServerCapabilities: host %s, version %s, is_dev %s"\ + % (self.host, self.version, self.is_dev) class ClientCapabilities(object): @@ -203,7 +183,7 @@ class Shotgun(object): "(\D?([01]\d|2[0-3])\D?([0-5]\d)\D?([0-5]\d)?\D?(\d{3})?)?$") def __init__(self, base_url, script_name, api_key, - convert_datetimes_to_utc=True, http_proxy=None): + convert_datetimes_to_utc=True, http_proxy=None, connect=True): """Initialises a new instance of the Shotgun client. :param base_url: http or https url to the shotgun server. @@ -222,6 +202,8 @@ def __init__(self, base_url, script_name, api_key, :param http_proxy: Optional, URL for the http proxy server, of the form http://proxy.com:8080 + + :param connect: If True, connect to the server. Only used for testing. """ self.config = _Config() self.config.api_key = api_key @@ -258,7 +240,8 @@ def __init__(self, base_url, script_name, api_key, self._server_caps = None #test to ensure the the server supports the json API #call to server will only be made once and will raise error - sc = self.server_caps + if connect: + sc = self.server_caps # ======================================================================== # API Functions @@ -303,7 +286,6 @@ def info(self): :returns: dict of the server meta data. """ - return self._call_rpc("info", None, include_script_name=False) def find_one(self, entity_type, filters, fields=None, order=None, @@ -378,14 +360,13 @@ def find(self, entity_type, filters, fields=None, order=None, "return_fields": fields or ["id"], "filters": filters, "return_only" : (retired_only and 'retired') or "active", + "return_paging_info" : True, "paging": { "entities_per_page": self.config.records_per_page, "current_page": 1 } } - if self.server_caps.has_paging: - params["return_paging_info"] = True if order: sort_list = [] @@ -410,9 +391,7 @@ def find(self, entity_type, filters, fields=None, order=None, # if page is specified, then only return the page of records requested if page != 0: # No paging_info needed, so optimize it out. - if self.server_caps.has_paging: - params["return_paging_info"] = False - + params["return_paging_info"] = False params["paging"]["current_page"] = page records = self._call_rpc("read", params).get("entities", []) return self._parse_records(records) @@ -879,8 +858,7 @@ def entity_types(self): # ======================================================================== # RPC Functions - def _call_rpc(self, method, params, include_script_name=True, - first=False): + def _call_rpc(self, method, params, include_script_name=True, first=False): """Calls the specified method on the Shotgun Server sending the supplied payload. @@ -1547,11 +1525,6 @@ def iri2uri(uri): from gettext import gettext as _ import socket -# HACK: amorton sock module is included in this file now -# try: -# import socks -# except ImportError: -# socks = None # Build the appropriate socket wrapper for ssl try: @@ -1585,6 +1558,7 @@ def has_timeout(timeout): # python 2.6 # The httplib debug level, set to a non-zero value to get debug output debuglevel = 0 +#TODO do we still support python 2.3? # Python 2.3 support if sys.version_info < (2,4): @@ -2225,14 +2199,13 @@ def connect(self): try: if self.proxy_info and self.proxy_info.isgood(): # HACK: amorton socks module is included in the file - # self.sock = socks.socksocket(af, socktype, proto) self.sock = socksocket(af, socktype, proto) - # HACK: amorton enabled TCP_NODLEAY on socket + # HACK: amorton enabled TCP_NODELAY on socket self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) self.sock.setproxy(*self.proxy_info.astuple()) else: self.sock = socket.socket(af, socktype, proto) - # HACK: amorton enabled TCP_NODLEAY on socket + # HACK: amorton enabled TCP_NODELAY on socket self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) # Different from httplib: support timeouts. if has_timeout(self.timeout): @@ -2725,417 +2698,9 @@ def __getattr__(self, name): raise AttributeError, name - - - - - - - - - - - - - - - - -# ---------------------------------------------------------------------------- -# SocksiPy/socks.py - -"""SocksiPy - Python SOCKS module. -Version 1.00 - -Copyright 2006 Dan-Haim. All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. -3. Neither the name of Dan Haim nor the names of his contributors may be used - to endorse or promote products derived from this software without specific - prior written permission. - -THIS SOFTWARE IS PROVIDED BY DAN HAIM "AS IS" AND ANY EXPRESS OR IMPLIED -WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO -EVENT SHALL DAN HAIM OR HIS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA -OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMANGE. - - -This module provides a standard socket-like interface for Python -for tunneling connections through SOCKS proxies. - -""" - -import socket -import struct - -PROXY_TYPE_SOCKS4 = 1 -PROXY_TYPE_SOCKS5 = 2 -PROXY_TYPE_HTTP = 3 - -_defaultproxy = None -_orgsocket = socket.socket - -class ProxyError(Exception): - def __init__(self, value): - self.value = value - def __str__(self): - return repr(self.value) - -class GeneralProxyError(ProxyError): - def __init__(self, value): - self.value = value - def __str__(self): - return repr(self.value) - -class Socks5AuthError(ProxyError): - def __init__(self, value): - self.value = value - def __str__(self): - return repr(self.value) - -class Socks5Error(ProxyError): - def __init__(self, value): - self.value = value - def __str__(self): - return repr(self.value) - -class Socks4Error(ProxyError): - def __init__(self, value): - self.value = value - def __str__(self): - return repr(self.value) - -class HTTPError(ProxyError): - def __init__(self, value): - self.value = value - def __str__(self): - return repr(self.value) - -_generalerrors = ("success", - "invalid data", - "not connected", - "not available", - "bad proxy type", - "bad input") - -_socks5errors = ("succeeded", - "general SOCKS server failure", - "connection not allowed by ruleset", - "Network unreachable", - "Host unreachable", - "Connection refused", - "TTL expired", - "Command not supported", - "Address type not supported", - "Unknown error") - -_socks5autherrors = ("succeeded", - "authentication is required", - "all offered authentication methods were rejected", - "unknown username or invalid password", - "unknown error") - -_socks4errors = ("request granted", - "request rejected or failed", - "request rejected because SOCKS server cannot connect to identd on the client", - "request rejected because the client program and identd report different user-ids", - "unknown error") - -def setdefaultproxy(proxytype=None,addr=None,port=None,rdns=True,username=None,password=None): - """setdefaultproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) - Sets a default proxy which all further socksocket objects will use, - unless explicitly changed. - """ - global _defaultproxy - _defaultproxy = (proxytype,addr,port,rdns,username,password) - -class socksocket(socket.socket): - """socksocket([family[, type[, proto]]]) -> socket object - - Open a SOCKS enabled socket. The parameters are the same as - those of the standard socket init. In order for SOCKS to work, - you must specify family=AF_INET, type=SOCK_STREAM and proto=0. - """ - - def __init__(self, family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0, _sock=None): - _orgsocket.__init__(self,family,type,proto,_sock) - if _defaultproxy != None: - self.__proxy = _defaultproxy - else: - self.__proxy = (None, None, None, None, None, None) - self.__proxysockname = None - self.__proxypeername = None - - def __recvall(self, bytes): - """__recvall(bytes) -> data - Receive EXACTLY the number of bytes requested from the socket. - Blocks until the required number of bytes have been received. - """ - data = "" - while len(data) < bytes: - data = data + self.recv(bytes-len(data)) - return data - - def setproxy(self,proxytype=None,addr=None,port=None,rdns=True,username=None,password=None): - """setproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) - Sets the proxy to be used. - proxytype - The type of the proxy to be used. Three types - are supported: PROXY_TYPE_SOCKS4 (including socks4a), - PROXY_TYPE_SOCKS5 and PROXY_TYPE_HTTP - addr - The address of the server (IP or DNS). - port - The port of the server. Defaults to 1080 for SOCKS - servers and 8080 for HTTP proxy servers. - rdns - Should DNS queries be preformed on the remote side - (rather than the local side). The default is True. - Note: This has no effect with SOCKS4 servers. - username - Username to authenticate with to the server. - The default is no authentication. - password - Password to authenticate with to the server. - Only relevant when username is also provided. - """ - self.__proxy = (proxytype,addr,port,rdns,username,password) - - def __negotiatesocks5(self,destaddr,destport): - """__negotiatesocks5(self,destaddr,destport) - Negotiates a connection through a SOCKS5 server. - """ - # First we'll send the authentication packages we support. - if (self.__proxy[4]!=None) and (self.__proxy[5]!=None): - # The username/password details were supplied to the - # setproxy method so we support the USERNAME/PASSWORD - # authentication (in addition to the standard none). - self.sendall("\x05\x02\x00\x02") - else: - # No username/password were entered, therefore we - # only support connections with no authentication. - self.sendall("\x05\x01\x00") - # We'll receive the server's response to determine which - # method was selected - chosenauth = self.__recvall(2) - if chosenauth[0] != "\x05": - self.close() - raise GeneralProxyError((1,_generalerrors[1])) - # Check the chosen authentication method - if chosenauth[1] == "\x00": - # No authentication is required - pass - elif chosenauth[1] == "\x02": - # Okay, we need to perform a basic username/password - # authentication. - self.sendall("\x01" + chr(len(self.__proxy[4])) + self.__proxy[4] + chr(len(self.proxy[5])) + self.__proxy[5]) - authstat = self.__recvall(2) - if authstat[0] != "\x01": - # Bad response - self.close() - raise GeneralProxyError((1,_generalerrors[1])) - if authstat[1] != "\x00": - # Authentication failed - self.close() - raise Socks5AuthError,((3,_socks5autherrors[3])) - # Authentication succeeded - else: - # Reaching here is always bad - self.close() - if chosenauth[1] == "\xFF": - raise Socks5AuthError((2,_socks5autherrors[2])) - else: - raise GeneralProxyError((1,_generalerrors[1])) - # Now we can request the actual connection - req = "\x05\x01\x00" - # If the given destination address is an IP address, we'll - # use the IPv4 address request even if remote resolving was specified. - try: - ipaddr = socket.inet_aton(destaddr) - req = req + "\x01" + ipaddr - except socket.error: - # Well it's not an IP number, so it's probably a DNS name. - if self.__proxy[3]==True: - # Resolve remotely - ipaddr = None - req = req + "\x03" + chr(len(destaddr)) + destaddr - else: - # Resolve locally - ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) - req = req + "\x01" + ipaddr - req = req + struct.pack(">H",destport) - self.sendall(req) - # Get the response - resp = self.__recvall(4) - if resp[0] != "\x05": - self.close() - raise GeneralProxyError((1,_generalerrors[1])) - elif resp[1] != "\x00": - # Connection failed - self.close() - if ord(resp[1])<=8: - raise Socks5Error(ord(resp[1]),_generalerrors[ord(resp[1])]) - else: - raise Socks5Error(9,_generalerrors[9]) - # Get the bound address/port - elif resp[3] == "\x01": - boundaddr = self.__recvall(4) - elif resp[3] == "\x03": - resp = resp + self.recv(1) - boundaddr = self.__recvall(resp[4]) - else: - self.close() - raise GeneralProxyError((1,_generalerrors[1])) - boundport = struct.unpack(">H",self.__recvall(2))[0] - self.__proxysockname = (boundaddr,boundport) - if ipaddr != None: - self.__proxypeername = (socket.inet_ntoa(ipaddr),destport) - else: - self.__proxypeername = (destaddr,destport) - - def getproxysockname(self): - """getsockname() -> address info - Returns the bound IP address and port number at the proxy. - """ - return self.__proxysockname - - def getproxypeername(self): - """getproxypeername() -> address info - Returns the IP and port number of the proxy. - """ - return _orgsocket.getpeername(self) - - def getpeername(self): - """getpeername() -> address info - Returns the IP address and port number of the destination - machine (note: getproxypeername returns the proxy) - """ - return self.__proxypeername - - def __negotiatesocks4(self,destaddr,destport): - """__negotiatesocks4(self,destaddr,destport) - Negotiates a connection through a SOCKS4 server. - """ - # Check if the destination address provided is an IP address - rmtrslv = False - try: - ipaddr = socket.inet_aton(destaddr) - except socket.error: - # It's a DNS name. Check where it should be resolved. - if self.__proxy[3]==True: - ipaddr = "\x00\x00\x00\x01" - rmtrslv = True - else: - ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) - # Construct the request packet - req = "\x04\x01" + struct.pack(">H",destport) + ipaddr - # The username parameter is considered userid for SOCKS4 - if self.__proxy[4] != None: - req = req + self.__proxy[4] - req = req + "\x00" - # DNS name if remote resolving is required - # NOTE: This is actually an extension to the SOCKS4 protocol - # called SOCKS4A and may not be supported in all cases. - if rmtrslv==True: - req = req + destaddr + "\x00" - self.sendall(req) - # Get the response from the server - resp = self.__recvall(8) - if resp[0] != "\x00": - # Bad data - self.close() - raise GeneralProxyError((1,_generalerrors[1])) - if resp[1] != "\x5A": - # Server returned an error - self.close() - if ord(resp[1]) in (91,92,93): - self.close() - raise Socks4Error((ord(resp[1]),_socks4errors[ord(resp[1])-90])) - else: - raise Socks4Error((94,_socks4errors[4])) - # Get the bound address/port - self.__proxysockname = (socket.inet_ntoa(resp[4:]),struct.unpack(">H",resp[2:4])[0]) - if rmtrslv != None: - self.__proxypeername = (socket.inet_ntoa(ipaddr),destport) - else: - self.__proxypeername = (destaddr,destport) - - def __negotiatehttp(self,destaddr,destport): - """__negotiatehttp(self,destaddr,destport) - Negotiates a connection through an HTTP server. - """ - # If we need to resolve locally, we do this now - if self.__proxy[3] == False: - addr = socket.gethostbyname(destaddr) - else: - addr = destaddr - self.sendall("CONNECT " + addr + ":" + str(destport) + " HTTP/1.1\r\n" + "Host: " + destaddr + "\r\n\r\n") - # We read the response until we get the string "\r\n\r\n" - resp = self.recv(1) - while resp.find("\r\n\r\n")==-1: - resp = resp + self.recv(1) - # We just need the first line to check if the connection - # was successful - statusline = resp.splitlines()[0].split(" ",2) - if statusline[0] not in ("HTTP/1.0","HTTP/1.1"): - self.close() - raise GeneralProxyError((1,_generalerrors[1])) - try: - statuscode = int(statusline[1]) - except ValueError: - self.close() - raise GeneralProxyError((1,_generalerrors[1])) - if statuscode != 200: - self.close() - raise HTTPError((statuscode,statusline[2])) - self.__proxysockname = ("0.0.0.0",0) - self.__proxypeername = (addr,destport) - - def connect(self,destpair): - """connect(self,despair) - Connects to the specified destination through a proxy. - destpar - A tuple of the IP/DNS address and the port number. - (identical to socket's connect). - To select the proxy server use setproxy(). - """ - # Do a minimal input check first - if (type(destpair) in (list,tuple)==False) or (len(destpair)<2) or (type(destpair[0])!=str) or (type(destpair[1])!=int): - raise GeneralProxyError((5,_generalerrors[5])) - if self.__proxy[0] == PROXY_TYPE_SOCKS5: - if self.__proxy[2] != None: - portnum = self.__proxy[2] - else: - portnum = 1080 - _orgsocket.connect(self,(self.__proxy[1],portnum)) - self.__negotiatesocks5(destpair[0],destpair[1]) - elif self.__proxy[0] == PROXY_TYPE_SOCKS4: - if self.__proxy[2] != None: - portnum = self.__proxy[2] - else: - portnum = 1080 - _orgsocket.connect(self,(self.__proxy[1],portnum)) - self.__negotiatesocks4(destpair[0],destpair[1]) - elif self.__proxy[0] == PROXY_TYPE_HTTP: - if self.__proxy[2] != None: - portnum = self.__proxy[2] - else: - portnum = 8080 - _orgsocket.connect(self,(self.__proxy[1],portnum)) - self.__negotiatehttp(destpair[0],destpair[1]) - elif self.__proxy[0] == None: - _orgsocket.connect(self,(destpair[0],destpair[1])) - else: - raise GeneralProxyError((4,_generalerrors[4])) - - - def _create_summary_request(entity_type, filters, summary_fields, filter_operator, grouping): '''_create_summary_request assembles a request based on input''' + #TODO make this part of summary method? new_filters = {} if not filter_operator or filter_operator == "all": new_filters["logical_operator"] = "and" diff --git a/socks.py b/socks.py new file mode 100644 index 000000000..098a1f8a9 --- /dev/null +++ b/socks.py @@ -0,0 +1,396 @@ +#! /opt/local/bin/python +# ---------------------------------------------------------------------------- +# SocksiPy/socks.py + +"""SocksiPy - Python SOCKS module. +Version 1.00 + +Copyright 2006 Dan-Haim. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +3. Neither the name of Dan Haim nor the names of his contributors may be used + to endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY DAN HAIM "AS IS" AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL DAN HAIM OR HIS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMANGE. + + +This module provides a standard socket-like interface for Python +for tunneling connections through SOCKS proxies. + +""" + +import socket +import struct + +PROXY_TYPE_SOCKS4 = 1 +PROXY_TYPE_SOCKS5 = 2 +PROXY_TYPE_HTTP = 3 + +_defaultproxy = None +_orgsocket = socket.socket + +class ProxyError(Exception): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +class GeneralProxyError(ProxyError): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +class Socks5AuthError(ProxyError): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +class Socks5Error(ProxyError): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +class Socks4Error(ProxyError): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +class HTTPError(ProxyError): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) + +_generalerrors = ("success", + "invalid data", + "not connected", + "not available", + "bad proxy type", + "bad input") + +_socks5errors = ("succeeded", + "general SOCKS server failure", + "connection not allowed by ruleset", + "Network unreachable", + "Host unreachable", + "Connection refused", + "TTL expired", + "Command not supported", + "Address type not supported", + "Unknown error") + +_socks5autherrors = ("succeeded", + "authentication is required", + "all offered authentication methods were rejected", + "unknown username or invalid password", + "unknown error") + +_socks4errors = ("request granted", + "request rejected or failed", + "request rejected because SOCKS server cannot connect to identd on the client", + "request rejected because the client program and identd report different user-ids", + "unknown error") + +def setdefaultproxy(proxytype=None,addr=None,port=None,rdns=True,username=None,password=None): + """setdefaultproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) + Sets a default proxy which all further socksocket objects will use, + unless explicitly changed. + """ + global _defaultproxy + _defaultproxy = (proxytype,addr,port,rdns,username,password) + +class socksocket(socket.socket): + """socksocket([family[, type[, proto]]]) -> socket object + + Open a SOCKS enabled socket. The parameters are the same as + those of the standard socket init. In order for SOCKS to work, + you must specify family=AF_INET, type=SOCK_STREAM and proto=0. + """ + + def __init__(self, family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0, _sock=None): + _orgsocket.__init__(self,family,type,proto,_sock) + if _defaultproxy != None: + self.__proxy = _defaultproxy + else: + self.__proxy = (None, None, None, None, None, None) + self.__proxysockname = None + self.__proxypeername = None + + def __recvall(self, bytes): + """__recvall(bytes) -> data + Receive EXACTLY the number of bytes requested from the socket. + Blocks until the required number of bytes have been received. + """ + data = "" + while len(data) < bytes: + data = data + self.recv(bytes-len(data)) + return data + + def setproxy(self,proxytype=None,addr=None,port=None,rdns=True,username=None,password=None): + """setproxy(proxytype, addr[, port[, rdns[, username[, password]]]]) + Sets the proxy to be used. + proxytype - The type of the proxy to be used. Three types + are supported: PROXY_TYPE_SOCKS4 (including socks4a), + PROXY_TYPE_SOCKS5 and PROXY_TYPE_HTTP + addr - The address of the server (IP or DNS). + port - The port of the server. Defaults to 1080 for SOCKS + servers and 8080 for HTTP proxy servers. + rdns - Should DNS queries be preformed on the remote side + (rather than the local side). The default is True. + Note: This has no effect with SOCKS4 servers. + username - Username to authenticate with to the server. + The default is no authentication. + password - Password to authenticate with to the server. + Only relevant when username is also provided. + """ + self.__proxy = (proxytype,addr,port,rdns,username,password) + + def __negotiatesocks5(self,destaddr,destport): + """__negotiatesocks5(self,destaddr,destport) + Negotiates a connection through a SOCKS5 server. + """ + # First we'll send the authentication packages we support. + if (self.__proxy[4]!=None) and (self.__proxy[5]!=None): + # The username/password details were supplied to the + # setproxy method so we support the USERNAME/PASSWORD + # authentication (in addition to the standard none). + self.sendall("\x05\x02\x00\x02") + else: + # No username/password were entered, therefore we + # only support connections with no authentication. + self.sendall("\x05\x01\x00") + # We'll receive the server's response to determine which + # method was selected + chosenauth = self.__recvall(2) + if chosenauth[0] != "\x05": + self.close() + raise GeneralProxyError((1,_generalerrors[1])) + # Check the chosen authentication method + if chosenauth[1] == "\x00": + # No authentication is required + pass + elif chosenauth[1] == "\x02": + # Okay, we need to perform a basic username/password + # authentication. + self.sendall("\x01" + chr(len(self.__proxy[4])) + self.__proxy[4] + chr(len(self.proxy[5])) + self.__proxy[5]) + authstat = self.__recvall(2) + if authstat[0] != "\x01": + # Bad response + self.close() + raise GeneralProxyError((1,_generalerrors[1])) + if authstat[1] != "\x00": + # Authentication failed + self.close() + raise Socks5AuthError,((3,_socks5autherrors[3])) + # Authentication succeeded + else: + # Reaching here is always bad + self.close() + if chosenauth[1] == "\xFF": + raise Socks5AuthError((2,_socks5autherrors[2])) + else: + raise GeneralProxyError((1,_generalerrors[1])) + # Now we can request the actual connection + req = "\x05\x01\x00" + # If the given destination address is an IP address, we'll + # use the IPv4 address request even if remote resolving was specified. + try: + ipaddr = socket.inet_aton(destaddr) + req = req + "\x01" + ipaddr + except socket.error: + # Well it's not an IP number, so it's probably a DNS name. + if self.__proxy[3]==True: + # Resolve remotely + ipaddr = None + req = req + "\x03" + chr(len(destaddr)) + destaddr + else: + # Resolve locally + ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) + req = req + "\x01" + ipaddr + req = req + struct.pack(">H",destport) + self.sendall(req) + # Get the response + resp = self.__recvall(4) + if resp[0] != "\x05": + self.close() + raise GeneralProxyError((1,_generalerrors[1])) + elif resp[1] != "\x00": + # Connection failed + self.close() + if ord(resp[1])<=8: + raise Socks5Error(ord(resp[1]),_generalerrors[ord(resp[1])]) + else: + raise Socks5Error(9,_generalerrors[9]) + # Get the bound address/port + elif resp[3] == "\x01": + boundaddr = self.__recvall(4) + elif resp[3] == "\x03": + resp = resp + self.recv(1) + boundaddr = self.__recvall(resp[4]) + else: + self.close() + raise GeneralProxyError((1,_generalerrors[1])) + boundport = struct.unpack(">H",self.__recvall(2))[0] + self.__proxysockname = (boundaddr,boundport) + if ipaddr != None: + self.__proxypeername = (socket.inet_ntoa(ipaddr),destport) + else: + self.__proxypeername = (destaddr,destport) + + def getproxysockname(self): + """getsockname() -> address info + Returns the bound IP address and port number at the proxy. + """ + return self.__proxysockname + + def getproxypeername(self): + """getproxypeername() -> address info + Returns the IP and port number of the proxy. + """ + return _orgsocket.getpeername(self) + + def getpeername(self): + """getpeername() -> address info + Returns the IP address and port number of the destination + machine (note: getproxypeername returns the proxy) + """ + return self.__proxypeername + + def __negotiatesocks4(self,destaddr,destport): + """__negotiatesocks4(self,destaddr,destport) + Negotiates a connection through a SOCKS4 server. + """ + # Check if the destination address provided is an IP address + rmtrslv = False + try: + ipaddr = socket.inet_aton(destaddr) + except socket.error: + # It's a DNS name. Check where it should be resolved. + if self.__proxy[3]==True: + ipaddr = "\x00\x00\x00\x01" + rmtrslv = True + else: + ipaddr = socket.inet_aton(socket.gethostbyname(destaddr)) + # Construct the request packet + req = "\x04\x01" + struct.pack(">H",destport) + ipaddr + # The username parameter is considered userid for SOCKS4 + if self.__proxy[4] != None: + req = req + self.__proxy[4] + req = req + "\x00" + # DNS name if remote resolving is required + # NOTE: This is actually an extension to the SOCKS4 protocol + # called SOCKS4A and may not be supported in all cases. + if rmtrslv==True: + req = req + destaddr + "\x00" + self.sendall(req) + # Get the response from the server + resp = self.__recvall(8) + if resp[0] != "\x00": + # Bad data + self.close() + raise GeneralProxyError((1,_generalerrors[1])) + if resp[1] != "\x5A": + # Server returned an error + self.close() + if ord(resp[1]) in (91,92,93): + self.close() + raise Socks4Error((ord(resp[1]),_socks4errors[ord(resp[1])-90])) + else: + raise Socks4Error((94,_socks4errors[4])) + # Get the bound address/port + self.__proxysockname = (socket.inet_ntoa(resp[4:]),struct.unpack(">H",resp[2:4])[0]) + if rmtrslv != None: + self.__proxypeername = (socket.inet_ntoa(ipaddr),destport) + else: + self.__proxypeername = (destaddr,destport) + + def __negotiatehttp(self,destaddr,destport): + """__negotiatehttp(self,destaddr,destport) + Negotiates a connection through an HTTP server. + """ + # If we need to resolve locally, we do this now + if self.__proxy[3] == False: + addr = socket.gethostbyname(destaddr) + else: + addr = destaddr + self.sendall("CONNECT " + addr + ":" + str(destport) + " HTTP/1.1\r\n" + "Host: " + destaddr + "\r\n\r\n") + # We read the response until we get the string "\r\n\r\n" + resp = self.recv(1) + while resp.find("\r\n\r\n")==-1: + resp = resp + self.recv(1) + # We just need the first line to check if the connection + # was successful + statusline = resp.splitlines()[0].split(" ",2) + if statusline[0] not in ("HTTP/1.0","HTTP/1.1"): + self.close() + raise GeneralProxyError((1,_generalerrors[1])) + try: + statuscode = int(statusline[1]) + except ValueError: + self.close() + raise GeneralProxyError((1,_generalerrors[1])) + if statuscode != 200: + self.close() + raise HTTPError((statuscode,statusline[2])) + self.__proxysockname = ("0.0.0.0",0) + self.__proxypeername = (addr,destport) + + def connect(self,destpair): + """connect(self, destpair) + Connects to the specified destination through a proxy. + destpar - A tuple of the IP/DNS address and the port number. + (identical to socket's connect). + To select the proxy server use setproxy(). + """ + # Do a minimal input check first + if not (isinstance(destpair, (list, tuple)) and (len(destpair) > 1) and isinstance(destpair[0], str) and isinstance(destpair[1], int)): + raise GeneralProxyError((5,_generalerrors[5])) + + if (type(destpair) in (list,tuple)==False) or (len(destpair)<2) or (type(destpair[0])!=str) or (type(destpair[1])!=int): + raise GeneralProxyError((5,_generalerrors[5])) + if self.__proxy[0] == PROXY_TYPE_SOCKS5: + if self.__proxy[2] != None: + portnum = self.__proxy[2] + else: + portnum = 1080 + _orgsocket.connect(self,(self.__proxy[1],portnum)) + self.__negotiatesocks5(destpair[0],destpair[1]) + elif self.__proxy[0] == PROXY_TYPE_SOCKS4: + if self.__proxy[2] != None: + portnum = self.__proxy[2] + else: + portnum = 1080 + _orgsocket.connect(self,(self.__proxy[1],portnum)) + self.__negotiatesocks4(destpair[0],destpair[1]) + elif self.__proxy[0] == PROXY_TYPE_HTTP: + if self.__proxy[2] != None: + portnum = self.__proxy[2] + else: + portnum = 8080 + _orgsocket.connect(self,(self.__proxy[1],portnum)) + self.__negotiatehttp(destpair[0],destpair[1]) + elif self.__proxy[0] == None: + _orgsocket.connect(self,(destpair[0],destpair[1])) + else: + raise GeneralProxyError((4,_generalerrors[4])) + + diff --git a/tests/base.py b/tests/base.py index da3b652c8..2c6ce6955 100644 --- a/tests/base.py +++ b/tests/base.py @@ -18,7 +18,6 @@ class TestBase(unittest.TestCase): Sets up mocking and database test data.''' def __init__(self, *args, **kws): unittest.TestCase.__init__(self, *args, **kws) - self.is_mock = False self.human_user = None self.project = None self.shot = None @@ -26,42 +25,50 @@ def __init__(self, *args, **kws): self.version = None self.human_password = None self.server_url = None + self.connect = False def setUp(self): - config = SgTestConfig() - config.read_config(CONFIG_PATH) - self.human_password = config.human_password - self.server_url = config.server_url - self.script_name = config.script_name - self.api_key = config.api_key - self.http_proxy = config.http_proxy - self.session_uuid = config.session_uuid - - self.sg = api.Shotgun(config.server_url, config.script_name, - config.api_key, http_proxy=config.http_proxy) - - if config.session_uuid: - self.sg.set_session_uuid(config.session_uuid) - - if config.mock: - self._setup_mock() - self._setup_mock_data(config) - else: - self._setup_db(config) + self.config = SgTestConfig() + self.config.read_config(CONFIG_PATH) + self.human_password = self.config.human_password + self.server_url = self.config.server_url + self.script_name = self.config.script_name + self.api_key = self.config.api_key + self.http_proxy = self.config.http_proxy + self.session_uuid = self.config.session_uuid + + + self.sg = api.Shotgun(self.config.server_url, + self.config.script_name, + self.config.api_key, + http_proxy=self.config.http_proxy, + connect=self.connect) + + if self.config.session_uuid: + self.sg.set_session_uuid(self.config.session_uuid) def tearDown(self): self.sg = None - return + +class MockTestBase(TestBase): + '''Test base for tests mocking server interactions.''' + def setUp(self): + super(MockTestBase, self).setUp() + #TODO see if there is another way to stop sg connecting + self._setup_mock() + self._setup_mock_data() + + def _setup_mock(self): """Setup mocking on the ShotgunClient to stop it calling a live server """ #Replace the function used to make the final call to the server #eaiser than mocking the http connection + response self.sg._http_request = mock.Mock(spec=api.Shotgun._http_request, - return_value=((200, "OK"), {}, None)) + return_value=((200, "OK"), {}, None)) #also replace the function that is called to get the http connection #to avoid calling the server. OK to return a mock as we will not use @@ -75,9 +82,8 @@ def _setup_mock(self): #create the server caps directly to say we have the correct version self.sg._server_caps = api.ServerCapabilities(self.sg.config.server, - {"version" : [2,4,0]}) - self.is_mock = True - return + {"version" : [2,4,0]}) + def _mock_http(self, data, headers=None, status=None): """Setup a mock response from the SG server. @@ -92,15 +98,13 @@ def _mock_http(self, data, headers=None, status=None): if not isinstance(data, basestring): data = json.dumps(data, ensure_ascii=False, encoding="utf-8") - resp_headers = { - 'cache-control': 'no-cache', - 'connection': 'close', - 'content-length': (data and str(len(data))) or 0 , - 'content-type': 'application/json; charset=utf-8', - 'date': 'Wed, 13 Apr 2011 04:18:58 GMT', - 'server': 'Apache/2.2.3 (CentOS)', - 'status': '200 OK' - } + resp_headers = { 'cache-control': 'no-cache', + 'connection': 'close', + 'content-length': (data and str(len(data))) or 0 , + 'content-type': 'application/json; charset=utf-8', + 'date': 'Wed, 13 Apr 2011 04:18:58 GMT', + 'server': 'Apache/2.2.3 (CentOS)', + 'status': '200 OK' } if headers: resp_headers.update(headers) @@ -110,12 +114,9 @@ def _mock_http(self, data, headers=None, status=None): self._setup_mock() self.sg._http_request.return_value = (status, resp_headers, data) - self.is_mock = True - return - + def _assert_http_method(self, method, params, check_auth=True): """Asserts _http_request is called with the method and params.""" - args, _ = self.sg._http_request.call_args arg_body = args[2] assert isinstance(arg_body, basestring) @@ -133,71 +134,57 @@ def _assert_http_method(self, method, params, check_auth=True): rpc_args = arg_params[len(arg_params)-1] self.assertEqual(params, rpc_args) - return - - def _setup_mock_data(self, config): - self.human_user = { 'id':1, - 'login':config.human_login, - 'type':'HumanUser' } - self.project = { 'id':2, - 'name':config.project_name, - 'type':'Project' } - self.shot = { 'id':3, - 'code':config.shot_code, - 'type':'Shot' } - self.asset = { 'id':4, - 'code':config.asset_code, - 'type':'Asset' } - self.version = { 'id':5, - 'code':config.version_code, - 'type':'Version' } + + def _setup_mock_data(self): + self.human_user = { 'id':1, + 'login':self.config.human_login, + 'type':'HumanUser' } + self.project = { 'id':2, + 'name':self.config.project_name, + 'type':'Project' } + self.shot = { 'id':3, + 'code':self.config.shot_code, + 'type':'Shot' } + self.asset = { 'id':4, + 'code':self.config.asset_code, + 'type':'Asset' } + self.version = { 'id':5, + 'code':self.config.version_code, + 'type':'Version' } + +class LiveTestBase(TestBase): + '''Test base for tests relying on connection to server.''' + def setUp(self): + super(LiveTestBase, self).setUp() + self._setup_db(self.config) def _setup_db(self, config): - data = {'name':config.project_name} + data = {'name':self.config.project_name} self.project = _find_or_create_entity(self.sg, 'Project', data) - data = {'name':config.human_name, - 'login':config.human_login, - 'password_proxy':config.human_password} + data = {'name':self.config.human_name, + 'login':self.config.human_login, + 'password_proxy':self.config.human_password} self.human_user = _find_or_create_entity(self.sg, 'HumanUser', data) - data = {'code':config.asset_code, + data = {'code':self.config.asset_code, 'project':self.project} keys = ['code'] self.asset = _find_or_create_entity(self.sg, 'Asset', data, keys) data = {'project':self.project, - 'code':config.version_code, + 'code':self.config.version_code, 'entity':self.asset, 'user':self.human_user} keys = ['code','project'] self.version = _find_or_create_entity(self.sg, 'Version', data, keys) keys = ['code','project'] - data = {'code':config.shot_code, + data = {'code':self.config.shot_code, 'project':self.project} self.shot = _find_or_create_entity(self.sg, 'Shot', data, keys) - -def _find_or_create_entity(sg, entity_type, data, identifyiers=None): - '''Finds or creates entities. - @params: - sg - shogun_json.Shotgun instance - entity_type - entity type - data - dictionary of data for the entity - identifyiers -list of subset of keys from data which should be used to - uniquely identity the entity - @returns dicitonary of the entity values - ''' - identifyiers = identifyiers or ['name'] - fields = data.keys() - filters = [[key, 'is', data[key]] for key in identifyiers] - entity = sg.find_one(entity_type, filters, fields=fields) - entity = entity or sg.create(entity_type, data, return_fields=fields) - assert(entity) - return entity - class SgTestConfig(object): '''Reads test config and holds values''' def __init__(self): @@ -215,6 +202,7 @@ def __init__(self): self.version_code = None self.shot_code = None + def read_config(self, config_path): config_parser = ConfigParser() config_parser.read(config_path) @@ -222,6 +210,23 @@ def read_config(self, config_path): for option in config_parser.options(section): value = config_parser.get(section, option) setattr(self, option, value) - # cast non-sting attributes - self.mock = 'True' == str(self.mock) + + +def _find_or_create_entity(sg, entity_type, data, identifyiers=None): + '''Finds or creates entities. + @params: + sg - shogun_json.Shotgun instance + entity_type - entity type + data - dictionary of data for the entity + identifyiers -list of subset of keys from data which should be used to + uniquely identity the entity + @returns dicitonary of the entity values + ''' + identifyiers = identifyiers or ['name'] + fields = data.keys() + filters = [[key, 'is', data[key]] for key in identifyiers] + entity = sg.find_one(entity_type, filters, fields=fields) + entity = entity or sg.create(entity_type, data, return_fields=fields) + assert(entity) + return entity diff --git a/tests/test_api.py b/tests/test_api.py index 7046bd7cd..645ce31af 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -11,41 +11,20 @@ import base -class TestShotgunApi(base.TestBase): - - +class TestShotgunApi(base.LiveTestBase): def setUp(self): super(TestShotgunApi, self).setUp() def test_info(self): """Called info""" - - self._mock_http({ - 'version': [2, 4, 0, u'Dev'] - }) - + #TODO do more to check results self.sg.info() def test_server_dates(self): """Pass datetimes to the server""" - t = { - 'project': self.project, - 'start_date': datetime.date.today(), - } - self._mock_http({ - "results" : { - "start_date" : "2011-04-27", - "project" : { - "name" : "Demo Project", - "type" : "Project", - "id" : 4 - }, - "type" : "Task", - "sg_status_list" : "wtg", - "id" : 197, - "content" : "New Task" - } - }) + #TODO check results + t = { 'project': self.project, + 'start_date': datetime.date.today() } self.sg.create('Task', t, ['content', 'sg_status_list']) @@ -70,57 +49,22 @@ def test_batch(self): } }] - self._mock_http({ - "results" : [ - { - "code" : "New Shot 5", - "project" : { - "name" : "Demo Project", - "type" : "Project", - "id" : 4 - }, - "type" : "Shot", - "id" : 870 - }, - { - "code" : "Changed 1", - "type" : "Shot", - "id" : self.shot['id'] - }] - }) new_shot, updated_shot = self.sg.batch(requests) self.assertEqual(self.shot['id'], updated_shot["id"]) self.assertTrue(new_shot.get("id")) new_shot_id = new_shot["id"] - requests = [ - { - "request_type" : "delete", - "entity_type" : "Shot", - "entity_id" : new_shot_id - }] + requests = [{ "request_type" : "delete", + "entity_type" : "Shot", + "entity_id" : new_shot_id + }] - self._mock_http({"results" : [True]}) result = self.sg.batch(requests)[0] self.assertEqual(True, result) - return def test_create_update_delete(self): """Called create, update, delete, revive""" - - #Create - self._mock_http( - {'results': {'code': 'JohnnyApple_Design01_FaceFinal', - 'description': 'fixed rig per director final notes', - 'entity': {'id': 1, 'name': 'Asset 1', 'type': 'Asset'}, - 'id': 3, - 'project': {'id': 1, 'name': 'Demo Project', 'type': 'Project'}, - 'sg_status_list': 'rev', - 'type': 'Version', - 'user': {'id': 2, 'name': 'Aaron Morton', 'type': 'HumanUser'}}} - ) - data = { 'project': self.project, 'code':'JohnnyApple_Design01_FaceFinal', @@ -133,14 +77,9 @@ def test_create_update_delete(self): version = self.sg.create("Version", data, return_fields = ["id"]) self.assertTrue(isinstance(version, dict)) self.assertTrue("id" in version) + #TODO check results more thoroughly #TODO: test returned fields are requested fields - #Update - self._mock_http( - {'results': {'description': 'updated test', - 'id': version["id"], 'type': 'Version'}} - ) - data = data = { "description" : "updated test" } @@ -148,45 +87,21 @@ def test_create_update_delete(self): self.assertTrue(isinstance(version, dict)) self.assertTrue("id" in version) - #Delete - self._mock_http( - {'results': True} - ) rv = self.sg.delete("Version", version["id"]) self.assertEqual(True, rv) - self._mock_http( - {'results': False} - ) rv = self.sg.delete("Version", version["id"]) self.assertEqual(False, rv) - #Revive - self._mock_http( - {'results': True} - ) rv = self.sg.revive("Version", version["id"]) self.assertEqual(True, rv) - self._mock_http( - {'results': False} - ) rv = self.sg.revive("Version", version["id"]) self.assertEqual(False, rv) def test_find(self): """Called find, find_one for known entities""" - - self._mock_http( - {'results': {'entities': [self.version], - 'paging_info': {'current_page': 1, - 'entities_per_page': 500, - 'entity_count': 1, - 'page_count': 1}}} - ) - - filters = [ - ['project','is', self.project], - ['id','is', self.version['id']] - ] + filters = [] + filters.append(['project','is', self.project]) + filters.append(['id','is', self.version['id']]) fields = ['id'] @@ -201,28 +116,20 @@ def test_find(self): self.assertEqual("Version", version["type"]) self.assertEqual(self.version['id'], version["id"]) + def test_get_session_token(self): """Got session UUID""" - - uuid = "c6b57a9e207d13c74e6226eaba5eab77" - self._mock_http( - {"session_id" : uuid} - ) - + #TODO test results rv = self.sg._get_session_token() - #we only know what the value is if we mocked the repsonse - if self.is_mock: - self.assertEqual(uuid, rv) self.assertTrue(rv) - return + def test_upload_download(self): """Upload and download a thumbnail""" - #upload / download only works against a live server becuase it does #not use the standard http interface - if self.is_mock: - print "upload / down tests skipped when mock enabled." + if 'localhost' in self.server_url: + print "upload / down tests skipped for localhost" return this_dir, _ = os.path.split(__file__) @@ -244,28 +151,19 @@ def test_upload_download(self): orig_file = open(path, "rb").read() self.assertEqual(orig_file, attach_file) - return + def test_deprecated_functions(self): """Deprecated functions raise errors""" self.assertRaises(api.ShotgunError, self.sg.schema, "foo") self.assertRaises(api.ShotgunError, self.sg.entity_types) + def test_simple_summary(self): '''test_simple_summary tests simple query using summarize.''' summeries = [{'field': 'id', 'type': 'count'}] grouping = [{'direction': 'asc', 'field': 'id', 'type': 'exact'}] filters = [['project', 'is', self.project]] - self._mock_http({"results":{"groups":[{"group_name":"861", - "summaries":{"id":1}, - "group_value":"861"}, - {"group_name":"888", - "summaries":{"id":1}, - "group_value":"888"}], - "summaries":{"id":11} - } - } - ) result = self.sg.summarize('Shot', filters=filters, summary_fields=summeries, diff --git a/tests/test_api_long.py b/tests/test_api_long.py index cf162eb56..549eca003 100644 --- a/tests/test_api_long.py +++ b/tests/test_api_long.py @@ -3,22 +3,12 @@ Includes the schema functions and the automated searching for all entity types """ -import base, dummy_data +import base -class TestShotgunApiLong(base.TestBase): +class TestShotgunApiLong(base.LiveTestBase): def test_automated_find(self): """Called find for each entity type and read all fields""" - - #we just ned to get some response in the mock - self._mock_http( - {'results': {'entities': [{'id': -1, 'type': 'Mystery'}], - 'paging_info': {'current_page': 1, - 'entities_per_page': 500, - 'entity_count': 1, - 'page_count': 1}}} - ) - all_entities = self.sg.schema_entity_read().keys() direction = "asc" filter_operator = "all" @@ -35,6 +25,7 @@ def test_automated_find(self): continue #trying to use some different code paths to the other find test + #TODO for our test project, we haven't populated these entities.... order = [{'field_name': fields.keys()[0], 'direction': direction}] if "project" in fields: filters = [['project', 'is', self.project]] @@ -42,8 +33,8 @@ def test_automated_find(self): filters = [] records = self.sg.find(entity_type, filters, fields=fields.keys(), - order=order, filter_operator=filter_operator, limit=limit, - page=page) + order=order, filter_operator=filter_operator, + limit=limit, page=page) self.assertTrue(isinstance(records, list)) @@ -57,48 +48,39 @@ def test_automated_find(self): direction = "desc" limit = (limit % 5) + 1 page = (page % 3) + 1 - return + def test_schema(self): """Called schema functions""" - self._mock_http(dummy_data.schema_entity_read) schema = self.sg.schema_entity_read() self.assertTrue(schema, dict) self.assertTrue(len(schema) > 0) - self._mock_http(dummy_data.schema_read) schema = self.sg.schema_read() self.assertTrue(schema, dict) self.assertTrue(len(schema) > 0) - self._mock_http(dummy_data.schema_field_read_version) schema = self.sg.schema_field_read("Version") self.assertTrue(schema, dict) self.assertTrue(len(schema) > 0) - self._mock_http(dummy_data.schema_field_read_version_user) schema = self.sg.schema_field_read("Version", field_name="user") self.assertTrue(schema, dict) self.assertTrue(len(schema) > 0) self.assertTrue("user" in schema) - self._mock_http({"results":"sg_monkeys"}) - properties = { - "description" : "How many monkeys were needed" - } + properties = { "description" : "How many monkeys were needed" } new_field_name = self.sg.schema_field_create("Version", "number", - "Monkey Count", properties=properties) + "Monkey Count", + properties=properties) - self._mock_http({"results":True}) - properties = { - "description" : "How many monkeys turned up" - } - ret_val = self.sg.schema_field_update("Version", new_field_name, - properties) + properties = {"description" : "How many monkeys turned up"} + ret_val = self.sg.schema_field_update("Version", + new_field_name, + properties) self.assertTrue(ret_val) - self._mock_http({"results":True}) ret_val = self.sg.schema_field_delete("Version", new_field_name) self.assertTrue(ret_val) diff --git a/tests/test_client.py b/tests/test_client.py index 81f11e3da..a1fe1ceed 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -19,7 +19,8 @@ import base -class TestShotgunClient(base.TestBase): +class TestShotgunClient(base.MockTestBase): + '''Test case for shotgun api with server interactions mocked.''' def __init__(self, *args, **kws): super(TestShotgunClient, self).__init__(*args, **kws) @@ -32,29 +33,24 @@ def setUp(self): #always want the mock on self._setup_mock() - def test_detect_caps(self): + def test_detect_client_caps(self): """Client and server capabilities detected""" - + client_caps = self.sg.client_caps self.sg.connect() self.assertEqual(1, self.sg._http_request.call_count) - self.assertTrue(self.sg.client_caps is not None) - self.assertTrue(self.sg.client_caps.platform in ( - "windows", "linux", "mac")) - self.assertTrue(self.sg.client_caps.local_path_field.startswith( - "local_path")) - self.assertTrue(str(self.sg.client_caps).startswith( - "ClientCapabilities")) - self.assertTrue(self.sg.client_caps.py_version.startswith( - str(sys.version_info[0]))) - self.assertTrue(self.sg.client_caps.py_version.endswith( - str(sys.version_info[1]))) + self.assertTrue(client_caps is not None) + self.assertTrue(client_caps.platform in ("windows", "linux", "mac")) + self.assertTrue(client_caps.local_path_field.startswith("local_path")) + self.assertTrue(str(client_caps).startswith("ClientCapabilities")) + self.assertTrue(client_caps.py_version.startswith(str(sys.version_info[0]))) + self.assertTrue(client_caps.py_version.endswith(str(sys.version_info[1]))) - + def test_detect_server_caps(self): + '''test_detect_server_caps tests that ServerCapabilities object is made + with appropriate settings for given server version.''' #has paging is tested else where. - server_info = { - "version" : [9,9,9] - } + server_info = { "version" : [9,9,9] } self._mock_http(server_info) # ensrue the server caps is re-read self.sg._server_caps = None @@ -62,23 +58,16 @@ def test_detect_caps(self): self.assertFalse(self.sg.server_caps.is_dev) self.assertEqual((9,9,9), self.sg.server_caps.version) self.assertTrue(self.server_url.endswith(self.sg.server_caps.host)) - self.assertTrue(self.sg.server_caps.has_paging) - self.assertTrue(str(self.sg.server_caps).startswith( - "ServerCapabilities")) + self.assertTrue(str(self.sg.server_caps).startswith( "ServerCapabilities")) self.assertEqual(server_info, self.sg.server_info) - - self._mock_http({ - "version" : [9,9,9, "Dev"] - }) + self._mock_http({ "version" : [9,9,9, "Dev"] }) self.sg._server_caps = None self.assertTrue(self.sg.server_caps.is_dev) - return - def test_server_version(self): - """Server supports json API""" - + def test_server_version_json(self): + '''test_server_version_json tests expected versions for json support.''' sc = api.ServerCapabilities("foo", {"version" : (2,4,0)}) sc.version = (2,3,99) @@ -95,21 +84,17 @@ def test_server_version(self): sc.version = (2,5,0) sc._ensure_json_supported() - return - + def test_session_uuid(self): - """Session UUID is included in request""" - + """test_session_uuid tests session UUID is included in request""" #ok for the mock server to just return an error, we want to look at #whats in the request - self._mock_http({ - "message":"Go BANG", - "exception":True - }) + self._mock_http({ "message":"Go BANG", + "exception":True }) def auth_args(): - args, _ = self.sg._http_request.call_args - verb, path, body, headers = args + args = self.sg._http_request.call_args[0] + body = args[2] body = json.loads(body) return body["params"][0] @@ -121,8 +106,7 @@ def auth_args(): self.sg.set_session_uuid(my_uuid) self.assertRaises(api.Fault, self.sg.delete, "FakeType", 1) self.assertEqual(my_uuid, auth_args()["session_uuid"]) - return - + def test_config(self): """Client config can be created""" x = api._Config() @@ -133,8 +117,8 @@ def test_url(self): login = self.human_user['login'] password = self.human_password - self.assertRaises(ValueError, api.Shotgun, None, None, None) - self.assertRaises(ValueError, api.Shotgun, "file://foo.com",None,None) + self.assertRaises(ValueError, api.Shotgun, None, None, None, connect=False) + self.assertRaises(ValueError, api.Shotgun, "file://foo.com",None,None, connect=False) self.assertEqual("/api3/json", self.sg.config.api_path) @@ -142,11 +126,9 @@ def test_url(self): login_password = "%s:%s" % (login, password) # login:password@domain auth_url = "%s%s@%s" % (self.uri_prefix, login_password, self.domain) - sg = api.Shotgun(auth_url, None, None) + sg = api.Shotgun(auth_url, None, None, connect=False) expected = "Basic " + base64.encodestring(login_password).strip() self.assertEqual(expected, sg.config.authorization) - - return def test_authorization(self): """Authorization passed to server""" @@ -156,12 +138,9 @@ def test_authorization(self): # login:password@domain auth_url = "%s%s@%s" % (self.uri_prefix, login_password, self.domain) - - self.sg = api.Shotgun(auth_url, "foo", "bar") + self.sg = api.Shotgun(auth_url, "foo", "bar", connect=False) self._setup_mock() - self._mock_http({ - 'version': [2, 4, 0, u'Dev'] - }) + self._mock_http({ 'version': [2, 4, 0, u'Dev'] }) self.sg.info() @@ -170,66 +149,41 @@ def test_authorization(self): expected = "Basic " + base64.encodestring(login_password).strip() self.assertEqual(expected, headers.get("Authorization")) - return def test_connect_close(self): """Connection is closed and opened.""" - #The mock created an existing mock connection, self.sg.connect() self.assertEqual(0, self.mock_conn.request.call_count) self.sg.close() self.assertEqual(None, self.sg._connection) - return - def test_has_paging(self): - """Server paging detected""" - - #tricky because we now only support version > 2.4 - sc = api.ServerCapabilities("foo", {"version" : (2,4,0)}) - - self.assertFalse(sc._is_paging((0,0,0)), - "no version has no paging") - self.assertFalse(sc._is_paging((2,3,3)), - "2,3,3, has no paging") - self.assertTrue(sc._is_paging((2,3,4)), - "2,3,4, has paging") - self.assertTrue(sc._is_paging((2,3,5)), - "2,3,5, has paging") - self.assertTrue(sc._is_paging((2,4,0)), - "any 2.4 has paging") def test_network_retry(self): """Network failure is retried""" - self.sg._http_request.side_effect = api.HttpLib2Error self.assertRaises(api.HttpLib2Error, self.sg.info) self.assertTrue( self.sg.config.max_rpc_attempts ==self.sg._http_request.call_count, "Call is repeated") - return def test_http_error(self): """HTTP error raised and not retried.""" - self._mock_http( - "big old error string", - status=(500, "Internal Server Error") - ) + self._mock_http( "big old error string", + status=(500, "Internal Server Error")) self.assertRaises(RuntimeError, self.sg.info) - self.assertEqual(1, self.sg._http_request.call_count, - "Call is not repeated") - return + self.assertEqual(1, + self.sg._http_request.call_count, + "Call is not repeated") def test_rpc_error(self): """RPC error transformed into Python error""" - self._mock_http({ - "message":"Go BANG", - "exception":True - }) + self._mock_http({ "message":"Go BANG", + "exception":True }) self.assertRaises(api.Fault, self.sg.info) @@ -241,47 +195,38 @@ def test_rpc_error(self): def test_call_rpc(self): """Named rpc method is called and results handled""" - d = { - "no-results" : "data without a results key" - } + d = { "no-results" : "data without a results key" } self._mock_http(d) rv = self.sg._call_rpc("no-results", None) self._assert_http_method("no-results", None) - self.assertEqual(d, rv, - "rpc response without results key is returned as-is") + expected = "rpc response without results key is returned as-is" + self.assertEqual(d, rv, expected ) - d = { - "results" : {"singleton" : "result"} - } + d = { "results" : {"singleton" : "result"} } self._mock_http(d) rv = self.sg._call_rpc("singleton", None) self._assert_http_method("singleton", None) - self.assertEqual(d["results"], rv, - "rpc response with singleton result") + expected = "rpc response with singleton result" + self.assertEqual(d["results"], rv, expected ) - d = { - "results" : ["foo", "bar"] - } + d = { "results" : ["foo", "bar"] } a = {"some" : "args"} self._mock_http(d) rv = self.sg._call_rpc("list", a) self._assert_http_method("list", a) - self.assertEqual(d["results"], rv, - "rpc response with list result") + expected = "rpc response with list result" + self.assertEqual(d["results"], rv, expected ) - d = { - "results" : ["foo", "bar"] - } + d = { "results" : ["foo", "bar"] } a = {"some" : "args"} self._mock_http(d) rv = self.sg._call_rpc("list-first", a, first=True) self._assert_http_method("list-first", a) - self.assertEqual(d["results"][0], rv, - "rpc response with list result, first item") + expected = "rpc response with list result, first item" + self.assertEqual(d["results"][0], rv, expected ) def test_transform_data(self): """Outbound data is transformed""" - timestamp = time.time() #microseconds will be last during transforms now = datetime.datetime.fromtimestamp(timestamp).replace( @@ -334,7 +279,6 @@ def assert_wire(wire, match): #times will become datetime over the wire wire["time"] = wire["time"].time() self.assertEqual(local, wire) - return def test_encode_payload(self): """Request body is encoded as JSON""" @@ -403,7 +347,6 @@ def test_parse_records(self): self.assertEqual("/foo/bar.jpg", modified["foo"]["local_path"]) self.assertEqual("file:///foo/bar.jpg", modified["foo"]["url"]) - return def test_thumb_url(self): """Thumbnail endpoint used to get thumbnail url""" @@ -436,56 +379,7 @@ def test_thumb_url(self): self._mock_http(resp, headers={"content-type" : "text/plain"}) self.assertRaises(RuntimeError, self.sg._build_thumb_url, "FakeAsset", 456) - return - -class TestCreateSummaryRequest(base.TestBase): - '''Test case for _create_summary_request function and parameter - validation as it exists in Shotgun.summarize. - - Does not require database connection or test data.''' - - def test_filter_operator_none(self): - expected_logical_operator = 'and' - filter_operator = None - result = api._create_summary_request('',[],None,filter_operator, None) - actual_logical_operator = result['filters']['logical_operator'] - self.assertEqual(expected_logical_operator, actual_logical_operator) - - def test_filter_operator_all(self): - expected_logical_operator = 'and' - filter_operator = 'all' - result = api._create_summary_request('',[],None,filter_operator, None) - actual_logical_operator = result['filters']['logical_operator'] - self.assertEqual(expected_logical_operator, actual_logical_operator) - - def test_filter_operator_none(self): - expected_logical_operator = 'or' - filter_operator = 'or' - result = api._create_summary_request('',[],None,filter_operator, None) - actual_logical_operator = result['filters']['logical_operator'] - self.assertEqual(expected_logical_operator, actual_logical_operator) - - def test_filters(self): - path = 'path' - relation = 'relation' - value = 'value' - expected_condition = {'path':path, 'relation':relation, 'value':value} - result = api._create_summary_request('', [[path, relation, value]], None, None, None) - actual_condition = result['filters']['conditions'][0] - - def test_grouping(self): - result = api._create_summary_request('', [], None, None, None) - self.assertFalse(result.has_key('grouping')) - grouping = ['something'] - result = api._create_summary_request('', [], None, None, grouping) - self.assertEqual(grouping, result['grouping']) - def test_filters_type(self): - '''test_filters_type tests that filters parameter is a list''' - self.assertRaises(ValueError, self.sg.summarize, '', 'not a list', []) - def test_grouping_type(self): - '''test_grouping_type tests that grouping parameter is a list or None''' - self.assertRaises(ValueError, self.sg.summarize, '', [], [], grouping='Not a list') if __name__ == '__main__': unittest.main() diff --git a/tests/tests_unit.py b/tests/tests_unit.py new file mode 100644 index 000000000..3461f5a68 --- /dev/null +++ b/tests/tests_unit.py @@ -0,0 +1,214 @@ +#! /opt/local/bin/python +import unittest +from tests import base +from mock import patch, Mock +import shotgun_api3 as api +import socks + +class TestShotgunInit(unittest.TestCase): + '''Test case for Shotgun.__init__''' + + def test_http_proxy(self): + '''test_http_proxy tests setting of http proxy attributes.''' + server_path = 'http://server_path' + script_name = 'script_name' + api_key = 'api_key' + proxy_server = 'somedomain.com' + proxy_port = 3000 + http_proxy = 'https://%s:%s/somepage.html' % (proxy_server, proxy_port) + + sg = api.Shotgun(server_path, + script_name, + api_key, + http_proxy=http_proxy, + connect=False) + self.assertEquals(sg.config.proxy_server, proxy_server) + self.assertEquals(sg.config.proxy_port, proxy_port) + + +class TestCreateSummaryRequest(unittest.TestCase): + '''Test case for _create_summary_request function and parameter + validation as it exists in Shotgun.summarize. + + Does not require database connection or test data.''' + + def setUp(self): + server_path = 'http://server_path' + script_name = 'script_name' + api_key = 'api_key' + + self.sg = api.Shotgun(server_path, + script_name, + api_key, + connect=False) + + + def test_filter_operator_none(self): + expected_logical_operator = 'and' + filter_operator = None + result = api._create_summary_request('',[],None,filter_operator, None) + actual_logical_operator = result['filters']['logical_operator'] + self.assertEqual(expected_logical_operator, actual_logical_operator) + + def test_filter_operator_all(self): + expected_logical_operator = 'and' + filter_operator = 'all' + result = api._create_summary_request('',[],None,filter_operator, None) + actual_logical_operator = result['filters']['logical_operator'] + self.assertEqual(expected_logical_operator, actual_logical_operator) + + def test_filter_operator_none(self): + expected_logical_operator = 'or' + filter_operator = 'or' + result = api._create_summary_request('',[],None,filter_operator, None) + actual_logical_operator = result['filters']['logical_operator'] + self.assertEqual(expected_logical_operator, actual_logical_operator) + + def test_filters(self): + path = 'path' + relation = 'relation' + value = 'value' + expected_condition = {'path':path, 'relation':relation, 'value':value} + result = api._create_summary_request('', [[path, relation, value]], None, None, None) + actual_condition = result['filters']['conditions'][0] + + def test_grouping(self): + result = api._create_summary_request('', [], None, None, None) + self.assertFalse(result.has_key('grouping')) + grouping = ['something'] + result = api._create_summary_request('', [], None, None, grouping) + self.assertEqual(grouping, result['grouping']) + + def test_filters_type(self): + '''test_filters_type tests that filters parameter is a list''' + self.assertRaises(ValueError, self.sg.summarize, '', 'not a list', 'bad meta') + + def test_grouping_type(self): + '''test_grouping_type tests that grouping parameter is a list or None''' + self.assertRaises(ValueError, self.sg.summarize, '', [], [], grouping='Not a list') + + +class TestServerCapabilities(unittest.TestCase): + def test_no_server_version(self): + self.assertRaises(api.ShotgunError, api.ServerCapabilities, 'host', {}) + + + def test_bad_version(self): + '''test_bad_meta tests passing bad meta data type''' + self.assertRaises(api.ShotgunError, api.ServerCapabilities, 'host', {'version':(0,0,0)}) + + def test_dev_version(self): + serverCapabilities = api.ServerCapabilities('host', {'version':(3,4,0,'Dev')}) + self.assertEqual(serverCapabilities.version, (3,4,0)) + self.assertTrue(serverCapabilities.is_dev) + + serverCapabilities = api.ServerCapabilities('host', {'version':(2,4,0)}) + self.assertEqual(serverCapabilities.version, (2,4,0)) + self.assertFalse(serverCapabilities.is_dev) + +class TestClientCapabilities(unittest.TestCase): + + def test_darwin(self): + self.assert_platform('Darwin', 'mac') + + def test_windows(self): + self.assert_platform('Windows','windows') + + def test_linux(self): + self.assert_platform('Linux', 'linux') + + @patch('shotgun_api3.platform') + def assert_platform(self, sys_ret_val, expected, mock_platform): + mock_platform.system.return_value = sys_ret_val + expected_local_path_field = "local_path_%s" % expected + + client_caps = api.ClientCapabilities() + self.assertEquals(client_caps.platform, expected) + self.assertEquals(client_caps.local_path_field, expected_local_path_field) + + @patch('shotgun_api3.platform') + def test_no_platform(self, mock_platform): + mock_platform.system.return_value = "unsupported" + client_caps = api.ClientCapabilities() + self.assertIsNone(client_caps.platform) + self.assertIsNone(client_caps.local_path_field) + + + @patch('shotgun_api3.sys') + def test_py_version(self, mock_sys): + major = 2 + minor = 7 + micro = 3 + mock_sys.version_info = (major, minor, micro, 'final', 0) + expected_py_version = "%s.%s" % (major, minor) + client_caps = api.ClientCapabilities() + self.assertEquals(client_caps.py_version, expected_py_version) + +class TestSockSocket(unittest.TestCase): + def test_defaultproxy(self): + socks._defaultproxy = object() + ssocket = api.socksocket() + self.assertEquals(ssocket._socksocket__proxy, socks._defaultproxy) + + def test_connect_bad_destpair(self): + '''test_connect_bad_destpair tests various bad destpair parameters.''' + ssocket = api.socksocket() + #not list or tuple + self.assertRaises(api.GeneralProxyError, ssocket.connect, 'not a list or tuple') + #wrong length + self.assertRaises(api.GeneralProxyError, ssocket.connect, [1]) + #first item str + self.assertRaises(api.GeneralProxyError, ssocket.connect, [1, 2]) + #second item int + self.assertRaises(api.GeneralProxyError, ssocket.connect, ['1', '2']) + + def test_connect_sock5(self): + self._assert_connect_proxy_type(api.PROXY_TYPE_SOCKS5, sock5_calls=1) + + def test_connect_sock4(self): + self._assert_connect_proxy_type(api.PROXY_TYPE_SOCKS4, sock4_calls=1) + + def test_connect_http(self): + self._assert_connect_proxy_type(api.PROXY_TYPE_HTTP, http_calls=1) + + def test_connect_proxy_type_none(self): + self._assert_connect_proxy_type(None) + + def test_unkown_proxy_type(self): + ssocket = api.socksocket() + ssocket._socksocket__proxy = (99, 'address', 1080) + self.assertRaises(api.GeneralProxyError, ssocket.connect, ('dns',9999)) + + + @patch('socks._orgsocket.connect') + def _assert_connect_proxy_type(self, + proxy_type, + _orgsocket_connect, + sock5_calls=0, + sock4_calls=0, + http_calls=0): + ssocket = api.socksocket() + ssocket._socksocket__proxy = (proxy_type, 'address', 1080) + ssocket._socksocket__negotiatesocks5 = Mock(name='__negotiatesocks5') + ssocket._socksocket__negotiatesocks4 = Mock(name='__negotiatesocks4') + ssocket._socksocket__negotiatehttp = Mock(name='__negotiatehttp') + + self.assertEquals(ssocket._socksocket__negotiatesocks5.call_count, 0) + self.assertEquals(ssocket._socksocket__negotiatesocks4.call_count, 0) + self.assertEquals(ssocket._socksocket__negotiatehttp.call_count, 0) + self.assertEquals(_orgsocket_connect.call_count, 0) + + ssocket.connect(('dns', 8080)) + self.assertEquals(ssocket._socksocket__negotiatesocks5.call_count, sock5_calls) + self.assertEquals(ssocket._socksocket__negotiatesocks4.call_count, sock4_calls) + self.assertEquals(ssocket._socksocket__negotiatehttp.call_count, http_calls) + self.assertEquals(_orgsocket_connect.call_count, 1) + + + + + + + + + From c8e08ed8e84a7b704b8b477d855ec44a7515a061 Mon Sep 17 00:00:00 2001 From: kennedy behrman Date: Tue, 19 Jul 2011 15:02:24 -0700 Subject: [PATCH 015/570] removed timeout setting --- shotgun_api3.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shotgun_api3.py b/shotgun_api3.py index d4e08a183..17cf0ba38 100755 --- a/shotgun_api3.py +++ b/shotgun_api3.py @@ -154,7 +154,7 @@ def __init__(self): """Container for the client configuration.""" self.max_rpc_attempts = 3 - self.timeout_secs = 3 + self.timeout_secs = None self.api_ver = 'api3' self.convert_datetimes_to_utc = True self.records_per_page = 500 @@ -2201,6 +2201,7 @@ def connect(self): # HACK: amorton socks module is included in the file self.sock = socksocket(af, socktype, proto) # HACK: amorton enabled TCP_NODELAY on socket + #TODO why disable nagle? self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) self.sock.setproxy(*self.proxy_info.astuple()) else: From f16692b2d8e77feb1c0dc12c86037feb9c267552 Mon Sep 17 00:00:00 2001 From: kennedy behrman Date: Wed, 20 Jul 2011 11:30:42 -0700 Subject: [PATCH 016/570] moved gae_restful_lib out to seperate file --- gae_restful_lib.py | 488 +++++++++++++++++++++++++++++++++++++++++++++ shotgun_api3.py | 287 +------------------------- 2 files changed, 497 insertions(+), 278 deletions(-) create mode 100644 gae_restful_lib.py diff --git a/gae_restful_lib.py b/gae_restful_lib.py new file mode 100644 index 000000000..f105fc59e --- /dev/null +++ b/gae_restful_lib.py @@ -0,0 +1,488 @@ +""" +Copyright (C) 2008 Benjamin O'Steen + + This file is part of python-fedoracommons. + + python-fedoracommons is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + python-fedoracommons is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with python-fedoracommons. If not, see . + + * Shotgun software: removing gae dependency +""" + +__license__ = 'GPL http://www.gnu.org/licenses/gpl.txt' +__author__ = "Benjamin O'Steen " +__version__ = '0.1' + +#from google.appengine.api import urlfetch + +import urlparse +from urllib import urlencode +import base64 +from base64 import encodestring + +import re +import md5 +import calendar +import time +import random +import sha +import hmac + +#from mimeTypes import * + +#import mimetypes + +from cStringIO import StringIO + + + +# For Auth implemnentation: Digest (from httplib2) +# TODO: !Important - add proper code attribution for httplib2 parts +USE_WWW_AUTH_STRICT_PARSING = 0 +conn = None +# In regex below: +# [^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+ matches a "token" as defined by HTTP +# "(?:[^\0-\x08\x0A-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?" matches a "quoted-string" as defined by HTTP, when LWS have already been replaced by a single space +# Actually, as an auth-param value can be either a token or a quoted-string, they are combined in a single pattern which matches both: +# \"?((?<=\")(?:[^\0-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?(?=\")|(?@,;:\\\"/[\]?={} \t]+(?!\"))\"? +WWW_AUTH_STRICT = re.compile(r"^(?:\s*(?:,\s*)?([^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+)\s*=\s*\"?((?<=\")(?:[^\0-\x08\x0A-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?(?=\")|(?@,;:\\\"/[\]?={} \t]+(?!\"))\"?)(.*)$") +WWW_AUTH_RELAXED = re.compile(r"^(?:\s*(?:,\s*)?([^ \t\r\n=]+)\s*=\s*\"?((?<=\")(?:[^\\\"]|\\.)*?(?=\")|(? 0: + service = "cl" + # No point in guessing Base or Spreadsheet + #elif request_uri.find("spreadsheets") > 0: + # service = "wise" + + auth = dict(Email=credentials[0], Passwd=credentials[1], service=service, source=headers['user-agent']) + resp, content = self.http.request("https://www.google.com/accounts/ClientLogin", method="POST", body=urlencode(auth), headers={'Content-Type': 'application/x-www-form-urlencoded'}) + lines = content.split('\n') + d = dict([tuple(line.split("=", 1)) for line in lines if line]) + if resp.status == 403: + self.Auth = "" + else: + self.Auth = d['Auth'] + + def request(self, method, request_uri, headers, content): + """Modify the request headers to add the appropriate + Authorization header.""" + headers['authorization'] = 'GoogleLogin Auth=' + self.Auth + + +class Credentials(object): + def __init__(self): + self.credentials = [] + + def add(self, name, password, domain=""): + self.credentials.append((domain.lower(), name, password)) + + def clear(self): + self.credentials = [] + + def iter(self, domain): + for (cdomain, name, password) in self.credentials: + if cdomain == "" or domain == cdomain: + yield (name, password) + +#AUTH_SCHEME_CLASSES = { +# "basic": BasicAuthentication, +# "Basic": BasicAuthentication, +# "wsse": WsseAuthentication, +# "digest": DigestAuthentication, +# "Digest": DigestAuthentication, +# "hmacdigest": HmacDigestAuthentication +#} +# +#AUTH_SCHEME_ORDER = ["hmacdigest", "digest", "Digest", "wsse", "basic", "Basic"] +# +#URLFETCH_METHOD_STRING = {urlfetch.GET:'GET', +# urlfetch.PUT:'PUT', +# urlfetch.DELETE:'DELETE', +# urlfetch.POST:'POST', +# urlfetch.HEAD:'HEAD' +# } +# +# +#class GAE_Connection: +# def __init__(self, base_url, username=None, password=None): +# self.base_url = base_url +# m = mimeTypes() +# self.mimetypes = m.getDictionary() +# +# # Name/password +# self.credentials = Credentials() +# +# if username and password: +# self.add_credentials(username, password, domain="") +# +# # authorization objects +# self.authorizations = [] +# +# self.url = urlparse.urlparse(base_url) +# +# (scheme, netloc, path, query, fragment) = urlparse.urlsplit(base_url) +# +# self.scheme = scheme +# self.host = netloc +# self.path = path +# +# def _auth_from_challenge(self, host, request_uri, headers, response, content): +# """A generator that creates Authorization objects +# that can be applied to requests. +# """ +# challenges = _parse_www_authenticate(response, 'www-authenticate') +# for cred in self.credentials.iter(host): +# for scheme in AUTH_SCHEME_ORDER: +# if challenges.has_key(scheme): +# yield AUTH_SCHEME_CLASSES[scheme](cred, host, request_uri, headers, response, content, self) +# +# def add_credentials(self, name, password, domain=""): +# """Add a name and password that will be used +# any time a request requires authentication.""" +# self.credentials.add(name, password, domain) +# +# def clear_credentials(self): +# """Remove all the names and passwords +# that are used for authentication""" +# self.credentials.clear() +# self.authorizations = [] +# +# def request_get(self, resource, args = None, headers={}): +# return self.request(resource, urlfetch.GET, args, headers=headers) +# +# def request_delete(self, resource, args = None, headers={}): +# return self.request(resource, urlfetch.DELETE, args, headers=headers) +# +# def request_post(self, resource, args = None, body = None, filename=None, headers={}): +# return self.request(resource, urlfetch.POST, args , body = body, filename=filename, headers=headers) +# +# def request_put(self, resource, args = None, body = None, filename=None, headers={}): +# return self.request(resource, urlfetch.PUT, args , body = body, filename=filename, headers=headers) +# +# def request_head(self, resource, args = None, body = None, filename=None, headers={}): +# return self.request(resource, urlfetch.HEAD, args , body = body, filename=filename, headers=headers) +# +# def _conn_request(self, conn, request_uri, method, body, headers): +# # Shim to allow easy reuse of httplib2 auth methods - conn param is not used +# urlfetch_response = urlfetch.fetch(request_uri, method=method, payload=body, headers=headers) +# r_headers={'status':urlfetch_response.status_code} +# for header_key in urlfetch_response.headers: +# r_headers[header_key.lower()] = urlfetch_response.headers[header_key] +# +# return (r_headers, urlfetch_response.content.decode('UTF-8')) +# +# def get_content_type(self, filename): +# extension = filename.split('.')[-1] +# guessed_mimetype = self.mimetypes.get(extension, mimetypes.guess_type(filename)[0]) +# return guessed_mimetype or 'application/octet-stream' +# +# def request(self, resource, method = urlfetch.GET, args = None, body = None, filename=None, headers={}): +# params = None +# path = resource +# headers['User-Agent'] = 'Basic Agent' +# +# if not headers.get('Content-Type', None): +# headers['Content-Type']='text/plain' +# +# request_path = [] +# if self.path != "/": +# if self.path.endswith('/'): +# request_path.append(self.path[:-1]) +# else: +# request_path.append(self.path) +# if path.startswith('/'): +# request_path.append(path[1:]) +# else: +# request_path.append(path) +# full_path = u'/'.join(request_path) +# +# if args: +# full_path += u"?%s" % (urlencode(args)) +# +# request_uri = u"%s://%s%s" % (self.scheme, self.host, full_path) +# +# auths = [(auth.depth(request_uri), auth) for auth in self.authorizations if auth.inscope(host, request_uri)] +# auth = auths and sorted(auths)[0][1] or None +# if auth: +# auth.request(method, request_uri, headers, body) +# +# (response, content) = self._conn_request(conn, request_uri, method, body, headers) +# +# if auth: +# if auth.response(response, body): +# auth.request(URLFETCH_METHOD_STRING[method], request_uri, headers, body) +# +# (response, content) = self._conn_request(conn, request_uri, method, body, headers) +# +# if response['status'] == 401: +# #return {u"body":u"".join(["%s: %s" % (key, response[key]) for key in response])} +# for authorization in self._auth_from_challenge(self.host, request_uri, headers, response, content): +# authorization.request(URLFETCH_METHOD_STRING[method], request_uri, headers, body) +# +# (response, content) = self._conn_request(conn, request_uri, method, body, headers) +# +# if response['status'] != 401: +# self.authorizations.append(authorization) +# authorization.response(response, body) +# break +# +# return {u'headers':response, u'body':content} + diff --git a/shotgun_api3.py b/shotgun_api3.py index 17cf0ba38..131503664 100755 --- a/shotgun_api3.py +++ b/shotgun_api3.py @@ -48,6 +48,10 @@ import urllib2 # used for image upload import urlparse from socks import * +from gae_restful_lib import (Credentials, WsseAuthentication, + HmacDigestAuthentication, DigestAuthentication, + BasicAuthentication, Authentication, + GoogleLoginAuthentication) log = logging.getLogger("shotgun_api3") @@ -728,7 +732,7 @@ def upload(self, entity_type, entity_id, path, field_name=None, :returns: Id of the new attachment. """ - path = os.path.abspath(os.path.expanduser(path or "")) + path = os.path.abspath(os.path.expanduser(path or "")) if not os.path.isfile(path): raise ShotgunError("Path must be a valid file, got '%s'" % path) @@ -795,7 +799,7 @@ def download_attachment(self, attachment_id): :returns: binary data as a string """ - sid = self._get_session_token() + sid = self._get_session_token() cj = cookielib.LWPCookieJar() c = cookielib.Cookie('0', '_session_id', sid, None, False, self.config.server, False, False, "/", True, False, None, True, @@ -1702,41 +1706,6 @@ def _parse_cache_control(headers): # Set to true to turn on, usefull for testing servers. USE_WWW_AUTH_STRICT_PARSING = 0 -# In regex below: -# [^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+ matches a "token" as defined by HTTP -# "(?:[^\0-\x08\x0A-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?" matches a "quoted-string" as defined by HTTP, when LWS have already been replaced by a single space -# Actually, as an auth-param value can be either a token or a quoted-string, they are combined in a single pattern which matches both: -# \"?((?<=\")(?:[^\0-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?(?=\")|(?@,;:\\\"/[\]?={} \t]+(?!\"))\"? -WWW_AUTH_STRICT = re.compile(r"^(?:\s*(?:,\s*)?([^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+)\s*=\s*\"?((?<=\")(?:[^\0-\x08\x0A-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?(?=\")|(?@,;:\\\"/[\]?={} \t]+(?!\"))\"?)(.*)$") -WWW_AUTH_RELAXED = re.compile(r"^(?:\s*(?:,\s*)?([^ \t\r\n=]+)\s*=\s*\"?((?<=\")(?:[^\\\"]|\\.)*?(?=\")|(? 0: - service = "cl" - # No point in guessing Base or Spreadsheet - #elif request_uri.find("spreadsheets") > 0: - # service = "wise" - - auth = dict(Email=credentials[0], Passwd=credentials[1], service=service, source=headers['user-agent']) - resp, content = self.http.request("https://www.google.com/accounts/ClientLogin", method="POST", body=urlencode(auth), headers={'Content-Type': 'application/x-www-form-urlencoded'}) - lines = content.split('\n') - d = dict([tuple(line.split("=", 1)) for line in lines if line]) - if resp.status == 403: - self.Auth = "" - else: - self.Auth = d['Auth'] - - def request(self, method, request_uri, headers, content): - """Modify the request headers to add the appropriate - Authorization header.""" - headers['authorization'] = 'GoogleLogin Auth=' + self.Auth - AUTH_SCHEME_CLASSES = { "basic": BasicAuthentication, @@ -2141,20 +1887,6 @@ def delete(self, key): if os.path.exists(cacheFullPath): os.remove(cacheFullPath) -class Credentials(object): - def __init__(self): - self.credentials = [] - - def add(self, name, password, domain=""): - self.credentials.append((domain.lower(), name, password)) - - def clear(self): - self.credentials = [] - - def iter(self, domain): - for (cdomain, name, password) in self.credentials: - if cdomain == "" or domain == cdomain: - yield (name, password) class KeyCerts(Credentials): """Identical to Credentials except that @@ -2201,7 +1933,6 @@ def connect(self): # HACK: amorton socks module is included in the file self.sock = socksocket(af, socktype, proto) # HACK: amorton enabled TCP_NODELAY on socket - #TODO why disable nagle? self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) self.sock.setproxy(*self.proxy_info.astuple()) else: @@ -2560,7 +2291,7 @@ def request(self, uri, method="GET", body=None, headers=None, redirections=DEFAU break if cached_value and method in ["GET", "HEAD"] and self.cache and 'range' not in headers: - if info.has_key('-x-permanent-redirect-url'): + if info.has_key('-x-permanent-redirect-url'): # Should cached permanent redirects be counted in our redirection count? For now, yes. (response, new_content) = self.request(info['-x-permanent-redirect-url'], "GET", headers = headers, redirections = redirections - 1) response.previous = Response(info) From c0fc67fe3900e5fd13d9f78229fb3cca3a05bc94 Mon Sep 17 00:00:00 2001 From: kennedy behrman Date: Wed, 20 Jul 2011 13:19:35 -0700 Subject: [PATCH 017/570] extracted httplib2 from shotgun_api3; removed unneeded gae_restful_lib.py;added httplib2 tests: --- gae_restful_lib.py | 488 -------------- httplib2/__init__.py | 1222 +++++++++++++++++++++++++++++++++++ httplib2/iri2uri.py | 110 ++++ shotgun_api3.py | 1061 +------------------------------ tests/httplib2test.py | 1398 +++++++++++++++++++++++++++++++++++++++++ tests/test_client.py | 5 +- 6 files changed, 2734 insertions(+), 1550 deletions(-) delete mode 100644 gae_restful_lib.py create mode 100755 httplib2/__init__.py create mode 100755 httplib2/iri2uri.py create mode 100755 tests/httplib2test.py diff --git a/gae_restful_lib.py b/gae_restful_lib.py deleted file mode 100644 index f105fc59e..000000000 --- a/gae_restful_lib.py +++ /dev/null @@ -1,488 +0,0 @@ -""" -Copyright (C) 2008 Benjamin O'Steen - - This file is part of python-fedoracommons. - - python-fedoracommons is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - python-fedoracommons is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with python-fedoracommons. If not, see . - - * Shotgun software: removing gae dependency -""" - -__license__ = 'GPL http://www.gnu.org/licenses/gpl.txt' -__author__ = "Benjamin O'Steen " -__version__ = '0.1' - -#from google.appengine.api import urlfetch - -import urlparse -from urllib import urlencode -import base64 -from base64 import encodestring - -import re -import md5 -import calendar -import time -import random -import sha -import hmac - -#from mimeTypes import * - -#import mimetypes - -from cStringIO import StringIO - - - -# For Auth implemnentation: Digest (from httplib2) -# TODO: !Important - add proper code attribution for httplib2 parts -USE_WWW_AUTH_STRICT_PARSING = 0 -conn = None -# In regex below: -# [^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+ matches a "token" as defined by HTTP -# "(?:[^\0-\x08\x0A-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?" matches a "quoted-string" as defined by HTTP, when LWS have already been replaced by a single space -# Actually, as an auth-param value can be either a token or a quoted-string, they are combined in a single pattern which matches both: -# \"?((?<=\")(?:[^\0-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?(?=\")|(?@,;:\\\"/[\]?={} \t]+(?!\"))\"? -WWW_AUTH_STRICT = re.compile(r"^(?:\s*(?:,\s*)?([^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+)\s*=\s*\"?((?<=\")(?:[^\0-\x08\x0A-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?(?=\")|(?@,;:\\\"/[\]?={} \t]+(?!\"))\"?)(.*)$") -WWW_AUTH_RELAXED = re.compile(r"^(?:\s*(?:,\s*)?([^ \t\r\n=]+)\s*=\s*\"?((?<=\")(?:[^\\\"]|\\.)*?(?=\")|(? 0: - service = "cl" - # No point in guessing Base or Spreadsheet - #elif request_uri.find("spreadsheets") > 0: - # service = "wise" - - auth = dict(Email=credentials[0], Passwd=credentials[1], service=service, source=headers['user-agent']) - resp, content = self.http.request("https://www.google.com/accounts/ClientLogin", method="POST", body=urlencode(auth), headers={'Content-Type': 'application/x-www-form-urlencoded'}) - lines = content.split('\n') - d = dict([tuple(line.split("=", 1)) for line in lines if line]) - if resp.status == 403: - self.Auth = "" - else: - self.Auth = d['Auth'] - - def request(self, method, request_uri, headers, content): - """Modify the request headers to add the appropriate - Authorization header.""" - headers['authorization'] = 'GoogleLogin Auth=' + self.Auth - - -class Credentials(object): - def __init__(self): - self.credentials = [] - - def add(self, name, password, domain=""): - self.credentials.append((domain.lower(), name, password)) - - def clear(self): - self.credentials = [] - - def iter(self, domain): - for (cdomain, name, password) in self.credentials: - if cdomain == "" or domain == cdomain: - yield (name, password) - -#AUTH_SCHEME_CLASSES = { -# "basic": BasicAuthentication, -# "Basic": BasicAuthentication, -# "wsse": WsseAuthentication, -# "digest": DigestAuthentication, -# "Digest": DigestAuthentication, -# "hmacdigest": HmacDigestAuthentication -#} -# -#AUTH_SCHEME_ORDER = ["hmacdigest", "digest", "Digest", "wsse", "basic", "Basic"] -# -#URLFETCH_METHOD_STRING = {urlfetch.GET:'GET', -# urlfetch.PUT:'PUT', -# urlfetch.DELETE:'DELETE', -# urlfetch.POST:'POST', -# urlfetch.HEAD:'HEAD' -# } -# -# -#class GAE_Connection: -# def __init__(self, base_url, username=None, password=None): -# self.base_url = base_url -# m = mimeTypes() -# self.mimetypes = m.getDictionary() -# -# # Name/password -# self.credentials = Credentials() -# -# if username and password: -# self.add_credentials(username, password, domain="") -# -# # authorization objects -# self.authorizations = [] -# -# self.url = urlparse.urlparse(base_url) -# -# (scheme, netloc, path, query, fragment) = urlparse.urlsplit(base_url) -# -# self.scheme = scheme -# self.host = netloc -# self.path = path -# -# def _auth_from_challenge(self, host, request_uri, headers, response, content): -# """A generator that creates Authorization objects -# that can be applied to requests. -# """ -# challenges = _parse_www_authenticate(response, 'www-authenticate') -# for cred in self.credentials.iter(host): -# for scheme in AUTH_SCHEME_ORDER: -# if challenges.has_key(scheme): -# yield AUTH_SCHEME_CLASSES[scheme](cred, host, request_uri, headers, response, content, self) -# -# def add_credentials(self, name, password, domain=""): -# """Add a name and password that will be used -# any time a request requires authentication.""" -# self.credentials.add(name, password, domain) -# -# def clear_credentials(self): -# """Remove all the names and passwords -# that are used for authentication""" -# self.credentials.clear() -# self.authorizations = [] -# -# def request_get(self, resource, args = None, headers={}): -# return self.request(resource, urlfetch.GET, args, headers=headers) -# -# def request_delete(self, resource, args = None, headers={}): -# return self.request(resource, urlfetch.DELETE, args, headers=headers) -# -# def request_post(self, resource, args = None, body = None, filename=None, headers={}): -# return self.request(resource, urlfetch.POST, args , body = body, filename=filename, headers=headers) -# -# def request_put(self, resource, args = None, body = None, filename=None, headers={}): -# return self.request(resource, urlfetch.PUT, args , body = body, filename=filename, headers=headers) -# -# def request_head(self, resource, args = None, body = None, filename=None, headers={}): -# return self.request(resource, urlfetch.HEAD, args , body = body, filename=filename, headers=headers) -# -# def _conn_request(self, conn, request_uri, method, body, headers): -# # Shim to allow easy reuse of httplib2 auth methods - conn param is not used -# urlfetch_response = urlfetch.fetch(request_uri, method=method, payload=body, headers=headers) -# r_headers={'status':urlfetch_response.status_code} -# for header_key in urlfetch_response.headers: -# r_headers[header_key.lower()] = urlfetch_response.headers[header_key] -# -# return (r_headers, urlfetch_response.content.decode('UTF-8')) -# -# def get_content_type(self, filename): -# extension = filename.split('.')[-1] -# guessed_mimetype = self.mimetypes.get(extension, mimetypes.guess_type(filename)[0]) -# return guessed_mimetype or 'application/octet-stream' -# -# def request(self, resource, method = urlfetch.GET, args = None, body = None, filename=None, headers={}): -# params = None -# path = resource -# headers['User-Agent'] = 'Basic Agent' -# -# if not headers.get('Content-Type', None): -# headers['Content-Type']='text/plain' -# -# request_path = [] -# if self.path != "/": -# if self.path.endswith('/'): -# request_path.append(self.path[:-1]) -# else: -# request_path.append(self.path) -# if path.startswith('/'): -# request_path.append(path[1:]) -# else: -# request_path.append(path) -# full_path = u'/'.join(request_path) -# -# if args: -# full_path += u"?%s" % (urlencode(args)) -# -# request_uri = u"%s://%s%s" % (self.scheme, self.host, full_path) -# -# auths = [(auth.depth(request_uri), auth) for auth in self.authorizations if auth.inscope(host, request_uri)] -# auth = auths and sorted(auths)[0][1] or None -# if auth: -# auth.request(method, request_uri, headers, body) -# -# (response, content) = self._conn_request(conn, request_uri, method, body, headers) -# -# if auth: -# if auth.response(response, body): -# auth.request(URLFETCH_METHOD_STRING[method], request_uri, headers, body) -# -# (response, content) = self._conn_request(conn, request_uri, method, body, headers) -# -# if response['status'] == 401: -# #return {u"body":u"".join(["%s: %s" % (key, response[key]) for key in response])} -# for authorization in self._auth_from_challenge(self.host, request_uri, headers, response, content): -# authorization.request(URLFETCH_METHOD_STRING[method], request_uri, headers, body) -# -# (response, content) = self._conn_request(conn, request_uri, method, body, headers) -# -# if response['status'] != 401: -# self.authorizations.append(authorization) -# authorization.response(response, body) -# break -# -# return {u'headers':response, u'body':content} - diff --git a/httplib2/__init__.py b/httplib2/__init__.py new file mode 100755 index 000000000..179a9f6b1 --- /dev/null +++ b/httplib2/__init__.py @@ -0,0 +1,1222 @@ +from __future__ import generators +""" +httplib2 + +A caching http interface that supports ETags and gzip +to conserve bandwidth. + +Requires Python 2.3 or later + +Changelog: +2007-08-18, Rick: Modified so it's able to use a socks proxy if needed. + +""" + +__author__ = "Joe Gregorio (joe@bitworking.org)" +__copyright__ = "Copyright 2006, Joe Gregorio" +__contributors__ = ["Thomas Broyer (t.broyer@ltgt.net)", + "James Antill", + "Xavier Verges Farrero", + "Jonathan Feinberg", + "Blair Zajac", + "Sam Ruby", + "Louis Nyffenegger"] +__license__ = "MIT" +__version__ = "$Rev$" + +import re +import sys +import email +import email.Utils +import email.Message +import email.FeedParser +import StringIO +import gzip +import zlib +import httplib +import urlparse +import base64 +import os +import copy +import calendar +import time +import random +# remove depracated warning in python2.6 +try: + from hashlib import sha1 as _sha, md5 as _md5 +except ImportError: + import sha + import md5 + _sha = sha.new + _md5 = md5.new +import hmac +from gettext import gettext as _ +import socket + +try: + import socks +except ImportError: + socks = None + +# Build the appropriate socket wrapper for ssl +try: + import ssl # python 2.6 + _ssl_wrap_socket = ssl.wrap_socket +except ImportError: + def _ssl_wrap_socket(sock, key_file, cert_file): + ssl_sock = socket.ssl(sock, key_file, cert_file) + return httplib.FakeSocket(sock, ssl_sock) + + +if sys.version_info >= (2,3): + from iri2uri import iri2uri +else: + def iri2uri(uri): + return uri + +def has_timeout(timeout): # python 2.6 + if hasattr(socket, '_GLOBAL_DEFAULT_TIMEOUT'): + return (timeout is not None and timeout is not socket._GLOBAL_DEFAULT_TIMEOUT) + return (timeout is not None) + +__all__ = ['Http', 'Response', 'ProxyInfo', 'HttpLib2Error', + 'RedirectMissingLocation', 'RedirectLimit', 'FailedToDecompressContent', + 'UnimplementedDigestAuthOptionError', 'UnimplementedHmacDigestAuthOptionError', + 'debuglevel'] + + +# The httplib debug level, set to a non-zero value to get debug output +debuglevel = 0 + + +# Python 2.3 support +if sys.version_info < (2,4): + def sorted(seq): + seq.sort() + return seq + +# Python 2.3 support +def HTTPResponse__getheaders(self): + """Return list of (header, value) tuples.""" + if self.msg is None: + raise httplib.ResponseNotReady() + return self.msg.items() + +if not hasattr(httplib.HTTPResponse, 'getheaders'): + httplib.HTTPResponse.getheaders = HTTPResponse__getheaders + +# All exceptions raised here derive from HttpLib2Error +class HttpLib2Error(Exception): pass + +# Some exceptions can be caught and optionally +# be turned back into responses. +class HttpLib2ErrorWithResponse(HttpLib2Error): + def __init__(self, desc, response, content): + self.response = response + self.content = content + HttpLib2Error.__init__(self, desc) + +class RedirectMissingLocation(HttpLib2ErrorWithResponse): pass +class RedirectLimit(HttpLib2ErrorWithResponse): pass +class FailedToDecompressContent(HttpLib2ErrorWithResponse): pass +class UnimplementedDigestAuthOptionError(HttpLib2ErrorWithResponse): pass +class UnimplementedHmacDigestAuthOptionError(HttpLib2ErrorWithResponse): pass + +class RelativeURIError(HttpLib2Error): pass +class ServerNotFoundError(HttpLib2Error): pass + +# Open Items: +# ----------- +# Proxy support + +# Are we removing the cached content too soon on PUT (only delete on 200 Maybe?) + +# Pluggable cache storage (supports storing the cache in +# flat files by default. We need a plug-in architecture +# that can support Berkeley DB and Squid) + +# == Known Issues == +# Does not handle a resource that uses conneg and Last-Modified but no ETag as a cache validator. +# Does not handle Cache-Control: max-stale +# Does not use Age: headers when calculating cache freshness. + + +# The number of redirections to follow before giving up. +# Note that only GET redirects are automatically followed. +# Will also honor 301 requests by saving that info and never +# requesting that URI again. +DEFAULT_MAX_REDIRECTS = 5 + +# Which headers are hop-by-hop headers by default +HOP_BY_HOP = ['connection', 'keep-alive', 'proxy-authenticate', 'proxy-authorization', 'te', 'trailers', 'transfer-encoding', 'upgrade'] + +def _get_end2end_headers(response): + hopbyhop = list(HOP_BY_HOP) + hopbyhop.extend([x.strip() for x in response.get('connection', '').split(',')]) + return [header for header in response.keys() if header not in hopbyhop] + +URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?") + +def parse_uri(uri): + """Parses a URI using the regex given in Appendix B of RFC 3986. + + (scheme, authority, path, query, fragment) = parse_uri(uri) + """ + groups = URI.match(uri).groups() + return (groups[1], groups[3], groups[4], groups[6], groups[8]) + +def urlnorm(uri): + (scheme, authority, path, query, fragment) = parse_uri(uri) + if not scheme or not authority: + raise RelativeURIError("Only absolute URIs are allowed. uri = %s" % uri) + authority = authority.lower() + scheme = scheme.lower() + if not path: + path = "/" + # Could do syntax based normalization of the URI before + # computing the digest. See Section 6.2.2 of Std 66. + request_uri = query and "?".join([path, query]) or path + scheme = scheme.lower() + defrag_uri = scheme + "://" + authority + request_uri + return scheme, authority, request_uri, defrag_uri + + +# Cache filename construction (original borrowed from Venus http://intertwingly.net/code/venus/) +re_url_scheme = re.compile(r'^\w+://') +re_slash = re.compile(r'[?/:|]+') + +def safename(filename): + """Return a filename suitable for the cache. + + Strips dangerous and common characters to create a filename we + can use to store the cache in. + """ + + try: + if re_url_scheme.match(filename): + if isinstance(filename,str): + filename = filename.decode('utf-8') + filename = filename.encode('idna') + else: + filename = filename.encode('idna') + except UnicodeError: + pass + if isinstance(filename,unicode): + filename=filename.encode('utf-8') + filemd5 = _md5(filename).hexdigest() + filename = re_url_scheme.sub("", filename) + filename = re_slash.sub(",", filename) + + # limit length of filename + if len(filename)>200: + filename=filename[:200] + return ",".join((filename, filemd5)) + +NORMALIZE_SPACE = re.compile(r'(?:\r\n)?[ \t]+') +def _normalize_headers(headers): + return dict([ (key.lower(), NORMALIZE_SPACE.sub(value, ' ').strip()) for (key, value) in headers.iteritems()]) + +def _parse_cache_control(headers): + retval = {} + if headers.has_key('cache-control'): + parts = headers['cache-control'].split(',') + parts_with_args = [tuple([x.strip().lower() for x in part.split("=", 1)]) for part in parts if -1 != part.find("=")] + parts_wo_args = [(name.strip().lower(), 1) for name in parts if -1 == name.find("=")] + retval = dict(parts_with_args + parts_wo_args) + return retval + +# Whether to use a strict mode to parse WWW-Authenticate headers +# Might lead to bad results in case of ill-formed header value, +# so disabled by default, falling back to relaxed parsing. +# Set to true to turn on, usefull for testing servers. +USE_WWW_AUTH_STRICT_PARSING = 0 + +# In regex below: +# [^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+ matches a "token" as defined by HTTP +# "(?:[^\0-\x08\x0A-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?" matches a "quoted-string" as defined by HTTP, when LWS have already been replaced by a single space +# Actually, as an auth-param value can be either a token or a quoted-string, they are combined in a single pattern which matches both: +# \"?((?<=\")(?:[^\0-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?(?=\")|(?@,;:\\\"/[\]?={} \t]+(?!\"))\"? +WWW_AUTH_STRICT = re.compile(r"^(?:\s*(?:,\s*)?([^\0-\x1f\x7f-\xff()<>@,;:\\\"/[\]?={} \t]+)\s*=\s*\"?((?<=\")(?:[^\0-\x08\x0A-\x1f\x7f-\xff\\\"]|\\[\0-\x7f])*?(?=\")|(?@,;:\\\"/[\]?={} \t]+(?!\"))\"?)(.*)$") +WWW_AUTH_RELAXED = re.compile(r"^(?:\s*(?:,\s*)?([^ \t\r\n=]+)\s*=\s*\"?((?<=\")(?:[^\\\"]|\\.)*?(?=\")|(? current_age: + retval = "FRESH" + return retval + +def _decompressContent(response, new_content): + content = new_content + try: + encoding = response.get('content-encoding', None) + if encoding in ['gzip', 'deflate']: + if encoding == 'gzip': + content = gzip.GzipFile(fileobj=StringIO.StringIO(new_content)).read() + if encoding == 'deflate': + content = zlib.decompress(content) + response['content-length'] = str(len(content)) + # Record the historical presence of the encoding in a way the won't interfere. + response['-content-encoding'] = response['content-encoding'] + del response['content-encoding'] + except IOError: + content = "" + raise FailedToDecompressContent(_("Content purported to be compressed with %s but failed to decompress.") % response.get('content-encoding'), response, content) + return content + +def _updateCache(request_headers, response_headers, content, cache, cachekey): + if cachekey: + cc = _parse_cache_control(request_headers) + cc_response = _parse_cache_control(response_headers) + if cc.has_key('no-store') or cc_response.has_key('no-store'): + cache.delete(cachekey) + else: + info = email.Message.Message() + for key, value in response_headers.iteritems(): + if key not in ['status','content-encoding','transfer-encoding']: + info[key] = value + + # Add annotations to the cache to indicate what headers + # are variant for this request. + vary = response_headers.get('vary', None) + if vary: + vary_headers = vary.lower().replace(' ', '').split(',') + for header in vary_headers: + key = '-varied-%s' % header + try: + info[key] = request_headers[header] + except KeyError: + pass + + status = response_headers.status + if status == 304: + status = 200 + + status_header = 'status: %d\r\n' % response_headers.status + + header_str = info.as_string() + + header_str = re.sub("\r(?!\n)|(? 0: + service = "cl" + # No point in guessing Base or Spreadsheet + #elif request_uri.find("spreadsheets") > 0: + # service = "wise" + + auth = dict(Email=credentials[0], Passwd=credentials[1], service=service, source=headers['user-agent']) + resp, content = self.http.request("https://www.google.com/accounts/ClientLogin", method="POST", body=urlencode(auth), headers={'Content-Type': 'application/x-www-form-urlencoded'}) + lines = content.split('\n') + d = dict([tuple(line.split("=", 1)) for line in lines if line]) + if resp.status == 403: + self.Auth = "" + else: + self.Auth = d['Auth'] + + def request(self, method, request_uri, headers, content): + """Modify the request headers to add the appropriate + Authorization header.""" + headers['authorization'] = 'GoogleLogin Auth=' + self.Auth + + +AUTH_SCHEME_CLASSES = { + "basic": BasicAuthentication, + "wsse": WsseAuthentication, + "digest": DigestAuthentication, + "hmacdigest": HmacDigestAuthentication, + "googlelogin": GoogleLoginAuthentication +} + +AUTH_SCHEME_ORDER = ["hmacdigest", "googlelogin", "digest", "wsse", "basic"] + +class FileCache(object): + """Uses a local directory as a store for cached files. + Not really safe to use if multiple threads or processes are going to + be running on the same cache. + """ + def __init__(self, cache, safe=safename): # use safe=lambda x: md5.new(x).hexdigest() for the old behavior + self.cache = cache + self.safe = safe + if not os.path.exists(cache): + os.makedirs(self.cache) + + def get(self, key): + retval = None + cacheFullPath = os.path.join(self.cache, self.safe(key)) + try: + f = file(cacheFullPath, "rb") + retval = f.read() + f.close() + except IOError: + pass + return retval + + def set(self, key, value): + cacheFullPath = os.path.join(self.cache, self.safe(key)) + f = file(cacheFullPath, "wb") + f.write(value) + f.close() + + def delete(self, key): + cacheFullPath = os.path.join(self.cache, self.safe(key)) + if os.path.exists(cacheFullPath): + os.remove(cacheFullPath) + +class Credentials(object): + def __init__(self): + self.credentials = [] + + def add(self, name, password, domain=""): + self.credentials.append((domain.lower(), name, password)) + + def clear(self): + self.credentials = [] + + def iter(self, domain): + for (cdomain, name, password) in self.credentials: + if cdomain == "" or domain == cdomain: + yield (name, password) + +class KeyCerts(Credentials): + """Identical to Credentials except that + name/password are mapped to key/cert.""" + pass + + +class ProxyInfo(object): + """Collect information required to use a proxy.""" + def __init__(self, proxy_type, proxy_host, proxy_port, proxy_rdns=None, proxy_user=None, proxy_pass=None): + """The parameter proxy_type must be set to one of socks.PROXY_TYPE_XXX + constants. For example: + +p = ProxyInfo(proxy_type=socks.PROXY_TYPE_HTTP, proxy_host='localhost', proxy_port=8000) + """ + self.proxy_type, self.proxy_host, self.proxy_port, self.proxy_rdns, self.proxy_user, self.proxy_pass = proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass + + def astuple(self): + return (self.proxy_type, self.proxy_host, self.proxy_port, self.proxy_rdns, + self.proxy_user, self.proxy_pass) + + def isgood(self): + return socks and (self.proxy_host != None) and (self.proxy_port != None) + + +class HTTPConnectionWithTimeout(httplib.HTTPConnection): + """HTTPConnection subclass that supports timeouts""" + + def __init__(self, host, port=None, strict=None, timeout=None, proxy_info=None): + httplib.HTTPConnection.__init__(self, host, port, strict) + self.timeout = timeout + self.proxy_info = proxy_info + + def connect(self): + """Connect to the host and port specified in __init__.""" + # Mostly verbatim from httplib.py. + msg = "getaddrinfo returns an empty list" + for res in socket.getaddrinfo(self.host, self.port, 0, + socket.SOCK_STREAM): + af, socktype, proto, canonname, sa = res + try: + if self.proxy_info and self.proxy_info.isgood(): + self.sock = socks.socksocket(af, socktype, proto) + # HACK: amorton enabled TCP_NODELAY on socket + self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + self.sock.setproxy(*self.proxy_info.astuple()) + else: + self.sock = socket.socket(af, socktype, proto) + # HACK: amorton enabled TCP_NODELAY on socket + self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + # Different from httplib: support timeouts. + if has_timeout(self.timeout): + self.sock.settimeout(self.timeout) + # End of difference from httplib. + if self.debuglevel > 0: + print "connect: (%s, %s)" % (self.host, self.port) + + self.sock.connect(sa) + except socket.error, msg: + if self.debuglevel > 0: + print 'connect fail:', (self.host, self.port) + if self.sock: + self.sock.close() + self.sock = None + continue + break + if not self.sock: + raise socket.error, msg + +class HTTPSConnectionWithTimeout(httplib.HTTPSConnection): + "This class allows communication via SSL." + + def __init__(self, host, port=None, key_file=None, cert_file=None, + strict=None, timeout=None, proxy_info=None): + httplib.HTTPSConnection.__init__(self, host, port=port, key_file=key_file, + cert_file=cert_file, strict=strict) + self.timeout = timeout + self.proxy_info = proxy_info + + def connect(self): + "Connect to a host on a given (SSL) port." + + if self.proxy_info and self.proxy_info.isgood(): + sock = socks.socksocket(socket.AF_INET, socket.SOCK_STREAM) + # HACK: amorton enabled TCP_NODLEAY on socket + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + sock.setproxy(*self.proxy_info.astuple()) + else: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + # HACK: amorton enabled TCP_NODLEAY on socket + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + + if has_timeout(self.timeout): + sock.settimeout(self.timeout) + sock.connect((self.host, self.port)) + self.sock =_ssl_wrap_socket(sock, self.key_file, self.cert_file) + + + +class Http(object): + """An HTTP client that handles: +- all methods +- caching +- ETags +- compression, +- HTTPS +- Basic +- Digest +- WSSE + +and more. + """ + def __init__(self, cache=None, timeout=None, proxy_info=None): + """The value of proxy_info is a ProxyInfo instance. + +If 'cache' is a string then it is used as a directory name +for a disk cache. Otherwise it must be an object that supports +the same interface as FileCache.""" + self.proxy_info = proxy_info + # Map domain name to an httplib connection + self.connections = {} + # The location of the cache, for now a directory + # where cached responses are held. + if cache and isinstance(cache, str): + self.cache = FileCache(cache) + else: + self.cache = cache + + # Name/password + self.credentials = Credentials() + + # Key/cert + self.certificates = KeyCerts() + + # authorization objects + self.authorizations = [] + + # If set to False then no redirects are followed, even safe ones. + self.follow_redirects = True + + # Which HTTP methods do we apply optimistic concurrency to, i.e. + # which methods get an "if-match:" etag header added to them. + self.optimistic_concurrency_methods = ["PUT"] + + # If 'follow_redirects' is True, and this is set to True then + # all redirecs are followed, including unsafe ones. + self.follow_all_redirects = False + + self.ignore_etag = False + + self.force_exception_to_status_code = False + + self.timeout = timeout + + def _auth_from_challenge(self, host, request_uri, headers, response, content): + """A generator that creates Authorization objects + that can be applied to requests. + """ + challenges = _parse_www_authenticate(response, 'www-authenticate') + for cred in self.credentials.iter(host): + for scheme in AUTH_SCHEME_ORDER: + if challenges.has_key(scheme): + yield AUTH_SCHEME_CLASSES[scheme](cred, host, request_uri, headers, response, content, self) + + def add_credentials(self, name, password, domain=""): + """Add a name and password that will be used + any time a request requires authentication.""" + self.credentials.add(name, password, domain) + + def add_certificate(self, key, cert, domain): + """Add a key and cert that will be used + any time a request requires authentication.""" + self.certificates.add(key, cert, domain) + + def clear_credentials(self): + """Remove all the names and passwords + that are used for authentication""" + self.credentials.clear() + self.authorizations = [] + + def _conn_request(self, conn, request_uri, method, body, headers): + for i in range(2): + try: + conn.request(method, request_uri, body, headers) + except socket.gaierror: + conn.close() + raise ServerNotFoundError("Unable to find the server at %s" % conn.host) + except (socket.error, httplib.HTTPException): + # Just because the server closed the connection doesn't apparently mean + # that the server didn't send a response. + # amorton: patch from http://code.google.com/p/httplib2/source/detail?r=0cff83696d + if conn.sock is None: + if i == 0: + conn.close() + conn.connect() + continue + else: + conn.close() + raise + if i == 0: + conn.close() + conn.connect() + continue + try: + response = conn.getresponse() + except (socket.error, httplib.HTTPException): + if i == 0: + conn.close() + conn.connect() + continue + else: + raise + else: + content = "" + if method == "HEAD": + response.close() + else: + content = response.read() + response = Response(response) + if method != "HEAD": + content = _decompressContent(response, content) + break + return (response, content) + + + def _request(self, conn, host, absolute_uri, request_uri, method, body, headers, redirections, cachekey): + """Do the actual request using the connection object + and also follow one level of redirects if necessary""" + + auths = [(auth.depth(request_uri), auth) for auth in self.authorizations if auth.inscope(host, request_uri)] + auth = auths and sorted(auths)[0][1] or None + if auth: + auth.request(method, request_uri, headers, body) + + (response, content) = self._conn_request(conn, request_uri, method, body, headers) + + if auth: + if auth.response(response, body): + auth.request(method, request_uri, headers, body) + (response, content) = self._conn_request(conn, request_uri, method, body, headers ) + response._stale_digest = 1 + + if response.status == 401: + for authorization in self._auth_from_challenge(host, request_uri, headers, response, content): + authorization.request(method, request_uri, headers, body) + (response, content) = self._conn_request(conn, request_uri, method, body, headers, ) + if response.status != 401: + self.authorizations.append(authorization) + authorization.response(response, body) + break + + if (self.follow_all_redirects or (method in ["GET", "HEAD"]) or response.status == 303): + if self.follow_redirects and response.status in [300, 301, 302, 303, 307]: + # Pick out the location header and basically start from the beginning + # remembering first to strip the ETag header and decrement our 'depth' + if redirections: + if not response.has_key('location') and response.status != 300: + raise RedirectMissingLocation( _("Redirected but the response is missing a Location: header."), response, content) + # Fix-up relative redirects (which violate an RFC 2616 MUST) + if response.has_key('location'): + location = response['location'] + (scheme, authority, path, query, fragment) = parse_uri(location) + if authority == None: + response['location'] = urlparse.urljoin(absolute_uri, location) + if response.status == 301 and method in ["GET", "HEAD"]: + response['-x-permanent-redirect-url'] = response['location'] + if not response.has_key('content-location'): + response['content-location'] = absolute_uri + _updateCache(headers, response, content, self.cache, cachekey) + if headers.has_key('if-none-match'): + del headers['if-none-match'] + if headers.has_key('if-modified-since'): + del headers['if-modified-since'] + if response.has_key('location'): + location = response['location'] + old_response = copy.deepcopy(response) + if not old_response.has_key('content-location'): + old_response['content-location'] = absolute_uri + redirect_method = ((response.status == 303) and (method not in ["GET", "HEAD"])) and "GET" or method + (response, content) = self.request(location, redirect_method, body=body, headers = headers, redirections = redirections - 1) + response.previous = old_response + else: + raise RedirectLimit( _("Redirected more times than rediection_limit allows."), response, content) + elif response.status in [200, 203] and method == "GET": + # Don't cache 206's since we aren't going to handle byte range requests + if not response.has_key('content-location'): + response['content-location'] = absolute_uri + _updateCache(headers, response, content, self.cache, cachekey) + + return (response, content) + + def _normalize_headers(self, headers): + return _normalize_headers(headers) + +# Need to catch and rebrand some exceptions +# Then need to optionally turn all exceptions into status codes +# including all socket.* and httplib.* exceptions. + + + def request(self, uri, method="GET", body=None, headers=None, redirections=DEFAULT_MAX_REDIRECTS, connection_type=None): + """ Performs a single HTTP request. +The 'uri' is the URI of the HTTP resource and can begin +with either 'http' or 'https'. The value of 'uri' must be an absolute URI. + +The 'method' is the HTTP method to perform, such as GET, POST, DELETE, etc. +There is no restriction on the methods allowed. + +The 'body' is the entity body to be sent with the request. It is a string +object. + +Any extra headers that are to be sent with the request should be provided in the +'headers' dictionary. + +The maximum number of redirect to follow before raising an +exception is 'redirections. The default is 5. + +The return value is a tuple of (response, content), the first +being and instance of the 'Response' class, the second being +a string that contains the response entity body. + """ + try: + if headers is None: + headers = {} + else: + headers = self._normalize_headers(headers) + + if not headers.has_key('user-agent'): + headers['user-agent'] = "Python-httplib2/%s" % __version__ + + uri = iri2uri(uri) + + (scheme, authority, request_uri, defrag_uri) = urlnorm(uri) + domain_port = authority.split(":")[0:2] + if len(domain_port) == 2 and domain_port[1] == '443' and scheme == 'http': + scheme = 'https' + authority = domain_port[0] + + conn_key = scheme+":"+authority + if conn_key in self.connections: + conn = self.connections[conn_key] + else: + if not connection_type: + connection_type = (scheme == 'https') and HTTPSConnectionWithTimeout or HTTPConnectionWithTimeout + certs = list(self.certificates.iter(authority)) + if scheme == 'https' and certs: + conn = self.connections[conn_key] = connection_type(authority, key_file=certs[0][0], + cert_file=certs[0][1], timeout=self.timeout, proxy_info=self.proxy_info) + else: + conn = self.connections[conn_key] = connection_type(authority, timeout=self.timeout, proxy_info=self.proxy_info) + conn.set_debuglevel(debuglevel) + + if method in ["GET", "HEAD"] and 'range' not in headers and 'accept-encoding' not in headers: + headers['accept-encoding'] = 'gzip, deflate' + + info = email.Message.Message() + cached_value = None + if self.cache: + cachekey = defrag_uri + cached_value = self.cache.get(cachekey) + if cached_value: + # info = email.message_from_string(cached_value) + # + # Need to replace the line above with the kludge below + # to fix the non-existent bug not fixed in this + # bug report: http://mail.python.org/pipermail/python-bugs-list/2005-September/030289.html + try: + info, content = cached_value.split('\r\n\r\n', 1) + feedparser = email.FeedParser.FeedParser() + feedparser.feed(info) + info = feedparser.close() + feedparser._parse = None + except IndexError: + self.cache.delete(cachekey) + cachekey = None + cached_value = None + else: + cachekey = None + + if method in self.optimistic_concurrency_methods and self.cache and info.has_key('etag') and not self.ignore_etag and 'if-match' not in headers: + # http://www.w3.org/1999/04/Editing/ + headers['if-match'] = info['etag'] + + if method not in ["GET", "HEAD"] and self.cache and cachekey: + # RFC 2616 Section 13.10 + self.cache.delete(cachekey) + + # Check the vary header in the cache to see if this request + # matches what varies in the cache. + if method in ['GET', 'HEAD'] and 'vary' in info: + vary = info['vary'] + vary_headers = vary.lower().replace(' ', '').split(',') + for header in vary_headers: + key = '-varied-%s' % header + value = info[key] + if headers.get(header, '') != value: + cached_value = None + break + + if cached_value and method in ["GET", "HEAD"] and self.cache and 'range' not in headers: + if info.has_key('-x-permanent-redirect-url'): + # Should cached permanent redirects be counted in our redirection count? For now, yes. + (response, new_content) = self.request(info['-x-permanent-redirect-url'], "GET", headers = headers, redirections = redirections - 1) + response.previous = Response(info) + response.previous.fromcache = True + else: + # Determine our course of action: + # Is the cached entry fresh or stale? + # Has the client requested a non-cached response? + # + # There seems to be three possible answers: + # 1. [FRESH] Return the cache entry w/o doing a GET + # 2. [STALE] Do the GET (but add in cache validators if available) + # 3. [TRANSPARENT] Do a GET w/o any cache validators (Cache-Control: no-cache) on the request + entry_disposition = _entry_disposition(info, headers) + + if entry_disposition == "FRESH": + if not cached_value: + info['status'] = '504' + content = "" + response = Response(info) + if cached_value: + response.fromcache = True + return (response, content) + + if entry_disposition == "STALE": + if info.has_key('etag') and not self.ignore_etag and not 'if-none-match' in headers: + headers['if-none-match'] = info['etag'] + if info.has_key('last-modified') and not 'last-modified' in headers: + headers['if-modified-since'] = info['last-modified'] + elif entry_disposition == "TRANSPARENT": + pass + + (response, new_content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey) + + if response.status == 304 and method == "GET": + # Rewrite the cache entry with the new end-to-end headers + # Take all headers that are in response + # and overwrite their values in info. + # unless they are hop-by-hop, or are listed in the connection header. + + for key in _get_end2end_headers(response): + info[key] = response[key] + merged_response = Response(info) + if hasattr(response, "_stale_digest"): + merged_response._stale_digest = response._stale_digest + _updateCache(headers, merged_response, content, self.cache, cachekey) + response = merged_response + response.status = 200 + response.fromcache = True + + elif response.status == 200: + content = new_content + else: + self.cache.delete(cachekey) + content = new_content + else: + cc = _parse_cache_control(headers) + if cc.has_key('only-if-cached'): + info['status'] = '504' + response = Response(info) + content = "" + else: + (response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey) + except Exception, e: + if self.force_exception_to_status_code: + if isinstance(e, HttpLib2ErrorWithResponse): + response = e.response + content = e.content + response.status = 500 + response.reason = str(e) + elif isinstance(e, socket.timeout): + content = "Request Timeout" + response = Response( { + "content-type": "text/plain", + "status": "408", + "content-length": len(content) + }) + response.reason = "Request Timeout" + else: + content = str(e) + response = Response( { + "content-type": "text/plain", + "status": "400", + "content-length": len(content) + }) + response.reason = "Bad Request" + else: + raise + + + return (response, content) + + + +class Response(dict): + """An object more like email.Message than httplib.HTTPResponse.""" + + """Is this response from our local cache""" + fromcache = False + + """HTTP protocol version used by server. 10 for HTTP/1.0, 11 for HTTP/1.1. """ + version = 11 + + "Status code returned by server. " + status = 200 + + """Reason phrase returned by server.""" + reason = "Ok" + + previous = None + + def __init__(self, info): + # info is either an email.Message or + # an httplib.HTTPResponse object. + if isinstance(info, httplib.HTTPResponse): + for key, value in info.getheaders(): + self[key.lower()] = value + self.status = info.status + self['status'] = str(self.status) + self.reason = info.reason + self.version = info.version + elif isinstance(info, email.Message.Message): + for key, value in info.items(): + self[key] = value + self.status = int(self['status']) + else: + for key, value in info.iteritems(): + self[key] = value + self.status = int(self.get('status', self.status)) + + + def __getattr__(self, name): + if name == 'dict': + return self + else: + raise AttributeError, name diff --git a/httplib2/iri2uri.py b/httplib2/iri2uri.py new file mode 100755 index 000000000..70667edf8 --- /dev/null +++ b/httplib2/iri2uri.py @@ -0,0 +1,110 @@ +""" +iri2uri + +Converts an IRI to a URI. + +""" +__author__ = "Joe Gregorio (joe@bitworking.org)" +__copyright__ = "Copyright 2006, Joe Gregorio" +__contributors__ = [] +__version__ = "1.0.0" +__license__ = "MIT" +__history__ = """ +""" + +import urlparse + + +# Convert an IRI to a URI following the rules in RFC 3987 +# +# The characters we need to enocde and escape are defined in the spec: +# +# iprivate = %xE000-F8FF / %xF0000-FFFFD / %x100000-10FFFD +# ucschar = %xA0-D7FF / %xF900-FDCF / %xFDF0-FFEF +# / %x10000-1FFFD / %x20000-2FFFD / %x30000-3FFFD +# / %x40000-4FFFD / %x50000-5FFFD / %x60000-6FFFD +# / %x70000-7FFFD / %x80000-8FFFD / %x90000-9FFFD +# / %xA0000-AFFFD / %xB0000-BFFFD / %xC0000-CFFFD +# / %xD0000-DFFFD / %xE1000-EFFFD + +escape_range = [ + (0xA0, 0xD7FF ), + (0xE000, 0xF8FF ), + (0xF900, 0xFDCF ), + (0xFDF0, 0xFFEF), + (0x10000, 0x1FFFD ), + (0x20000, 0x2FFFD ), + (0x30000, 0x3FFFD), + (0x40000, 0x4FFFD ), + (0x50000, 0x5FFFD ), + (0x60000, 0x6FFFD), + (0x70000, 0x7FFFD ), + (0x80000, 0x8FFFD ), + (0x90000, 0x9FFFD), + (0xA0000, 0xAFFFD ), + (0xB0000, 0xBFFFD ), + (0xC0000, 0xCFFFD), + (0xD0000, 0xDFFFD ), + (0xE1000, 0xEFFFD), + (0xF0000, 0xFFFFD ), + (0x100000, 0x10FFFD) +] + +def encode(c): + retval = c + i = ord(c) + for low, high in escape_range: + if i < low: + break + if i >= low and i <= high: + retval = "".join(["%%%2X" % ord(o) for o in c.encode('utf-8')]) + break + return retval + + +def iri2uri(uri): + """Convert an IRI to a URI. Note that IRIs must be + passed in a unicode strings. That is, do not utf-8 encode + the IRI before passing it into the function.""" + if isinstance(uri ,unicode): + (scheme, authority, path, query, fragment) = urlparse.urlsplit(uri) + authority = authority.encode('idna') + # For each character in 'ucschar' or 'iprivate' + # 1. encode as utf-8 + # 2. then %-encode each octet of that utf-8 + uri = urlparse.urlunsplit((scheme, authority, path, query, fragment)) + uri = "".join([encode(c) for c in uri]) + return uri + +if __name__ == "__main__": + import unittest + + class Test(unittest.TestCase): + + def test_uris(self): + """Test that URIs are invariant under the transformation.""" + invariant = [ + u"ftp://ftp.is.co.za/rfc/rfc1808.txt", + u"http://www.ietf.org/rfc/rfc2396.txt", + u"ldap://[2001:db8::7]/c=GB?objectClass?one", + u"mailto:John.Doe@example.com", + u"news:comp.infosystems.www.servers.unix", + u"tel:+1-816-555-1212", + u"telnet://192.0.2.16:80/", + u"urn:oasis:names:specification:docbook:dtd:xml:4.1.2" ] + for uri in invariant: + self.assertEqual(uri, iri2uri(uri)) + + def test_iri(self): + """ Test that the right type of escaping is done for each part of the URI.""" + self.assertEqual("http://xn--o3h.com/%E2%98%84", iri2uri(u"http://\N{COMET}.com/\N{COMET}")) + self.assertEqual("http://bitworking.org/?fred=%E2%98%84", iri2uri(u"http://bitworking.org/?fred=\N{COMET}")) + self.assertEqual("http://bitworking.org/#%E2%98%84", iri2uri(u"http://bitworking.org/#\N{COMET}")) + self.assertEqual("#%E2%98%84", iri2uri(u"#\N{COMET}")) + self.assertEqual("/fred?bar=%E2%98%9A#%E2%98%84", iri2uri(u"/fred?bar=\N{BLACK LEFT POINTING INDEX}#\N{COMET}")) + self.assertEqual("/fred?bar=%E2%98%9A#%E2%98%84", iri2uri(iri2uri(u"/fred?bar=\N{BLACK LEFT POINTING INDEX}#\N{COMET}"))) + self.assertNotEqual("/fred?bar=%E2%98%9A#%E2%98%84", iri2uri(u"/fred?bar=\N{BLACK LEFT POINTING INDEX}#\N{COMET}".encode('utf-8'))) + + unittest.main() + + diff --git a/shotgun_api3.py b/shotgun_api3.py index 131503664..1078d6b18 100755 --- a/shotgun_api3.py +++ b/shotgun_api3.py @@ -28,9 +28,6 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# needed for httplib2, future imports must be first -from __future__ import generators - import base64 import cookielib # used for attachment upload import cStringIO # used for attachment upload @@ -48,10 +45,7 @@ import urllib2 # used for image upload import urlparse from socks import * -from gae_restful_lib import (Credentials, WsseAuthentication, - HmacDigestAuthentication, DigestAuthentication, - BasicAuthentication, Authentication, - GoogleLoginAuthentication) +from httplib2 import Http log = logging.getLogger("shotgun_api3") @@ -1377,1059 +1371,6 @@ def _isdst(self, dt): sg_timezone = SgTimezone() - - - - - - - - - - - - - -# ---------------------------------------------------------------------------- -# Included external modules, left as is or otherwise noted here an in the code -# search for "amorton" -# -# - httplib2: contents from __init__.py and iri2uri.py -# - removed reference to the socks module, it is included directly. -# - set TCP_NODELAY on the sockets see -# http://code.google.com/p/httplib2/issues/detail?id=28 -# - patch to handle socket not opening applied -# see http://code.google.com/p/httplib2/source/detail?r=0cff83696d -# - socks module from http://socksipy.sourceforge.net/ -# - -# ---------------------------------------------------------------------------- -# httplib2/iri2uri.py - -import urlparse - - -# Convert an IRI to a URI following the rules in RFC 3987 -# -# The characters we need to enocde and escape are defined in the spec: -# -# iprivate = %xE000-F8FF / %xF0000-FFFFD / %x100000-10FFFD -# ucschar = %xA0-D7FF / %xF900-FDCF / %xFDF0-FFEF -# / %x10000-1FFFD / %x20000-2FFFD / %x30000-3FFFD -# / %x40000-4FFFD / %x50000-5FFFD / %x60000-6FFFD -# / %x70000-7FFFD / %x80000-8FFFD / %x90000-9FFFD -# / %xA0000-AFFFD / %xB0000-BFFFD / %xC0000-CFFFD -# / %xD0000-DFFFD / %xE1000-EFFFD - -escape_range = [ - (0xA0, 0xD7FF ), - (0xE000, 0xF8FF ), - (0xF900, 0xFDCF ), - (0xFDF0, 0xFFEF), - (0x10000, 0x1FFFD ), - (0x20000, 0x2FFFD ), - (0x30000, 0x3FFFD), - (0x40000, 0x4FFFD ), - (0x50000, 0x5FFFD ), - (0x60000, 0x6FFFD), - (0x70000, 0x7FFFD ), - (0x80000, 0x8FFFD ), - (0x90000, 0x9FFFD), - (0xA0000, 0xAFFFD ), - (0xB0000, 0xBFFFD ), - (0xC0000, 0xCFFFD), - (0xD0000, 0xDFFFD ), - (0xE1000, 0xEFFFD), - (0xF0000, 0xFFFFD ), - (0x100000, 0x10FFFD) -] - -def encode(c): - retval = c - i = ord(c) - for low, high in escape_range: - if i < low: - break - if i >= low and i <= high: - retval = "".join(["%%%2X" % ord(o) for o in c.encode('utf-8')]) - break - return retval - - -def iri2uri(uri): - """Convert an IRI to a URI. Note that IRIs must be - passed in a unicode strings. That is, do not utf-8 encode - the IRI before passing it into the function.""" - if isinstance(uri ,unicode): - (scheme, authority, path, query, fragment) = urlparse.urlsplit(uri) - authority = authority.encode('idna') - # For each character in 'ucschar' or 'iprivate' - # 1. encode as utf-8 - # 2. then %-encode each octet of that utf-8 - uri = urlparse.urlunsplit((scheme, authority, path, query, fragment)) - uri = "".join([encode(c) for c in uri]) - return uri - -# ---------------------------------------------------------------------------- -# httplib2/__init__.py - - -# HACK: Amorton: future import moved to top of file -""" -httplib2 - -A caching http interface that supports ETags and gzip -to conserve bandwidth. - -Requires Python 2.3 or later - -Changelog: -2007-08-18, Rick: Modified so it's able to use a socks proxy if needed. - -""" - -__author__ = "Joe Gregorio (joe@bitworking.org)" -__copyright__ = "Copyright 2006, Joe Gregorio" -__contributors__ = ["Thomas Broyer (t.broyer@ltgt.net)", - "James Antill", - "Xavier Verges Farrero", - "Jonathan Feinberg", - "Blair Zajac", - "Sam Ruby", - "Louis Nyffenegger"] -__license__ = "MIT" -__version__ = "$Rev$" - -import re -import sys -import email -import email.Utils -import email.Message -import email.FeedParser -import StringIO -import gzip -import zlib -import httplib -import urlparse -import base64 -import os -import copy -import calendar -import time -import random -# remove depracated warning in python2.6 -try: - from hashlib import sha1 as _sha, md5 as _md5 -except ImportError: - import sha - import md5 - _sha = sha.new - _md5 = md5.new -import hmac -from gettext import gettext as _ -import socket - - -# Build the appropriate socket wrapper for ssl -try: - import ssl # python 2.6 - _ssl_wrap_socket = ssl.wrap_socket -except ImportError: - def _ssl_wrap_socket(sock, key_file, cert_file): - ssl_sock = socket.ssl(sock, key_file, cert_file) - return httplib.FakeSocket(sock, ssl_sock) - - -if sys.version_info >= (2,3): - # amorton: code pulled into this file, see above - #from iri2uri import iri2uri - pass -else: - def iri2uri(uri): - return uri - -def has_timeout(timeout): # python 2.6 - if hasattr(socket, '_GLOBAL_DEFAULT_TIMEOUT'): - return (timeout is not None and timeout is not socket._GLOBAL_DEFAULT_TIMEOUT) - return (timeout is not None) - -__all__ = ['Http', 'Response', 'ProxyInfo', 'HttpLib2Error', - 'RedirectMissingLocation', 'RedirectLimit', 'FailedToDecompressContent', - 'UnimplementedDigestAuthOptionError', 'UnimplementedHmacDigestAuthOptionError', - 'debuglevel'] - - -# The httplib debug level, set to a non-zero value to get debug output -debuglevel = 0 - -#TODO do we still support python 2.3? - -# Python 2.3 support -if sys.version_info < (2,4): - def sorted(seq): - seq.sort() - return seq - -# Python 2.3 support -def HTTPResponse__getheaders(self): - """Return list of (header, value) tuples.""" - if self.msg is None: - raise httplib.ResponseNotReady() - return self.msg.items() - -if not hasattr(httplib.HTTPResponse, 'getheaders'): - httplib.HTTPResponse.getheaders = HTTPResponse__getheaders - -# All exceptions raised here derive from HttpLib2Error -class HttpLib2Error(Exception): pass - -# Some exceptions can be caught and optionally -# be turned back into responses. -class HttpLib2ErrorWithResponse(HttpLib2Error): - def __init__(self, desc, response, content): - self.response = response - self.content = content - HttpLib2Error.__init__(self, desc) - -class RedirectMissingLocation(HttpLib2ErrorWithResponse): pass -class RedirectLimit(HttpLib2ErrorWithResponse): pass -class FailedToDecompressContent(HttpLib2ErrorWithResponse): pass -class UnimplementedDigestAuthOptionError(HttpLib2ErrorWithResponse): pass -class UnimplementedHmacDigestAuthOptionError(HttpLib2ErrorWithResponse): pass - -class RelativeURIError(HttpLib2Error): pass -class ServerNotFoundError(HttpLib2Error): pass - -# Open Items: -# ----------- -# Proxy support - -# Are we removing the cached content too soon on PUT (only delete on 200 Maybe?) - -# Pluggable cache storage (supports storing the cache in -# flat files by default. We need a plug-in architecture -# that can support Berkeley DB and Squid) - -# == Known Issues == -# Does not handle a resource that uses conneg and Last-Modified but no ETag as a cache validator. -# Does not handle Cache-Control: max-stale -# Does not use Age: headers when calculating cache freshness. - - -# The number of redirections to follow before giving up. -# Note that only GET redirects are automatically followed. -# Will also honor 301 requests by saving that info and never -# requesting that URI again. -DEFAULT_MAX_REDIRECTS = 5 - -# Which headers are hop-by-hop headers by default -HOP_BY_HOP = ['connection', 'keep-alive', 'proxy-authenticate', 'proxy-authorization', 'te', 'trailers', 'transfer-encoding', 'upgrade'] - -def _get_end2end_headers(response): - hopbyhop = list(HOP_BY_HOP) - hopbyhop.extend([x.strip() for x in response.get('connection', '').split(',')]) - return [header for header in response.keys() if header not in hopbyhop] - -URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?") - -def parse_uri(uri): - """Parses a URI using the regex given in Appendix B of RFC 3986. - - (scheme, authority, path, query, fragment) = parse_uri(uri) - """ - groups = URI.match(uri).groups() - return (groups[1], groups[3], groups[4], groups[6], groups[8]) - -def urlnorm(uri): - (scheme, authority, path, query, fragment) = parse_uri(uri) - if not scheme or not authority: - raise RelativeURIError("Only absolute URIs are allowed. uri = %s" % uri) - authority = authority.lower() - scheme = scheme.lower() - if not path: - path = "/" - # Could do syntax based normalization of the URI before - # computing the digest. See Section 6.2.2 of Std 66. - request_uri = query and "?".join([path, query]) or path - scheme = scheme.lower() - defrag_uri = scheme + "://" + authority + request_uri - return scheme, authority, request_uri, defrag_uri - - -# Cache filename construction (original borrowed from Venus http://intertwingly.net/code/venus/) -re_url_scheme = re.compile(r'^\w+://') -re_slash = re.compile(r'[?/:|]+') - -def safename(filename): - """Return a filename suitable for the cache. - - Strips dangerous and common characters to create a filename we - can use to store the cache in. - """ - - try: - if re_url_scheme.match(filename): - if isinstance(filename,str): - filename = filename.decode('utf-8') - filename = filename.encode('idna') - else: - filename = filename.encode('idna') - except UnicodeError: - pass - if isinstance(filename,unicode): - filename=filename.encode('utf-8') - filemd5 = _md5(filename).hexdigest() - filename = re_url_scheme.sub("", filename) - filename = re_slash.sub(",", filename) - - # limit length of filename - if len(filename)>200: - filename=filename[:200] - return ",".join((filename, filemd5)) - -NORMALIZE_SPACE = re.compile(r'(?:\r\n)?[ \t]+') -def _normalize_headers(headers): - return dict([ (key.lower(), NORMALIZE_SPACE.sub(value, ' ').strip()) for (key, value) in headers.iteritems()]) - -def _parse_cache_control(headers): - retval = {} - if headers.has_key('cache-control'): - parts = headers['cache-control'].split(',') - parts_with_args = [tuple([x.strip().lower() for x in part.split("=", 1)]) for part in parts if -1 != part.find("=")] - parts_wo_args = [(name.strip().lower(), 1) for name in parts if -1 == name.find("=")] - retval = dict(parts_with_args + parts_wo_args) - return retval - -# Whether to use a strict mode to parse WWW-Authenticate headers -# Might lead to bad results in case of ill-formed header value, -# so disabled by default, falling back to relaxed parsing. -# Set to true to turn on, usefull for testing servers. -USE_WWW_AUTH_STRICT_PARSING = 0 - -def _entry_disposition(response_headers, request_headers): - """Determine freshness from the Date, Expires and Cache-Control headers. - - We don't handle the following: - - 1. Cache-Control: max-stale - 2. Age: headers are not used in the calculations. - - Not that this algorithm is simpler than you might think - because we are operating as a private (non-shared) cache. - This lets us ignore 's-maxage'. We can also ignore - 'proxy-invalidate' since we aren't a proxy. - We will never return a stale document as - fresh as a design decision, and thus the non-implementation - of 'max-stale'. This also lets us safely ignore 'must-revalidate' - since we operate as if every server has sent 'must-revalidate'. - Since we are private we get to ignore both 'public' and - 'private' parameters. We also ignore 'no-transform' since - we don't do any transformations. - The 'no-store' parameter is handled at a higher level. - So the only Cache-Control parameters we look at are: - - no-cache - only-if-cached - max-age - min-fresh - """ - - retval = "STALE" - cc = _parse_cache_control(request_headers) - cc_response = _parse_cache_control(response_headers) - - if request_headers.has_key('pragma') and request_headers['pragma'].lower().find('no-cache') != -1: - retval = "TRANSPARENT" - if 'cache-control' not in request_headers: - request_headers['cache-control'] = 'no-cache' - elif cc.has_key('no-cache'): - retval = "TRANSPARENT" - elif cc_response.has_key('no-cache'): - retval = "STALE" - elif cc.has_key('only-if-cached'): - retval = "FRESH" - elif response_headers.has_key('date'): - date = calendar.timegm(email.Utils.parsedate_tz(response_headers['date'])) - now = time.time() - current_age = max(0, now - date) - if cc_response.has_key('max-age'): - try: - freshness_lifetime = int(cc_response['max-age']) - except ValueError: - freshness_lifetime = 0 - elif response_headers.has_key('expires'): - expires = email.Utils.parsedate_tz(response_headers['expires']) - if None == expires: - freshness_lifetime = 0 - else: - freshness_lifetime = max(0, calendar.timegm(expires) - date) - else: - freshness_lifetime = 0 - if cc.has_key('max-age'): - try: - freshness_lifetime = int(cc['max-age']) - except ValueError: - freshness_lifetime = 0 - if cc.has_key('min-fresh'): - try: - min_fresh = int(cc['min-fresh']) - except ValueError: - min_fresh = 0 - current_age += min_fresh - if freshness_lifetime > current_age: - retval = "FRESH" - return retval - -def _decompressContent(response, new_content): - content = new_content - try: - encoding = response.get('content-encoding', None) - if encoding in ['gzip', 'deflate']: - if encoding == 'gzip': - content = gzip.GzipFile(fileobj=StringIO.StringIO(new_content)).read() - if encoding == 'deflate': - content = zlib.decompress(content) - response['content-length'] = str(len(content)) - # Record the historical presence of the encoding in a way the won't interfere. - response['-content-encoding'] = response['content-encoding'] - del response['content-encoding'] - except IOError: - content = "" - raise FailedToDecompressContent(_("Content purported to be compressed with %s but failed to decompress.") % response.get('content-encoding'), response, content) - return content - -def _updateCache(request_headers, response_headers, content, cache, cachekey): - if cachekey: - cc = _parse_cache_control(request_headers) - cc_response = _parse_cache_control(response_headers) - if cc.has_key('no-store') or cc_response.has_key('no-store'): - cache.delete(cachekey) - else: - info = email.Message.Message() - for key, value in response_headers.iteritems(): - if key not in ['status','content-encoding','transfer-encoding']: - info[key] = value - - # Add annotations to the cache to indicate what headers - # are variant for this request. - vary = response_headers.get('vary', None) - if vary: - vary_headers = vary.lower().replace(' ', '').split(',') - for header in vary_headers: - key = '-varied-%s' % header - try: - info[key] = request_headers[header] - except KeyError: - pass - - status = response_headers.status - if status == 304: - status = 200 - - status_header = 'status: %d\r\n' % response_headers.status - - header_str = info.as_string() - - header_str = re.sub("\r(?!\n)|(? 0: - print "connect: (%s, %s)" % (self.host, self.port) - - self.sock.connect(sa) - except socket.error, msg: - if self.debuglevel > 0: - print 'connect fail:', (self.host, self.port) - if self.sock: - self.sock.close() - self.sock = None - continue - break - if not self.sock: - raise socket.error, msg - -class HTTPSConnectionWithTimeout(httplib.HTTPSConnection): - "This class allows communication via SSL." - - def __init__(self, host, port=None, key_file=None, cert_file=None, - strict=None, timeout=None, proxy_info=None): - httplib.HTTPSConnection.__init__(self, host, port=port, key_file=key_file, - cert_file=cert_file, strict=strict) - self.timeout = timeout - self.proxy_info = proxy_info - - def connect(self): - "Connect to a host on a given (SSL) port." - - if self.proxy_info and self.proxy_info.isgood(): - # HACK: amorton sock module is included in this file now - #sock = socks.socksocket(socket.AF_INET, socket.SOCK_STREAM) - sock = socksocket(socket.AF_INET, socket.SOCK_STREAM) - # HACK: amorton enabled TCP_NODLEAY on socket - sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - sock.setproxy(*self.proxy_info.astuple()) - else: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - # HACK: amorton enabled TCP_NODLEAY on socket - sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - - if has_timeout(self.timeout): - sock.settimeout(self.timeout) - sock.connect((self.host, self.port)) - self.sock =_ssl_wrap_socket(sock, self.key_file, self.cert_file) - - - -class Http(object): - """An HTTP client that handles: -- all methods -- caching -- ETags -- compression, -- HTTPS -- Basic -- Digest -- WSSE - -and more. - """ - def __init__(self, cache=None, timeout=None, proxy_info=None): - """The value of proxy_info is a ProxyInfo instance. - -If 'cache' is a string then it is used as a directory name -for a disk cache. Otherwise it must be an object that supports -the same interface as FileCache.""" - self.proxy_info = proxy_info - # Map domain name to an httplib connection - self.connections = {} - # The location of the cache, for now a directory - # where cached responses are held. - if cache and isinstance(cache, str): - self.cache = FileCache(cache) - else: - self.cache = cache - - # Name/password - self.credentials = Credentials() - - # Key/cert - self.certificates = KeyCerts() - - # authorization objects - self.authorizations = [] - - # If set to False then no redirects are followed, even safe ones. - self.follow_redirects = True - - # Which HTTP methods do we apply optimistic concurrency to, i.e. - # which methods get an "if-match:" etag header added to them. - self.optimistic_concurrency_methods = ["PUT"] - - # If 'follow_redirects' is True, and this is set to True then - # all redirecs are followed, including unsafe ones. - self.follow_all_redirects = False - - self.ignore_etag = False - - self.force_exception_to_status_code = False - - self.timeout = timeout - - def _auth_from_challenge(self, host, request_uri, headers, response, content): - """A generator that creates Authorization objects - that can be applied to requests. - """ - challenges = _parse_www_authenticate(response, 'www-authenticate') - for cred in self.credentials.iter(host): - for scheme in AUTH_SCHEME_ORDER: - if challenges.has_key(scheme): - yield AUTH_SCHEME_CLASSES[scheme](cred, host, request_uri, headers, response, content, self) - - def add_credentials(self, name, password, domain=""): - """Add a name and password that will be used - any time a request requires authentication.""" - self.credentials.add(name, password, domain) - - def add_certificate(self, key, cert, domain): - """Add a key and cert that will be used - any time a request requires authentication.""" - self.certificates.add(key, cert, domain) - - def clear_credentials(self): - """Remove all the names and passwords - that are used for authentication""" - self.credentials.clear() - self.authorizations = [] - - def _conn_request(self, conn, request_uri, method, body, headers): - for i in range(2): - try: - conn.request(method, request_uri, body, headers) - except socket.gaierror: - conn.close() - raise ServerNotFoundError("Unable to find the server at %s" % conn.host) - except (socket.error, httplib.HTTPException): - # Just because the server closed the connection doesn't apparently mean - # that the server didn't send a response. - # amorton: patch from http://code.google.com/p/httplib2/source/detail?r=0cff83696d - if conn.sock is None: - if i == 0: - conn.close() - conn.connect() - continue - else: - conn.close() - raise - if i == 0: - conn.close() - conn.connect() - continue - pass - try: - response = conn.getresponse() - except (socket.error, httplib.HTTPException): - if i == 0: - conn.close() - conn.connect() - continue - else: - raise - else: - content = "" - if method == "HEAD": - response.close() - else: - content = response.read() - response = Response(response) - if method != "HEAD": - content = _decompressContent(response, content) - break - return (response, content) - - - def _request(self, conn, host, absolute_uri, request_uri, method, body, headers, redirections, cachekey): - """Do the actual request using the connection object - and also follow one level of redirects if necessary""" - - auths = [(auth.depth(request_uri), auth) for auth in self.authorizations if auth.inscope(host, request_uri)] - auth = auths and sorted(auths)[0][1] or None - if auth: - auth.request(method, request_uri, headers, body) - - (response, content) = self._conn_request(conn, request_uri, method, body, headers) - - if auth: - if auth.response(response, body): - auth.request(method, request_uri, headers, body) - (response, content) = self._conn_request(conn, request_uri, method, body, headers ) - response._stale_digest = 1 - - if response.status == 401: - for authorization in self._auth_from_challenge(host, request_uri, headers, response, content): - authorization.request(method, request_uri, headers, body) - (response, content) = self._conn_request(conn, request_uri, method, body, headers, ) - if response.status != 401: - self.authorizations.append(authorization) - authorization.response(response, body) - break - - if (self.follow_all_redirects or (method in ["GET", "HEAD"]) or response.status == 303): - if self.follow_redirects and response.status in [300, 301, 302, 303, 307]: - # Pick out the location header and basically start from the beginning - # remembering first to strip the ETag header and decrement our 'depth' - if redirections: - if not response.has_key('location') and response.status != 300: - raise RedirectMissingLocation( _("Redirected but the response is missing a Location: header."), response, content) - # Fix-up relative redirects (which violate an RFC 2616 MUST) - if response.has_key('location'): - location = response['location'] - (scheme, authority, path, query, fragment) = parse_uri(location) - if authority == None: - response['location'] = urlparse.urljoin(absolute_uri, location) - if response.status == 301 and method in ["GET", "HEAD"]: - response['-x-permanent-redirect-url'] = response['location'] - if not response.has_key('content-location'): - response['content-location'] = absolute_uri - _updateCache(headers, response, content, self.cache, cachekey) - if headers.has_key('if-none-match'): - del headers['if-none-match'] - if headers.has_key('if-modified-since'): - del headers['if-modified-since'] - if response.has_key('location'): - location = response['location'] - old_response = copy.deepcopy(response) - if not old_response.has_key('content-location'): - old_response['content-location'] = absolute_uri - redirect_method = ((response.status == 303) and (method not in ["GET", "HEAD"])) and "GET" or method - (response, content) = self.request(location, redirect_method, body=body, headers = headers, redirections = redirections - 1) - response.previous = old_response - else: - raise RedirectLimit( _("Redirected more times than rediection_limit allows."), response, content) - elif response.status in [200, 203] and method == "GET": - # Don't cache 206's since we aren't going to handle byte range requests - if not response.has_key('content-location'): - response['content-location'] = absolute_uri - _updateCache(headers, response, content, self.cache, cachekey) - - return (response, content) - - def _normalize_headers(self, headers): - return _normalize_headers(headers) - -# Need to catch and rebrand some exceptions -# Then need to optionally turn all exceptions into status codes -# including all socket.* and httplib.* exceptions. - - - def request(self, uri, method="GET", body=None, headers=None, redirections=DEFAULT_MAX_REDIRECTS, connection_type=None): - """ Performs a single HTTP request. -The 'uri' is the URI of the HTTP resource and can begin -with either 'http' or 'https'. The value of 'uri' must be an absolute URI. - -The 'method' is the HTTP method to perform, such as GET, POST, DELETE, etc. -There is no restriction on the methods allowed. - -The 'body' is the entity body to be sent with the request. It is a string -object. - -Any extra headers that are to be sent with the request should be provided in the -'headers' dictionary. - -The maximum number of redirect to follow before raising an -exception is 'redirections. The default is 5. - -The return value is a tuple of (response, content), the first -being and instance of the 'Response' class, the second being -a string that contains the response entity body. - """ - try: - if headers is None: - headers = {} - else: - headers = self._normalize_headers(headers) - - if not headers.has_key('user-agent'): - headers['user-agent'] = "Python-httplib2/%s" % __version__ - - uri = iri2uri(uri) - - (scheme, authority, request_uri, defrag_uri) = urlnorm(uri) - domain_port = authority.split(":")[0:2] - if len(domain_port) == 2 and domain_port[1] == '443' and scheme == 'http': - scheme = 'https' - authority = domain_port[0] - - conn_key = scheme+":"+authority - if conn_key in self.connections: - conn = self.connections[conn_key] - else: - if not connection_type: - connection_type = (scheme == 'https') and HTTPSConnectionWithTimeout or HTTPConnectionWithTimeout - certs = list(self.certificates.iter(authority)) - if scheme == 'https' and certs: - conn = self.connections[conn_key] = connection_type(authority, key_file=certs[0][0], - cert_file=certs[0][1], timeout=self.timeout, proxy_info=self.proxy_info) - else: - conn = self.connections[conn_key] = connection_type(authority, timeout=self.timeout, proxy_info=self.proxy_info) - conn.set_debuglevel(debuglevel) - - if method in ["GET", "HEAD"] and 'range' not in headers and 'accept-encoding' not in headers: - headers['accept-encoding'] = 'gzip, deflate' - - info = email.Message.Message() - cached_value = None - if self.cache: - cachekey = defrag_uri - cached_value = self.cache.get(cachekey) - if cached_value: - # info = email.message_from_string(cached_value) - # - # Need to replace the line above with the kludge below - # to fix the non-existent bug not fixed in this - # bug report: http://mail.python.org/pipermail/python-bugs-list/2005-September/030289.html - try: - info, content = cached_value.split('\r\n\r\n', 1) - feedparser = email.FeedParser.FeedParser() - feedparser.feed(info) - info = feedparser.close() - feedparser._parse = None - except IndexError: - self.cache.delete(cachekey) - cachekey = None - cached_value = None - else: - cachekey = None - - if method in self.optimistic_concurrency_methods and self.cache and info.has_key('etag') and not self.ignore_etag and 'if-match' not in headers: - # http://www.w3.org/1999/04/Editing/ - headers['if-match'] = info['etag'] - - if method not in ["GET", "HEAD"] and self.cache and cachekey: - # RFC 2616 Section 13.10 - self.cache.delete(cachekey) - - # Check the vary header in the cache to see if this request - # matches what varies in the cache. - if method in ['GET', 'HEAD'] and 'vary' in info: - vary = info['vary'] - vary_headers = vary.lower().replace(' ', '').split(',') - for header in vary_headers: - key = '-varied-%s' % header - value = info[key] - if headers.get(header, '') != value: - cached_value = None - break - - if cached_value and method in ["GET", "HEAD"] and self.cache and 'range' not in headers: - if info.has_key('-x-permanent-redirect-url'): - # Should cached permanent redirects be counted in our redirection count? For now, yes. - (response, new_content) = self.request(info['-x-permanent-redirect-url'], "GET", headers = headers, redirections = redirections - 1) - response.previous = Response(info) - response.previous.fromcache = True - else: - # Determine our course of action: - # Is the cached entry fresh or stale? - # Has the client requested a non-cached response? - # - # There seems to be three possible answers: - # 1. [FRESH] Return the cache entry w/o doing a GET - # 2. [STALE] Do the GET (but add in cache validators if available) - # 3. [TRANSPARENT] Do a GET w/o any cache validators (Cache-Control: no-cache) on the request - entry_disposition = _entry_disposition(info, headers) - - if entry_disposition == "FRESH": - if not cached_value: - info['status'] = '504' - content = "" - response = Response(info) - if cached_value: - response.fromcache = True - return (response, content) - - if entry_disposition == "STALE": - if info.has_key('etag') and not self.ignore_etag and not 'if-none-match' in headers: - headers['if-none-match'] = info['etag'] - if info.has_key('last-modified') and not 'last-modified' in headers: - headers['if-modified-since'] = info['last-modified'] - elif entry_disposition == "TRANSPARENT": - pass - - (response, new_content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey) - - if response.status == 304 and method == "GET": - # Rewrite the cache entry with the new end-to-end headers - # Take all headers that are in response - # and overwrite their values in info. - # unless they are hop-by-hop, or are listed in the connection header. - - for key in _get_end2end_headers(response): - info[key] = response[key] - merged_response = Response(info) - if hasattr(response, "_stale_digest"): - merged_response._stale_digest = response._stale_digest - _updateCache(headers, merged_response, content, self.cache, cachekey) - response = merged_response - response.status = 200 - response.fromcache = True - - elif response.status == 200: - content = new_content - else: - self.cache.delete(cachekey) - content = new_content - else: - cc = _parse_cache_control(headers) - if cc.has_key('only-if-cached'): - info['status'] = '504' - response = Response(info) - content = "" - else: - (response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey) - except Exception, e: - if self.force_exception_to_status_code: - if isinstance(e, HttpLib2ErrorWithResponse): - response = e.response - content = e.content - response.status = 500 - response.reason = str(e) - elif isinstance(e, socket.timeout): - content = "Request Timeout" - response = Response( { - "content-type": "text/plain", - "status": "408", - "content-length": len(content) - }) - response.reason = "Request Timeout" - else: - content = str(e) - response = Response( { - "content-type": "text/plain", - "status": "400", - "content-length": len(content) - }) - response.reason = "Bad Request" - else: - raise - - - return (response, content) - - - -class Response(dict): - """An object more like email.Message than httplib.HTTPResponse.""" - - """Is this response from our local cache""" - fromcache = False - - """HTTP protocol version used by server. 10 for HTTP/1.0, 11 for HTTP/1.1. """ - version = 11 - - "Status code returned by server. " - status = 200 - - """Reason phrase returned by server.""" - reason = "Ok" - - previous = None - - def __init__(self, info): - # info is either an email.Message or - # an httplib.HTTPResponse object. - if isinstance(info, httplib.HTTPResponse): - for key, value in info.getheaders(): - self[key.lower()] = value - self.status = info.status - self['status'] = str(self.status) - self.reason = info.reason - self.version = info.version - elif isinstance(info, email.Message.Message): - for key, value in info.items(): - self[key] = value - self.status = int(self['status']) - else: - for key, value in info.iteritems(): - self[key] = value - self.status = int(self.get('status', self.status)) - - - def __getattr__(self, name): - if name == 'dict': - return self - else: - raise AttributeError, name - - def _create_summary_request(entity_type, filters, summary_fields, filter_operator, grouping): '''_create_summary_request assembles a request based on input''' #TODO make this part of summary method? diff --git a/tests/httplib2test.py b/tests/httplib2test.py new file mode 100755 index 000000000..394aa876e --- /dev/null +++ b/tests/httplib2test.py @@ -0,0 +1,1398 @@ +#!/usr/bin/env python2.4 +""" +httplib2test + +A set of unit tests for httplib2.py. + +Requires Python 2.4 or later +""" + +__author__ = "Joe Gregorio (joe@bitworking.org)" +__copyright__ = "Copyright 2006, Joe Gregorio" +__contributors__ = [] +__license__ = "MIT" +__history__ = """ """ +__version__ = "0.1 ($Rev: 118 $)" + + +import sys +import unittest +import httplib +import httplib2 +import os +import urlparse +import time +import base64 +import StringIO + +# Python 2.3 support +if not hasattr(unittest.TestCase, 'assertTrue'): + unittest.TestCase.assertTrue = unittest.TestCase.failUnless + unittest.TestCase.assertFalse = unittest.TestCase.failIf + +# The test resources base uri +base = 'http://bitworking.org/projects/httplib2/test/' +#base = 'http://localhost/projects/httplib2/test/' +cacheDirName = ".cache" + + +class CredentialsTest(unittest.TestCase): + def test(self): + c = httplib2.Credentials() + c.add("joe", "password") + self.assertEqual(("joe", "password"), list(c.iter("bitworking.org"))[0]) + self.assertEqual(("joe", "password"), list(c.iter(""))[0]) + c.add("fred", "password2", "wellformedweb.org") + self.assertEqual(("joe", "password"), list(c.iter("bitworking.org"))[0]) + self.assertEqual(1, len(list(c.iter("bitworking.org")))) + self.assertEqual(2, len(list(c.iter("wellformedweb.org")))) + self.assertTrue(("fred", "password2") in list(c.iter("wellformedweb.org"))) + c.clear() + self.assertEqual(0, len(list(c.iter("bitworking.org")))) + c.add("fred", "password2", "wellformedweb.org") + self.assertTrue(("fred", "password2") in list(c.iter("wellformedweb.org"))) + self.assertEqual(0, len(list(c.iter("bitworking.org")))) + self.assertEqual(0, len(list(c.iter("")))) + + +class ParserTest(unittest.TestCase): + def testFromStd66(self): + self.assertEqual( ('http', 'example.com', '', None, None ), httplib2.parse_uri("http://example.com")) + self.assertEqual( ('https', 'example.com', '', None, None ), httplib2.parse_uri("https://example.com")) + self.assertEqual( ('https', 'example.com:8080', '', None, None ), httplib2.parse_uri("https://example.com:8080")) + self.assertEqual( ('http', 'example.com', '/', None, None ), httplib2.parse_uri("http://example.com/")) + self.assertEqual( ('http', 'example.com', '/path', None, None ), httplib2.parse_uri("http://example.com/path")) + self.assertEqual( ('http', 'example.com', '/path', 'a=1&b=2', None ), httplib2.parse_uri("http://example.com/path?a=1&b=2")) + self.assertEqual( ('http', 'example.com', '/path', 'a=1&b=2', 'fred' ), httplib2.parse_uri("http://example.com/path?a=1&b=2#fred")) + self.assertEqual( ('http', 'example.com', '/path', 'a=1&b=2', 'fred' ), httplib2.parse_uri("http://example.com/path?a=1&b=2#fred")) + + +class UrlNormTest(unittest.TestCase): + def test(self): + self.assertEqual( "http://example.org/", httplib2.urlnorm("http://example.org")[-1]) + self.assertEqual( "http://example.org/", httplib2.urlnorm("http://EXAMple.org")[-1]) + self.assertEqual( "http://example.org/?=b", httplib2.urlnorm("http://EXAMple.org?=b")[-1]) + self.assertEqual( "http://example.org/mypath?a=b", httplib2.urlnorm("http://EXAMple.org/mypath?a=b")[-1]) + self.assertEqual( "http://localhost:80/", httplib2.urlnorm("http://localhost:80")[-1]) + self.assertEqual( httplib2.urlnorm("http://localhost:80/"), httplib2.urlnorm("HTTP://LOCALHOST:80")) + try: + httplib2.urlnorm("/") + self.fail("Non-absolute URIs should raise an exception") + except httplib2.RelativeURIError: + pass + +class UrlSafenameTest(unittest.TestCase): + def test(self): + # Test that different URIs end up generating different safe names + self.assertEqual( "example.org,fred,a=b,58489f63a7a83c3b7794a6a398ee8b1f", httplib2.safename("http://example.org/fred/?a=b")) + self.assertEqual( "example.org,fred,a=b,8c5946d56fec453071f43329ff0be46b", httplib2.safename("http://example.org/fred?/a=b")) + self.assertEqual( "www.example.org,fred,a=b,499c44b8d844a011b67ea2c015116968", httplib2.safename("http://www.example.org/fred?/a=b")) + self.assertEqual( httplib2.safename(httplib2.urlnorm("http://www")[-1]), httplib2.safename(httplib2.urlnorm("http://WWW")[-1])) + self.assertEqual( "www.example.org,fred,a=b,692e843a333484ce0095b070497ab45d", httplib2.safename("https://www.example.org/fred?/a=b")) + self.assertNotEqual( httplib2.safename("http://www"), httplib2.safename("https://www")) + # Test the max length limits + uri = "http://" + ("w" * 200) + ".org" + uri2 = "http://" + ("w" * 201) + ".org" + self.assertNotEqual( httplib2.safename(uri2), httplib2.safename(uri)) + # Max length should be 200 + 1 (",") + 32 + self.assertEqual(233, len(httplib2.safename(uri2))) + self.assertEqual(233, len(httplib2.safename(uri))) + # Unicode + if sys.version_info >= (2,3): + self.assertEqual( "xn--http,-4y1d.org,fred,a=b,579924c35db315e5a32e3d9963388193", httplib2.safename(u"http://\u2304.org/fred/?a=b")) + +class _MyResponse(StringIO.StringIO): + def __init__(self, body, **kwargs): + StringIO.StringIO.__init__(self, body) + self.headers = kwargs + + def iteritems(self): + return self.headers.iteritems() + + +class _MyHTTPConnection(object): + "This class is just a mock of httplib.HTTPConnection used for testing" + + def __init__(self, host, port=None, key_file=None, cert_file=None, + strict=None, timeout=None, proxy_info=None): + self.host = host + self.port = port + self.timeout = timeout + self.log = "" + + def set_debuglevel(self, level): + pass + + def connect(self): + "Connect to a host on a given port." + pass + + def close(self): + pass + + def request(self, method, request_uri, body, headers): + pass + + def getresponse(self): + return _MyResponse("the body", status="200") + + +class HttpTest(unittest.TestCase): + def setUp(self): + if os.path.exists(cacheDirName): + [os.remove(os.path.join(cacheDirName, file)) for file in os.listdir(cacheDirName)] + self.http = httplib2.Http(cacheDirName) + self.http.clear_credentials() + + def testConnectionType(self): + self.http.force_exception_to_status_code = False + response, content = self.http.request("http://bitworking.org", connection_type=_MyHTTPConnection) + self.assertEqual(response['content-location'], "http://bitworking.org") + self.assertEqual(content, "the body") + + def testGetUnknownServer(self): + self.http.force_exception_to_status_code = False + try: + self.http.request("http://fred.bitworking.org/") + self.fail("An httplib2.ServerNotFoundError Exception must be thrown on an unresolvable server.") + except httplib2.ServerNotFoundError: + pass + + # Now test with exceptions turned off + self.http.force_exception_to_status_code = True + + (response, content) = self.http.request("http://fred.bitworking.org/") + self.assertEqual(response['content-type'], 'text/plain') + self.assertTrue(content.startswith("Unable to find")) + self.assertEqual(response.status, 400) + + def testGetIRI(self): + if sys.version_info >= (2,3): + uri = urlparse.urljoin(base, u"reflector/reflector.cgi?d=\N{CYRILLIC CAPITAL LETTER DJE}") + (response, content) = self.http.request(uri, "GET") + d = self.reflector(content) + self.assertTrue(d.has_key('QUERY_STRING')) + self.assertTrue(d['QUERY_STRING'].find('%D0%82') > 0) + + def testGetIsDefaultMethod(self): + # Test that GET is the default method + uri = urlparse.urljoin(base, "methods/method_reflector.cgi") + (response, content) = self.http.request(uri) + self.assertEqual(response['x-method'], "GET") + + def testDifferentMethods(self): + # Test that all methods can be used + uri = urlparse.urljoin(base, "methods/method_reflector.cgi") + for method in ["GET", "PUT", "DELETE", "POST"]: + (response, content) = self.http.request(uri, method, body=" ") + self.assertEqual(response['x-method'], method) + + def testHeadRead(self): + # Test that we don't try to read the response of a HEAD request + # since httplib blocks response.read() for HEAD requests. + # Oddly enough this doesn't appear as a problem when doing HEAD requests + # against Apache servers. + uri = "http://www.google.com/" + (response, content) = self.http.request(uri, "HEAD") + self.assertEqual(response.status, 200) + self.assertEqual(content, "") + + def testGetNoCache(self): + # Test that can do a GET w/o the cache turned on. + http = httplib2.Http() + uri = urlparse.urljoin(base, "304/test_etag.txt") + (response, content) = http.request(uri, "GET") + self.assertEqual(response.status, 200) + self.assertEqual(response.previous, None) + + def testGetOnlyIfCachedCacheHit(self): + # Test that can do a GET with cache and 'only-if-cached' + uri = urlparse.urljoin(base, "304/test_etag.txt") + (response, content) = self.http.request(uri, "GET") + (response, content) = self.http.request(uri, "GET", headers={'cache-control': 'only-if-cached'}) + self.assertEqual(response.fromcache, True) + self.assertEqual(response.status, 200) + + def testGetOnlyIfCachedCacheMiss(self): + # Test that can do a GET with no cache with 'only-if-cached' + uri = urlparse.urljoin(base, "304/test_etag.txt") + (response, content) = self.http.request(uri, "GET", headers={'cache-control': 'only-if-cached'}) + self.assertEqual(response.fromcache, False) + self.assertEqual(response.status, 504) + + def testGetOnlyIfCachedNoCacheAtAll(self): + # Test that can do a GET with no cache with 'only-if-cached' + # Of course, there might be an intermediary beyond us + # that responds to the 'only-if-cached', so this + # test can't really be guaranteed to pass. + http = httplib2.Http() + uri = urlparse.urljoin(base, "304/test_etag.txt") + (response, content) = http.request(uri, "GET", headers={'cache-control': 'only-if-cached'}) + self.assertEqual(response.fromcache, False) + self.assertEqual(response.status, 504) + + def testUserAgent(self): + # Test that we provide a default user-agent + uri = urlparse.urljoin(base, "user-agent/test.cgi") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 200) + self.assertTrue(content.startswith("Python-httplib2/")) + + def testUserAgentNonDefault(self): + # Test that the default user-agent can be over-ridden + + uri = urlparse.urljoin(base, "user-agent/test.cgi") + (response, content) = self.http.request(uri, "GET", headers={'User-Agent': 'fred/1.0'}) + self.assertEqual(response.status, 200) + self.assertTrue(content.startswith("fred/1.0")) + + def testGet300WithLocation(self): + # Test the we automatically follow 300 redirects if a Location: header is provided + uri = urlparse.urljoin(base, "300/with-location-header.asis") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 200) + self.assertEqual(content, "This is the final destination.\n") + self.assertEqual(response.previous.status, 300) + self.assertEqual(response.previous.fromcache, False) + + # Confirm that the intermediate 300 is not cached + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 200) + self.assertEqual(content, "This is the final destination.\n") + self.assertEqual(response.previous.status, 300) + self.assertEqual(response.previous.fromcache, False) + + def testGet300WithLocationNoRedirect(self): + # Test the we automatically follow 300 redirects if a Location: header is provided + self.http.follow_redirects = False + uri = urlparse.urljoin(base, "300/with-location-header.asis") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 300) + + def testGet300WithoutLocation(self): + # Not giving a Location: header in a 300 response is acceptable + # In which case we just return the 300 response + uri = urlparse.urljoin(base, "300/without-location-header.asis") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 300) + self.assertTrue(response['content-type'].startswith("text/html")) + self.assertEqual(response.previous, None) + + def testGet301(self): + # Test that we automatically follow 301 redirects + # and that we cache the 301 response + uri = urlparse.urljoin(base, "301/onestep.asis") + destination = urlparse.urljoin(base, "302/final-destination.txt") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 200) + self.assertTrue(response.has_key('content-location')) + self.assertEqual(response['content-location'], destination) + self.assertEqual(content, "This is the final destination.\n") + self.assertEqual(response.previous.status, 301) + self.assertEqual(response.previous.fromcache, False) + + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 200) + self.assertEqual(response['content-location'], destination) + self.assertEqual(content, "This is the final destination.\n") + self.assertEqual(response.previous.status, 301) + self.assertEqual(response.previous.fromcache, True) + + + def testGet301NoRedirect(self): + # Test that we automatically follow 301 redirects + # and that we cache the 301 response + self.http.follow_redirects = False + uri = urlparse.urljoin(base, "301/onestep.asis") + destination = urlparse.urljoin(base, "302/final-destination.txt") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 301) + + + def testGet302(self): + # Test that we automatically follow 302 redirects + # and that we DO NOT cache the 302 response + uri = urlparse.urljoin(base, "302/onestep.asis") + destination = urlparse.urljoin(base, "302/final-destination.txt") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 200) + self.assertEqual(response['content-location'], destination) + self.assertEqual(content, "This is the final destination.\n") + self.assertEqual(response.previous.status, 302) + self.assertEqual(response.previous.fromcache, False) + + uri = urlparse.urljoin(base, "302/onestep.asis") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 200) + self.assertEqual(response.fromcache, True) + self.assertEqual(response['content-location'], destination) + self.assertEqual(content, "This is the final destination.\n") + self.assertEqual(response.previous.status, 302) + self.assertEqual(response.previous.fromcache, False) + self.assertEqual(response.previous['content-location'], uri) + + uri = urlparse.urljoin(base, "302/twostep.asis") + + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 200) + self.assertEqual(response.fromcache, True) + self.assertEqual(content, "This is the final destination.\n") + self.assertEqual(response.previous.status, 302) + self.assertEqual(response.previous.fromcache, False) + + def testGet302RedirectionLimit(self): + # Test that we can set a lower redirection limit + # and that we raise an exception when we exceed + # that limit. + self.http.force_exception_to_status_code = False + + uri = urlparse.urljoin(base, "302/twostep.asis") + try: + (response, content) = self.http.request(uri, "GET", redirections = 1) + self.fail("This should not happen") + except httplib2.RedirectLimit: + pass + except Exception, e: + self.fail("Threw wrong kind of exception ") + + # Re-run the test with out the exceptions + self.http.force_exception_to_status_code = True + + (response, content) = self.http.request(uri, "GET", redirections = 1) + self.assertEqual(response.status, 500) + self.assertTrue(response.reason.startswith("Redirected more")) + self.assertEqual("302", response['status']) + self.assertTrue(content.startswith("")) + self.assertTrue(response.previous != None) + + def testGet302NoLocation(self): + # Test that we throw an exception when we get + # a 302 with no Location: header. + self.http.force_exception_to_status_code = False + uri = urlparse.urljoin(base, "302/no-location.asis") + try: + (response, content) = self.http.request(uri, "GET") + self.fail("Should never reach here") + except httplib2.RedirectMissingLocation: + pass + except Exception, e: + self.fail("Threw wrong kind of exception ") + + # Re-run the test with out the exceptions + self.http.force_exception_to_status_code = True + + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 500) + self.assertTrue(response.reason.startswith("Redirected but")) + self.assertEqual("302", response['status']) + self.assertTrue(content.startswith("This is content")) + +# KB Test failing with freshly checkout code +# def testGet302ViaHttps(self): +# # Google always redirects to http://google.com +# (response, content) = self.http.request("https://google.com", "GET") +# self.assertEqual(200, response.status) +# self.assertEqual(302, response.previous.status) + + def testGetViaHttps(self): + # Test that we can handle HTTPS + (response, content) = self.http.request("https://google.com/adsense/", "GET") + self.assertEqual(200, response.status) + + def testGetViaHttpsSpecViolationOnLocation(self): + # Test that we follow redirects through HTTPS + # even if they violate the spec by including + # a relative Location: header instead of an + # absolute one. + (response, content) = self.http.request("https://google.com/adsense", "GET") + self.assertEqual(200, response.status) + self.assertNotEqual(None, response.previous) + + + def testGetViaHttpsKeyCert(self): + # At this point I can only test + # that the key and cert files are passed in + # correctly to httplib. It would be nice to have + # a real https endpoint to test against. + http = httplib2.Http(timeout=2) + + http.add_certificate("akeyfile", "acertfile", "bitworking.org") + try: + (response, content) = http.request("https://bitworking.org", "GET") + except: + pass + self.assertEqual(http.connections["https:bitworking.org"].key_file, "akeyfile") + self.assertEqual(http.connections["https:bitworking.org"].cert_file, "acertfile") + + try: + (response, content) = http.request("https://notthere.bitworking.org", "GET") + except: + pass + self.assertEqual(http.connections["https:notthere.bitworking.org"].key_file, None) + self.assertEqual(http.connections["https:notthere.bitworking.org"].cert_file, None) + + + + + def testGet303(self): + # Do a follow-up GET on a Location: header + # returned from a POST that gave a 303. + uri = urlparse.urljoin(base, "303/303.cgi") + (response, content) = self.http.request(uri, "POST", " ") + self.assertEqual(response.status, 200) + self.assertEqual(content, "This is the final destination.\n") + self.assertEqual(response.previous.status, 303) + + def testGet303NoRedirect(self): + # Do a follow-up GET on a Location: header + # returned from a POST that gave a 303. + self.http.follow_redirects = False + uri = urlparse.urljoin(base, "303/303.cgi") + (response, content) = self.http.request(uri, "POST", " ") + self.assertEqual(response.status, 303) + + def test303ForDifferentMethods(self): + # Test that all methods can be used + uri = urlparse.urljoin(base, "303/redirect-to-reflector.cgi") + for (method, method_on_303) in [("PUT", "GET"), ("DELETE", "GET"), ("POST", "GET"), ("GET", "GET"), ("HEAD", "GET")]: + (response, content) = self.http.request(uri, method, body=" ") + self.assertEqual(response['x-method'], method_on_303) + + def testGet304(self): + # Test that we use ETags properly to validate our cache + uri = urlparse.urljoin(base, "304/test_etag.txt") + (response, content) = self.http.request(uri, "GET") + self.assertNotEqual(response['etag'], "") + + (response, content) = self.http.request(uri, "GET") + (response, content) = self.http.request(uri, "GET", headers = {'cache-control': 'must-revalidate'}) + self.assertEqual(response.status, 200) + self.assertEqual(response.fromcache, True) + + cache_file_name = os.path.join(cacheDirName, httplib2.safename(httplib2.urlnorm(uri)[-1])) + f = open(cache_file_name, "r") + status_line = f.readline() + f.close() + + self.assertTrue(status_line.startswith("status:")) + + (response, content) = self.http.request(uri, "HEAD") + self.assertEqual(response.status, 200) + self.assertEqual(response.fromcache, True) + + (response, content) = self.http.request(uri, "GET", headers = {'range': 'bytes=0-0'}) + self.assertEqual(response.status, 206) + self.assertEqual(response.fromcache, False) + + def testGetIgnoreEtag(self): + # Test that we can forcibly ignore ETags + uri = urlparse.urljoin(base, "reflector/reflector.cgi") + (response, content) = self.http.request(uri, "GET") + self.assertNotEqual(response['etag'], "") + + (response, content) = self.http.request(uri, "GET", headers = {'cache-control': 'max-age=0'}) + d = self.reflector(content) + self.assertTrue(d.has_key('HTTP_IF_NONE_MATCH')) + + self.http.ignore_etag = True + (response, content) = self.http.request(uri, "GET", headers = {'cache-control': 'max-age=0'}) + d = self.reflector(content) + self.assertEqual(response.fromcache, False) + self.assertFalse(d.has_key('HTTP_IF_NONE_MATCH')) + + def testOverrideEtag(self): + # Test that we can forcibly ignore ETags + uri = urlparse.urljoin(base, "reflector/reflector.cgi") + (response, content) = self.http.request(uri, "GET") + self.assertNotEqual(response['etag'], "") + + (response, content) = self.http.request(uri, "GET", headers = {'cache-control': 'max-age=0'}) + d = self.reflector(content) + self.assertTrue(d.has_key('HTTP_IF_NONE_MATCH')) + self.assertNotEqual(d['HTTP_IF_NONE_MATCH'], "fred") + + (response, content) = self.http.request(uri, "GET", headers = {'cache-control': 'max-age=0', 'if-none-match': 'fred'}) + d = self.reflector(content) + self.assertTrue(d.has_key('HTTP_IF_NONE_MATCH')) + self.assertEqual(d['HTTP_IF_NONE_MATCH'], "fred") + +#MAP-commented this out because it consistently fails +# def testGet304EndToEnd(self): +# # Test that end to end headers get overwritten in the cache +# uri = urlparse.urljoin(base, "304/end2end.cgi") +# (response, content) = self.http.request(uri, "GET") +# self.assertNotEqual(response['etag'], "") +# old_date = response['date'] +# time.sleep(2) +# +# (response, content) = self.http.request(uri, "GET", headers = {'Cache-Control': 'max-age=0'}) +# # The response should be from the cache, but the Date: header should be updated. +# new_date = response['date'] +# self.assertNotEqual(new_date, old_date) +# self.assertEqual(response.status, 200) +# self.assertEqual(response.fromcache, True) + + def testGet304LastModified(self): + # Test that we can still handle a 304 + # by only using the last-modified cache validator. + uri = urlparse.urljoin(base, "304/last-modified-only/last-modified-only.txt") + (response, content) = self.http.request(uri, "GET") + + self.assertNotEqual(response['last-modified'], "") + (response, content) = self.http.request(uri, "GET") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 200) + self.assertEqual(response.fromcache, True) + + def testGet307(self): + # Test that we do follow 307 redirects but + # do not cache the 307 + uri = urlparse.urljoin(base, "307/onestep.asis") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 200) + self.assertEqual(content, "This is the final destination.\n") + self.assertEqual(response.previous.status, 307) + self.assertEqual(response.previous.fromcache, False) + + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 200) + self.assertEqual(response.fromcache, True) + self.assertEqual(content, "This is the final destination.\n") + self.assertEqual(response.previous.status, 307) + self.assertEqual(response.previous.fromcache, False) + + def testGet410(self): + # Test that we pass 410's through + uri = urlparse.urljoin(base, "410/410.asis") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 410) + + def testVaryHeaderSimple(self): + """ + RFC 2616 13.6 + When the cache receives a subsequent request whose Request-URI + specifies one or more cache entries including a Vary header field, + the cache MUST NOT use such a cache entry to construct a response + to the new request unless all of the selecting request-headers + present in the new request match the corresponding stored + request-headers in the original request. + """ + # test that the vary header is sent + uri = urlparse.urljoin(base, "vary/accept.asis") + (response, content) = self.http.request(uri, "GET", headers={'Accept': 'text/plain'}) + self.assertEqual(response.status, 200) + self.assertTrue(response.has_key('vary')) + + # get the resource again, from the cache since accept header in this + # request is the same as the request + (response, content) = self.http.request(uri, "GET", headers={'Accept': 'text/plain'}) + self.assertEqual(response.status, 200) + self.assertEqual(response.fromcache, True, msg="Should be from cache") + + # get the resource again, not from cache since Accept headers does not match + (response, content) = self.http.request(uri, "GET", headers={'Accept': 'text/html'}) + self.assertEqual(response.status, 200) + self.assertEqual(response.fromcache, False, msg="Should not be from cache") + + # get the resource again, without any Accept header, so again no match + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 200) + self.assertEqual(response.fromcache, False, msg="Should not be from cache") + + def testNoVary(self): + # when there is no vary, a different Accept header (e.g.) should not + # impact if the cache is used + # test that the vary header is not sent + uri = urlparse.urljoin(base, "vary/no-vary.asis") + (response, content) = self.http.request(uri, "GET", headers={'Accept': 'text/plain'}) + self.assertEqual(response.status, 200) + self.assertFalse(response.has_key('vary')) + + (response, content) = self.http.request(uri, "GET", headers={'Accept': 'text/plain'}) + self.assertEqual(response.status, 200) + self.assertEqual(response.fromcache, True, msg="Should be from cache") + + (response, content) = self.http.request(uri, "GET", headers={'Accept': 'text/html'}) + self.assertEqual(response.status, 200) + self.assertEqual(response.fromcache, True, msg="Should be from cache") + + def testVaryHeaderDouble(self): + uri = urlparse.urljoin(base, "vary/accept-double.asis") + (response, content) = self.http.request(uri, "GET", headers={ + 'Accept': 'text/plain', 'Accept-Language': 'da, en-gb;q=0.8, en;q=0.7'}) + self.assertEqual(response.status, 200) + self.assertTrue(response.has_key('vary')) + + # we are from cache + (response, content) = self.http.request(uri, "GET", headers={ + 'Accept': 'text/plain', 'Accept-Language': 'da, en-gb;q=0.8, en;q=0.7'}) + self.assertEqual(response.fromcache, True, msg="Should be from cache") + + (response, content) = self.http.request(uri, "GET", headers={'Accept': 'text/plain'}) + self.assertEqual(response.status, 200) + self.assertEqual(response.fromcache, False) + + # get the resource again, not from cache, varied headers don't match exact + (response, content) = self.http.request(uri, "GET", headers={'Accept-Language': 'da'}) + self.assertEqual(response.status, 200) + self.assertEqual(response.fromcache, False, msg="Should not be from cache") + + + def testHeadGZip(self): + # Test that we don't try to decompress a HEAD response + uri = urlparse.urljoin(base, "gzip/final-destination.txt") + (response, content) = self.http.request(uri, "HEAD") + self.assertEqual(response.status, 200) + self.assertNotEqual(int(response['content-length']), 0) + self.assertEqual(content, "") + + def testGetGZip(self): + # Test that we support gzip compression + uri = urlparse.urljoin(base, "gzip/final-destination.txt") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 200) + self.assertFalse(response.has_key('content-encoding')) + self.assertTrue(response.has_key('-content-encoding')) + self.assertEqual(int(response['content-length']), len("This is the final destination.\n")) + self.assertEqual(content, "This is the final destination.\n") + + def testGetGZipFailure(self): + # Test that we raise a good exception when the gzip fails + self.http.force_exception_to_status_code = False + uri = urlparse.urljoin(base, "gzip/failed-compression.asis") + try: + (response, content) = self.http.request(uri, "GET") + self.fail("Should never reach here") + except httplib2.FailedToDecompressContent: + pass + except Exception: + self.fail("Threw wrong kind of exception") + + # Re-run the test with out the exceptions + self.http.force_exception_to_status_code = True + + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 500) + self.assertTrue(response.reason.startswith("Content purported")) + + def testTimeout(self): + self.http.force_exception_to_status_code = True + uri = urlparse.urljoin(base, "timeout/timeout.cgi") + try: + import socket + socket.setdefaulttimeout(1) + except: + # Don't run the test if we can't set the timeout + return + (response, content) = self.http.request(uri) + self.assertEqual(response.status, 408) + self.assertTrue(response.reason.startswith("Request Timeout")) + self.assertTrue(content.startswith("Request Timeout")) + + def testIndividualTimeout(self): + uri = urlparse.urljoin(base, "timeout/timeout.cgi") + http = httplib2.Http(timeout=1) + http.force_exception_to_status_code = True + + (response, content) = http.request(uri) + self.assertEqual(response.status, 408) + self.assertTrue(response.reason.startswith("Request Timeout")) + self.assertTrue(content.startswith("Request Timeout")) + + + def testHTTPSInitTimeout(self): + c = httplib2.HTTPSConnectionWithTimeout('localhost', 80, timeout=47) + self.assertEqual(47, c.timeout) + + def testGetDeflate(self): + # Test that we support deflate compression + uri = urlparse.urljoin(base, "deflate/deflated.asis") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 200) + self.assertFalse(response.has_key('content-encoding')) + self.assertEqual(int(response['content-length']), len("This is the final destination.")) + self.assertEqual(content, "This is the final destination.") + + def testGetDeflateFailure(self): + # Test that we raise a good exception when the deflate fails + self.http.force_exception_to_status_code = False + + uri = urlparse.urljoin(base, "deflate/failed-compression.asis") + try: + (response, content) = self.http.request(uri, "GET") + self.fail("Should never reach here") + except httplib2.FailedToDecompressContent: + pass + except Exception: + self.fail("Threw wrong kind of exception") + + # Re-run the test with out the exceptions + self.http.force_exception_to_status_code = True + + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 500) + self.assertTrue(response.reason.startswith("Content purported")) + + def testGetDuplicateHeaders(self): + # Test that duplicate headers get concatenated via ',' + uri = urlparse.urljoin(base, "duplicate-headers/multilink.asis") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 200) + self.assertEqual(content, "This is content\n") + self.assertEqual(response['link'].split(",")[0], '; rel="home"; title="BitWorking"') + + def testGetCacheControlNoCache(self): + # Test Cache-Control: no-cache on requests + uri = urlparse.urljoin(base, "304/test_etag.txt") + (response, content) = self.http.request(uri, "GET") + self.assertNotEqual(response['etag'], "") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 200) + self.assertEqual(response.fromcache, True) + + (response, content) = self.http.request(uri, "GET", headers={'Cache-Control': 'no-cache'}) + self.assertEqual(response.status, 200) + self.assertEqual(response.fromcache, False) + + def testGetCacheControlPragmaNoCache(self): + # Test Pragma: no-cache on requests + uri = urlparse.urljoin(base, "304/test_etag.txt") + (response, content) = self.http.request(uri, "GET") + self.assertNotEqual(response['etag'], "") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 200) + self.assertEqual(response.fromcache, True) + + (response, content) = self.http.request(uri, "GET", headers={'Pragma': 'no-cache'}) + self.assertEqual(response.status, 200) + self.assertEqual(response.fromcache, False) + + def testGetCacheControlNoStoreRequest(self): + # A no-store request means that the response should not be stored. + uri = urlparse.urljoin(base, "304/test_etag.txt") + + (response, content) = self.http.request(uri, "GET", headers={'Cache-Control': 'no-store'}) + self.assertEqual(response.status, 200) + self.assertEqual(response.fromcache, False) + + (response, content) = self.http.request(uri, "GET", headers={'Cache-Control': 'no-store'}) + self.assertEqual(response.status, 200) + self.assertEqual(response.fromcache, False) + + def testGetCacheControlNoStoreResponse(self): + # A no-store response means that the response should not be stored. + uri = urlparse.urljoin(base, "no-store/no-store.asis") + + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 200) + self.assertEqual(response.fromcache, False) + + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 200) + self.assertEqual(response.fromcache, False) + + def testGetCacheControlNoCacheNoStoreRequest(self): + # Test that a no-store, no-cache clears the entry from the cache + # even if it was cached previously. + uri = urlparse.urljoin(base, "304/test_etag.txt") + + (response, content) = self.http.request(uri, "GET") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.fromcache, True) + (response, content) = self.http.request(uri, "GET", headers={'Cache-Control': 'no-store, no-cache'}) + (response, content) = self.http.request(uri, "GET", headers={'Cache-Control': 'no-store, no-cache'}) + self.assertEqual(response.status, 200) + self.assertEqual(response.fromcache, False) + + def testUpdateInvalidatesCache(self): + # Test that calling PUT or DELETE on a + # URI that is cache invalidates that cache. + uri = urlparse.urljoin(base, "304/test_etag.txt") + + (response, content) = self.http.request(uri, "GET") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.fromcache, True) + (response, content) = self.http.request(uri, "DELETE") + self.assertEqual(response.status, 405) + + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.fromcache, False) + + def testUpdateUsesCachedETag(self): + # Test that we natively support http://www.w3.org/1999/04/Editing/ + uri = urlparse.urljoin(base, "conditional-updates/test.cgi") + + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 200) + self.assertEqual(response.fromcache, False) + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 200) + self.assertEqual(response.fromcache, True) + (response, content) = self.http.request(uri, "PUT", body="foo") + self.assertEqual(response.status, 200) + (response, content) = self.http.request(uri, "PUT", body="foo") + self.assertEqual(response.status, 412) + + def testUpdateUsesCachedETagAndOCMethod(self): + # Test that we natively support http://www.w3.org/1999/04/Editing/ + uri = urlparse.urljoin(base, "conditional-updates/test.cgi") + + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 200) + self.assertEqual(response.fromcache, False) + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 200) + self.assertEqual(response.fromcache, True) + self.http.optimistic_concurrency_methods.append("DELETE") + (response, content) = self.http.request(uri, "DELETE") + self.assertEqual(response.status, 200) + + + def testUpdateUsesCachedETagOverridden(self): + # Test that we natively support http://www.w3.org/1999/04/Editing/ + uri = urlparse.urljoin(base, "conditional-updates/test.cgi") + + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 200) + self.assertEqual(response.fromcache, False) + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 200) + self.assertEqual(response.fromcache, True) + (response, content) = self.http.request(uri, "PUT", body="foo", headers={'if-match': 'fred'}) + self.assertEqual(response.status, 412) + + def testBasicAuth(self): + # Test Basic Authentication + uri = urlparse.urljoin(base, "basic/file.txt") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 401) + + uri = urlparse.urljoin(base, "basic/") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 401) + + self.http.add_credentials('joe', 'password') + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 200) + + uri = urlparse.urljoin(base, "basic/file.txt") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 200) + + def testBasicAuthWithDomain(self): + # Test Basic Authentication + uri = urlparse.urljoin(base, "basic/file.txt") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 401) + + uri = urlparse.urljoin(base, "basic/") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 401) + + self.http.add_credentials('joe', 'password', "example.org") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 401) + + uri = urlparse.urljoin(base, "basic/file.txt") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 401) + + domain = urlparse.urlparse(base)[1] + self.http.add_credentials('joe', 'password', domain) + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 200) + + uri = urlparse.urljoin(base, "basic/file.txt") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 200) + + + + + + + def testBasicAuthTwoDifferentCredentials(self): + # Test Basic Authentication with multiple sets of credentials + uri = urlparse.urljoin(base, "basic2/file.txt") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 401) + + uri = urlparse.urljoin(base, "basic2/") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 401) + + self.http.add_credentials('fred', 'barney') + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 200) + + uri = urlparse.urljoin(base, "basic2/file.txt") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 200) + + def testBasicAuthNested(self): + # Test Basic Authentication with resources + # that are nested + uri = urlparse.urljoin(base, "basic-nested/") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 401) + + uri = urlparse.urljoin(base, "basic-nested/subdir") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 401) + + # Now add in credentials one at a time and test. + self.http.add_credentials('joe', 'password') + + uri = urlparse.urljoin(base, "basic-nested/") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 200) + + uri = urlparse.urljoin(base, "basic-nested/subdir") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 401) + + self.http.add_credentials('fred', 'barney') + + uri = urlparse.urljoin(base, "basic-nested/") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 200) + + uri = urlparse.urljoin(base, "basic-nested/subdir") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 200) + + def testDigestAuth(self): + # Test that we support Digest Authentication + uri = urlparse.urljoin(base, "digest/") + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 401) + + self.http.add_credentials('joe', 'password') + (response, content) = self.http.request(uri, "GET") + self.assertEqual(response.status, 200) + + uri = urlparse.urljoin(base, "digest/file.txt") + (response, content) = self.http.request(uri, "GET") + + def testDigestAuthNextNonceAndNC(self): + # Test that if the server sets nextnonce that we reset + # the nonce count back to 1 + uri = urlparse.urljoin(base, "digest/file.txt") + self.http.add_credentials('joe', 'password') + (response, content) = self.http.request(uri, "GET", headers = {"cache-control":"no-cache"}) + info = httplib2._parse_www_authenticate(response, 'authentication-info') + self.assertEqual(response.status, 200) + (response, content) = self.http.request(uri, "GET", headers = {"cache-control":"no-cache"}) + info2 = httplib2._parse_www_authenticate(response, 'authentication-info') + self.assertEqual(response.status, 200) + + if info.has_key('nextnonce'): + self.assertEqual(info2['nc'], 1) + + def testDigestAuthStale(self): + # Test that we can handle a nonce becoming stale + uri = urlparse.urljoin(base, "digest-expire/file.txt") + self.http.add_credentials('joe', 'password') + (response, content) = self.http.request(uri, "GET", headers = {"cache-control":"no-cache"}) + info = httplib2._parse_www_authenticate(response, 'authentication-info') + self.assertEqual(response.status, 200) + + time.sleep(3) + # Sleep long enough that the nonce becomes stale + + (response, content) = self.http.request(uri, "GET", headers = {"cache-control":"no-cache"}) + self.assertFalse(response.fromcache) + self.assertTrue(response._stale_digest) + info3 = httplib2._parse_www_authenticate(response, 'authentication-info') + self.assertEqual(response.status, 200) + + def reflector(self, content): + return dict( [tuple(x.split("=", 1)) for x in content.strip().split("\n")] ) + + def testReflector(self): + uri = urlparse.urljoin(base, "reflector/reflector.cgi") + (response, content) = self.http.request(uri, "GET") + d = self.reflector(content) + self.assertTrue(d.has_key('HTTP_USER_AGENT')) + + def testConnectionClose(self): + uri = "http://www.google.com/" + (response, content) = self.http.request(uri, "GET") + for c in self.http.connections.values(): + self.assertNotEqual(None, c.sock) + (response, content) = self.http.request(uri, "GET", headers={"connection": "close"}) + for c in self.http.connections.values(): + self.assertEqual(None, c.sock) + + +try: + import memcache + class HttpTestMemCached(HttpTest): + def setUp(self): + self.cache = memcache.Client(['127.0.0.1:11211'], debug=0) + #self.cache = memcache.Client(['10.0.0.4:11211'], debug=1) + self.http = httplib2.Http(self.cache) + self.cache.flush_all() + # Not exactly sure why the sleep is needed here, but + # if not present then some unit tests that rely on caching + # fail. Memcached seems to lose some sets immediately + # after a flush_all if the set is to a value that + # was previously cached. (Maybe the flush is handled async?) + time.sleep(1) + self.http.clear_credentials() +except: + pass + + + + +# ------------------------------------------------------------------------ + +class HttpPrivateTest(unittest.TestCase): + + def testParseCacheControl(self): + # Test that we can parse the Cache-Control header + self.assertEqual({}, httplib2._parse_cache_control({})) + self.assertEqual({'no-cache': 1}, httplib2._parse_cache_control({'cache-control': ' no-cache'})) + cc = httplib2._parse_cache_control({'cache-control': ' no-cache, max-age = 7200'}) + self.assertEqual(cc['no-cache'], 1) + self.assertEqual(cc['max-age'], '7200') + cc = httplib2._parse_cache_control({'cache-control': ' , '}) + self.assertEqual(cc[''], 1) + + try: + cc = httplib2._parse_cache_control({'cache-control': 'Max-age=3600;post-check=1800,pre-check=3600'}) + self.assertTrue("max-age" in cc) + except: + self.fail("Should not throw exception") + + def testNormalizeHeaders(self): + # Test that we normalize headers to lowercase + h = httplib2._normalize_headers({'Cache-Control': 'no-cache', 'Other': 'Stuff'}) + self.assertTrue(h.has_key('cache-control')) + self.assertTrue(h.has_key('other')) + self.assertEqual('Stuff', h['other']) + + def testExpirationModelTransparent(self): + # Test that no-cache makes our request TRANSPARENT + response_headers = { + 'cache-control': 'max-age=7200' + } + request_headers = { + 'cache-control': 'no-cache' + } + self.assertEqual("TRANSPARENT", httplib2._entry_disposition(response_headers, request_headers)) + + def testMaxAgeNonNumeric(self): + # Test that no-cache makes our request TRANSPARENT + response_headers = { + 'cache-control': 'max-age=fred, min-fresh=barney' + } + request_headers = { + } + self.assertEqual("STALE", httplib2._entry_disposition(response_headers, request_headers)) + + + def testExpirationModelNoCacheResponse(self): + # The date and expires point to an entry that should be + # FRESH, but the no-cache over-rides that. + now = time.time() + response_headers = { + 'date': time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(now)), + 'expires': time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(now+4)), + 'cache-control': 'no-cache' + } + request_headers = { + } + self.assertEqual("STALE", httplib2._entry_disposition(response_headers, request_headers)) + + def testExpirationModelStaleRequestMustReval(self): + # must-revalidate forces STALE + self.assertEqual("STALE", httplib2._entry_disposition({}, {'cache-control': 'must-revalidate'})) + + def testExpirationModelStaleResponseMustReval(self): + # must-revalidate forces STALE + self.assertEqual("STALE", httplib2._entry_disposition({'cache-control': 'must-revalidate'}, {})) + + def testExpirationModelFresh(self): + response_headers = { + 'date': time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime()), + 'cache-control': 'max-age=2' + } + request_headers = { + } + self.assertEqual("FRESH", httplib2._entry_disposition(response_headers, request_headers)) + time.sleep(3) + self.assertEqual("STALE", httplib2._entry_disposition(response_headers, request_headers)) + + def testExpirationMaxAge0(self): + response_headers = { + 'date': time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime()), + 'cache-control': 'max-age=0' + } + request_headers = { + } + self.assertEqual("STALE", httplib2._entry_disposition(response_headers, request_headers)) + + def testExpirationModelDateAndExpires(self): + now = time.time() + response_headers = { + 'date': time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(now)), + 'expires': time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(now+2)), + } + request_headers = { + } + self.assertEqual("FRESH", httplib2._entry_disposition(response_headers, request_headers)) + time.sleep(3) + self.assertEqual("STALE", httplib2._entry_disposition(response_headers, request_headers)) + + def testExpiresZero(self): + now = time.time() + response_headers = { + 'date': time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(now)), + 'expires': "0", + } + request_headers = { + } + self.assertEqual("STALE", httplib2._entry_disposition(response_headers, request_headers)) + + def testExpirationModelDateOnly(self): + now = time.time() + response_headers = { + 'date': time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(now+3)), + } + request_headers = { + } + self.assertEqual("STALE", httplib2._entry_disposition(response_headers, request_headers)) + + def testExpirationModelOnlyIfCached(self): + response_headers = { + } + request_headers = { + 'cache-control': 'only-if-cached', + } + self.assertEqual("FRESH", httplib2._entry_disposition(response_headers, request_headers)) + + def testExpirationModelMaxAgeBoth(self): + now = time.time() + response_headers = { + 'date': time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(now)), + 'cache-control': 'max-age=2' + } + request_headers = { + 'cache-control': 'max-age=0' + } + self.assertEqual("STALE", httplib2._entry_disposition(response_headers, request_headers)) + + def testExpirationModelDateAndExpiresMinFresh1(self): + now = time.time() + response_headers = { + 'date': time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(now)), + 'expires': time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(now+2)), + } + request_headers = { + 'cache-control': 'min-fresh=2' + } + self.assertEqual("STALE", httplib2._entry_disposition(response_headers, request_headers)) + + def testExpirationModelDateAndExpiresMinFresh2(self): + now = time.time() + response_headers = { + 'date': time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(now)), + 'expires': time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(now+4)), + } + request_headers = { + 'cache-control': 'min-fresh=2' + } + self.assertEqual("FRESH", httplib2._entry_disposition(response_headers, request_headers)) + + def testParseWWWAuthenticateEmpty(self): + res = httplib2._parse_www_authenticate({}) + self.assertEqual(len(res.keys()), 0) + + def testParseWWWAuthenticate(self): + # different uses of spaces around commas + res = httplib2._parse_www_authenticate({ 'www-authenticate': 'Test realm="test realm" , foo=foo ,bar="bar", baz=baz,qux=qux'}) + self.assertEqual(len(res.keys()), 1) + self.assertEqual(len(res['test'].keys()), 5) + + # tokens with non-alphanum + res = httplib2._parse_www_authenticate({ 'www-authenticate': 'T*!%#st realm=to*!%#en, to*!%#en="quoted string"'}) + self.assertEqual(len(res.keys()), 1) + self.assertEqual(len(res['t*!%#st'].keys()), 2) + + # quoted string with quoted pairs + res = httplib2._parse_www_authenticate({ 'www-authenticate': 'Test realm="a \\"test\\" realm"'}) + self.assertEqual(len(res.keys()), 1) + self.assertEqual(res['test']['realm'], 'a "test" realm') + + def testParseWWWAuthenticateStrict(self): + httplib2.USE_WWW_AUTH_STRICT_PARSING = 1; + self.testParseWWWAuthenticate(); + httplib2.USE_WWW_AUTH_STRICT_PARSING = 0; + + def testParseWWWAuthenticateBasic(self): + res = httplib2._parse_www_authenticate({ 'www-authenticate': 'Basic realm="me"'}) + basic = res['basic'] + self.assertEqual('me', basic['realm']) + + res = httplib2._parse_www_authenticate({ 'www-authenticate': 'Basic realm="me", algorithm="MD5"'}) + basic = res['basic'] + self.assertEqual('me', basic['realm']) + self.assertEqual('MD5', basic['algorithm']) + + res = httplib2._parse_www_authenticate({ 'www-authenticate': 'Basic realm="me", algorithm=MD5'}) + basic = res['basic'] + self.assertEqual('me', basic['realm']) + self.assertEqual('MD5', basic['algorithm']) + + def testParseWWWAuthenticateBasic2(self): + res = httplib2._parse_www_authenticate({ 'www-authenticate': 'Basic realm="me",other="fred" '}) + basic = res['basic'] + self.assertEqual('me', basic['realm']) + self.assertEqual('fred', basic['other']) + + def testParseWWWAuthenticateBasic3(self): + res = httplib2._parse_www_authenticate({ 'www-authenticate': 'Basic REAlm="me" '}) + basic = res['basic'] + self.assertEqual('me', basic['realm']) + + + def testParseWWWAuthenticateDigest(self): + res = httplib2._parse_www_authenticate({ 'www-authenticate': + 'Digest realm="testrealm@host.com", qop="auth,auth-int", nonce="dcd98b7102dd2f0e8b11d0f600bfb0c093", opaque="5ccc069c403ebaf9f0171e9517f40e41"'}) + digest = res['digest'] + self.assertEqual('testrealm@host.com', digest['realm']) + self.assertEqual('auth,auth-int', digest['qop']) + + + def testParseWWWAuthenticateMultiple(self): + res = httplib2._parse_www_authenticate({ 'www-authenticate': + 'Digest realm="testrealm@host.com", qop="auth,auth-int", nonce="dcd98b7102dd2f0e8b11d0f600bfb0c093", opaque="5ccc069c403ebaf9f0171e9517f40e41" Basic REAlm="me" '}) + digest = res['digest'] + self.assertEqual('testrealm@host.com', digest['realm']) + self.assertEqual('auth,auth-int', digest['qop']) + self.assertEqual('dcd98b7102dd2f0e8b11d0f600bfb0c093', digest['nonce']) + self.assertEqual('5ccc069c403ebaf9f0171e9517f40e41', digest['opaque']) + basic = res['basic'] + self.assertEqual('me', basic['realm']) + + def testParseWWWAuthenticateMultiple2(self): + # Handle an added comma between challenges, which might get thrown in if the challenges were + # originally sent in separate www-authenticate headers. + res = httplib2._parse_www_authenticate({ 'www-authenticate': + 'Digest realm="testrealm@host.com", qop="auth,auth-int", nonce="dcd98b7102dd2f0e8b11d0f600bfb0c093", opaque="5ccc069c403ebaf9f0171e9517f40e41", Basic REAlm="me" '}) + digest = res['digest'] + self.assertEqual('testrealm@host.com', digest['realm']) + self.assertEqual('auth,auth-int', digest['qop']) + self.assertEqual('dcd98b7102dd2f0e8b11d0f600bfb0c093', digest['nonce']) + self.assertEqual('5ccc069c403ebaf9f0171e9517f40e41', digest['opaque']) + basic = res['basic'] + self.assertEqual('me', basic['realm']) + + def testParseWWWAuthenticateMultiple3(self): + # Handle an added comma between challenges, which might get thrown in if the challenges were + # originally sent in separate www-authenticate headers. + res = httplib2._parse_www_authenticate({ 'www-authenticate': + 'Digest realm="testrealm@host.com", qop="auth,auth-int", nonce="dcd98b7102dd2f0e8b11d0f600bfb0c093", opaque="5ccc069c403ebaf9f0171e9517f40e41", Basic REAlm="me", WSSE realm="foo", profile="UsernameToken"'}) + digest = res['digest'] + self.assertEqual('testrealm@host.com', digest['realm']) + self.assertEqual('auth,auth-int', digest['qop']) + self.assertEqual('dcd98b7102dd2f0e8b11d0f600bfb0c093', digest['nonce']) + self.assertEqual('5ccc069c403ebaf9f0171e9517f40e41', digest['opaque']) + basic = res['basic'] + self.assertEqual('me', basic['realm']) + wsse = res['wsse'] + self.assertEqual('foo', wsse['realm']) + self.assertEqual('UsernameToken', wsse['profile']) + + def testParseWWWAuthenticateMultiple4(self): + res = httplib2._parse_www_authenticate({ 'www-authenticate': + 'Digest realm="test-real.m@host.com", qop \t=\t"\tauth,auth-int", nonce="(*)&^&$%#",opaque="5ccc069c403ebaf9f0171e9517f40e41", Basic REAlm="me", WSSE realm="foo", profile="UsernameToken"'}) + digest = res['digest'] + self.assertEqual('test-real.m@host.com', digest['realm']) + self.assertEqual('\tauth,auth-int', digest['qop']) + self.assertEqual('(*)&^&$%#', digest['nonce']) + + def testParseWWWAuthenticateMoreQuoteCombos(self): + res = httplib2._parse_www_authenticate({'www-authenticate':'Digest realm="myrealm", nonce="Ygk86AsKBAA=3516200d37f9a3230352fde99977bd6d472d4306", algorithm=MD5, qop="auth", stale=true'}) + digest = res['digest'] + self.assertEqual('myrealm', digest['realm']) + + def testDigestObject(self): + credentials = ('joe', 'password') + host = None + request_uri = '/projects/httplib2/test/digest/' + headers = {} + response = { + 'www-authenticate': 'Digest realm="myrealm", nonce="Ygk86AsKBAA=3516200d37f9a3230352fde99977bd6d472d4306", algorithm=MD5, qop="auth"' + } + content = "" + + d = httplib2.DigestAuthentication(credentials, host, request_uri, headers, response, content, None) + d.request("GET", request_uri, headers, content, cnonce="33033375ec278a46") + our_request = "Authorization: %s" % headers['Authorization'] + working_request = 'Authorization: Digest username="joe", realm="myrealm", nonce="Ygk86AsKBAA=3516200d37f9a3230352fde99977bd6d472d4306", uri="/projects/httplib2/test/digest/", algorithm=MD5, response="97ed129401f7cdc60e5db58a80f3ea8b", qop=auth, nc=00000001, cnonce="33033375ec278a46"' + self.assertEqual(our_request, working_request) + + + def testDigestObjectStale(self): + credentials = ('joe', 'password') + host = None + request_uri = '/projects/httplib2/test/digest/' + headers = {} + response = httplib2.Response({ }) + response['www-authenticate'] = 'Digest realm="myrealm", nonce="Ygk86AsKBAA=3516200d37f9a3230352fde99977bd6d472d4306", algorithm=MD5, qop="auth", stale=true' + response.status = 401 + content = "" + d = httplib2.DigestAuthentication(credentials, host, request_uri, headers, response, content, None) + # Returns true to force a retry + self.assertTrue( d.response(response, content) ) + + def testDigestObjectAuthInfo(self): + credentials = ('joe', 'password') + host = None + request_uri = '/projects/httplib2/test/digest/' + headers = {} + response = httplib2.Response({ }) + response['www-authenticate'] = 'Digest realm="myrealm", nonce="Ygk86AsKBAA=3516200d37f9a3230352fde99977bd6d472d4306", algorithm=MD5, qop="auth", stale=true' + response['authentication-info'] = 'nextnonce="fred"' + content = "" + d = httplib2.DigestAuthentication(credentials, host, request_uri, headers, response, content, None) + # Returns true to force a retry + self.assertFalse( d.response(response, content) ) + self.assertEqual('fred', d.challenge['nonce']) + self.assertEqual(1, d.challenge['nc']) + + def testWsseAlgorithm(self): + digest = httplib2._wsse_username_token("d36e316282959a9ed4c89851497a717f", "2003-12-15T14:43:07Z", "taadtaadpstcsm") + expected = "quR/EWLAV4xLf9Zqyw4pDmfV9OY=" + self.assertEqual(expected, digest) + + def testEnd2End(self): + # one end to end header + response = {'content-type': 'application/atom+xml', 'te': 'deflate'} + end2end = httplib2._get_end2end_headers(response) + self.assertTrue('content-type' in end2end) + self.assertTrue('te' not in end2end) + self.assertTrue('connection' not in end2end) + + # one end to end header that gets eliminated + response = {'connection': 'content-type', 'content-type': 'application/atom+xml', 'te': 'deflate'} + end2end = httplib2._get_end2end_headers(response) + self.assertTrue('content-type' not in end2end) + self.assertTrue('te' not in end2end) + self.assertTrue('connection' not in end2end) + + # Degenerate case of no headers + response = {} + end2end = httplib2._get_end2end_headers(response) + self.assertEquals(0, len(end2end)) + + # Degenerate case of connection referrring to a header not passed in + response = {'connection': 'content-type'} + end2end = httplib2._get_end2end_headers(response) + self.assertEquals(0, len(end2end)) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_client.py b/tests/test_client.py index a1fe1ceed..56f1deb53 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -13,6 +13,7 @@ import sys import time import unittest +import httplib2 import mock import shotgun_api3 as api @@ -161,9 +162,9 @@ def test_connect_close(self): def test_network_retry(self): """Network failure is retried""" - self.sg._http_request.side_effect = api.HttpLib2Error + self.sg._http_request.side_effect = httplib2.HttpLib2Error - self.assertRaises(api.HttpLib2Error, self.sg.info) + self.assertRaises(httplib2.HttpLib2Error, self.sg.info) self.assertTrue( self.sg.config.max_rpc_attempts ==self.sg._http_request.call_count, "Call is repeated") From abc20fea5902443d2f78cd9f1a03d0de5501f37a Mon Sep 17 00:00:00 2001 From: kennedy behrman Date: Fri, 22 Jul 2011 11:51:40 -0700 Subject: [PATCH 018/570] refactored module into package allowing seperation of third party modules --- shotgun_api3/__init__.py | 2 + shotgun_api3/lib/__init__.py | 0 .../lib/httplib2}/__init__.py | 0 .../lib/httplib2}/iri2uri.py | 0 shotgun_api3/lib/sgtimezone.py | 58 ++++++++++++ socks.py => shotgun_api3/lib/socks.py | 0 shotgun_api3.py => shotgun_api3/shotgun.py | 60 +----------- tests/base.py | 10 +- tests/test_client.py | 12 +-- tests/tests_unit.py | 92 ++++--------------- 10 files changed, 90 insertions(+), 144 deletions(-) create mode 100644 shotgun_api3/__init__.py create mode 100644 shotgun_api3/lib/__init__.py rename {httplib2 => shotgun_api3/lib/httplib2}/__init__.py (100%) rename {httplib2 => shotgun_api3/lib/httplib2}/iri2uri.py (100%) create mode 100644 shotgun_api3/lib/sgtimezone.py rename socks.py => shotgun_api3/lib/socks.py (100%) rename shotgun_api3.py => shotgun_api3/shotgun.py (96%) diff --git a/shotgun_api3/__init__.py b/shotgun_api3/__init__.py new file mode 100644 index 000000000..e1dbf1128 --- /dev/null +++ b/shotgun_api3/__init__.py @@ -0,0 +1,2 @@ +from shotgun import Shotgun, ShotgunError, Fault + diff --git a/shotgun_api3/lib/__init__.py b/shotgun_api3/lib/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/httplib2/__init__.py b/shotgun_api3/lib/httplib2/__init__.py similarity index 100% rename from httplib2/__init__.py rename to shotgun_api3/lib/httplib2/__init__.py diff --git a/httplib2/iri2uri.py b/shotgun_api3/lib/httplib2/iri2uri.py similarity index 100% rename from httplib2/iri2uri.py rename to shotgun_api3/lib/httplib2/iri2uri.py diff --git a/shotgun_api3/lib/sgtimezone.py b/shotgun_api3/lib/sgtimezone.py new file mode 100644 index 000000000..ced06365e --- /dev/null +++ b/shotgun_api3/lib/sgtimezone.py @@ -0,0 +1,58 @@ +#! /opt/local/bin/python +# ---------------------------------------------------------------------------- +# SG_TIMEZONE module +# this is rolled into the this shotgun api file to avoid having to require +# current users of api2 to install new modules and modify PYTHONPATH info. +# ---------------------------------------------------------------------------- + +class SgTimezone(object): + from datetime import tzinfo, timedelta, datetime + import time as _time + + ZERO = timedelta(0) + STDOFFSET = timedelta(seconds = -_time.timezone) + if _time.daylight: + DSTOFFSET = timedelta(seconds = -_time.altzone) + else: + DSTOFFSET = STDOFFSET + DSTDIFF = DSTOFFSET - STDOFFSET + + def __init__(self): + self.utc = self.UTC() + self.local = self.LocalTimezone() + + class UTC(tzinfo): + + def utcoffset(self, dt): + return SgTimezone.ZERO + + def tzname(self, dt): + return "UTC" + + def dst(self, dt): + return SgTimezone.ZERO + + class LocalTimezone(tzinfo): + + def utcoffset(self, dt): + if self._isdst(dt): + return SgTimezone.DSTOFFSET + else: + return SgTimezone.STDOFFSET + + def dst(self, dt): + if self._isdst(dt): + return SgTimezone.DSTDIFF + else: + return SgTimezone.ZERO + + def tzname(self, dt): + return _time.tzname[self._isdst(dt)] + + def _isdst(self, dt): + tt = (dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second, dt.weekday(), 0, -1) + import time as _time + stamp = _time.mktime(tt) + tt = _time.localtime(stamp) + return tt.tm_isdst > 0 + diff --git a/socks.py b/shotgun_api3/lib/socks.py similarity index 100% rename from socks.py rename to shotgun_api3/lib/socks.py diff --git a/shotgun_api3.py b/shotgun_api3/shotgun.py similarity index 96% rename from shotgun_api3.py rename to shotgun_api3/shotgun.py index 1078d6b18..ebb001664 100755 --- a/shotgun_api3.py +++ b/shotgun_api3/shotgun.py @@ -44,8 +44,8 @@ import urllib import urllib2 # used for image upload import urlparse -from socks import * -from httplib2 import Http +from lib.httplib2 import Http +from lib.sgtimezone import SgTimezone log = logging.getLogger("shotgun_api3") @@ -1311,62 +1311,6 @@ def https_request(self, request): -# ---------------------------------------------------------------------------- -# SG_TIMEZONE module -# this is rolled into the this shotgun api file to avoid having to require -# current users of api2 to install new modules and modify PYTHONPATH info. -# ---------------------------------------------------------------------------- - -class SgTimezone(object): - from datetime import tzinfo, timedelta, datetime - import time as _time - - ZERO = timedelta(0) - STDOFFSET = timedelta(seconds = -_time.timezone) - if _time.daylight: - DSTOFFSET = timedelta(seconds = -_time.altzone) - else: - DSTOFFSET = STDOFFSET - DSTDIFF = DSTOFFSET - STDOFFSET - - def __init__(self): - self.utc = self.UTC() - self.local = self.LocalTimezone() - - class UTC(tzinfo): - - def utcoffset(self, dt): - return SgTimezone.ZERO - - def tzname(self, dt): - return "UTC" - - def dst(self, dt): - return SgTimezone.ZERO - - class LocalTimezone(tzinfo): - - def utcoffset(self, dt): - if self._isdst(dt): - return SgTimezone.DSTOFFSET - else: - return SgTimezone.STDOFFSET - - def dst(self, dt): - if self._isdst(dt): - return SgTimezone.DSTDIFF - else: - return SgTimezone.ZERO - - def tzname(self, dt): - return _time.tzname[self._isdst(dt)] - - def _isdst(self, dt): - tt = (dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second, dt.weekday(), 0, -1) - import time as _time - stamp = _time.mktime(tt) - tt = _time.localtime(stamp) - return tt.tm_isdst > 0 sg_timezone = SgTimezone() diff --git a/tests/base.py b/tests/base.py index 2c6ce6955..7d0f38e3c 100644 --- a/tests/base.py +++ b/tests/base.py @@ -5,10 +5,12 @@ try: import simplejson as json except ImportError: - import json as json + import json import mock + import shotgun_api3 as api +from shotgun_api3.shotgun import ServerCapabilities CONFIG_PATH = 'tests/config' @@ -73,7 +75,7 @@ def _setup_mock(self): #also replace the function that is called to get the http connection #to avoid calling the server. OK to return a mock as we will not use #it - self.mock_conn = mock.Mock(spec=api.Http) + self.mock_conn = mock.Mock(spec=api.lib.httplib2.Http) #The Http objects connection property is a dict of connections #it is holding self.mock_conn.connections = dict() @@ -81,8 +83,8 @@ def _setup_mock(self): self.sg._get_connection = mock.Mock(return_value=self.mock_conn) #create the server caps directly to say we have the correct version - self.sg._server_caps = api.ServerCapabilities(self.sg.config.server, - {"version" : [2,4,0]}) + self.sg._server_caps = ServerCapabilities(self.sg.config.server, + {"version" : [2,4,0]}) def _mock_http(self, data, headers=None, status=None): diff --git a/tests/test_client.py b/tests/test_client.py index 56f1deb53..5fe79e7df 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -13,11 +13,11 @@ import sys import time import unittest -import httplib2 - import mock -import shotgun_api3 as api +import shotgun_api3.lib.httplib2 as httplib2 +import shotgun_api3 as api +from shotgun_api3.shotgun import ServerCapabilities import base class TestShotgunClient(base.MockTestBase): @@ -69,11 +69,11 @@ def test_detect_server_caps(self): def test_server_version_json(self): '''test_server_version_json tests expected versions for json support.''' - sc = api.ServerCapabilities("foo", {"version" : (2,4,0)}) + sc = ServerCapabilities("foo", {"version" : (2,4,0)}) sc.version = (2,3,99) self.assertRaises(api.ShotgunError, sc._ensure_json_supported) - self.assertRaises(api.ShotgunError, api.ServerCapabilities, "foo", + self.assertRaises(api.ShotgunError, ServerCapabilities, "foo", {"version" : (2,2,0)}) sc.version = (0,0,0) @@ -110,7 +110,7 @@ def auth_args(): def test_config(self): """Client config can be created""" - x = api._Config() + x = api.shotgun._Config() self.assertTrue(x is not None) def test_url(self): diff --git a/tests/tests_unit.py b/tests/tests_unit.py index 3461f5a68..d35764bd1 100644 --- a/tests/tests_unit.py +++ b/tests/tests_unit.py @@ -3,7 +3,6 @@ from tests import base from mock import patch, Mock import shotgun_api3 as api -import socks class TestShotgunInit(unittest.TestCase): '''Test case for Shotgun.__init__''' @@ -46,21 +45,21 @@ def setUp(self): def test_filter_operator_none(self): expected_logical_operator = 'and' filter_operator = None - result = api._create_summary_request('',[],None,filter_operator, None) + result = api.shotgun._create_summary_request('',[],None,filter_operator, None) actual_logical_operator = result['filters']['logical_operator'] self.assertEqual(expected_logical_operator, actual_logical_operator) def test_filter_operator_all(self): expected_logical_operator = 'and' filter_operator = 'all' - result = api._create_summary_request('',[],None,filter_operator, None) + result = api.shotgun._create_summary_request('',[],None,filter_operator, None) actual_logical_operator = result['filters']['logical_operator'] self.assertEqual(expected_logical_operator, actual_logical_operator) def test_filter_operator_none(self): expected_logical_operator = 'or' filter_operator = 'or' - result = api._create_summary_request('',[],None,filter_operator, None) + result = api.shotgun._create_summary_request('',[],None,filter_operator, None) actual_logical_operator = result['filters']['logical_operator'] self.assertEqual(expected_logical_operator, actual_logical_operator) @@ -69,14 +68,14 @@ def test_filters(self): relation = 'relation' value = 'value' expected_condition = {'path':path, 'relation':relation, 'value':value} - result = api._create_summary_request('', [[path, relation, value]], None, None, None) + result = api.shotgun._create_summary_request('', [[path, relation, value]], None, None, None) actual_condition = result['filters']['conditions'][0] def test_grouping(self): - result = api._create_summary_request('', [], None, None, None) + result = api.shotgun._create_summary_request('', [], None, None, None) self.assertFalse(result.has_key('grouping')) grouping = ['something'] - result = api._create_summary_request('', [], None, None, grouping) + result = api.shotgun._create_summary_request('', [], None, None, grouping) self.assertEqual(grouping, result['grouping']) def test_filters_type(self): @@ -90,19 +89,19 @@ def test_grouping_type(self): class TestServerCapabilities(unittest.TestCase): def test_no_server_version(self): - self.assertRaises(api.ShotgunError, api.ServerCapabilities, 'host', {}) + self.assertRaises(api.ShotgunError, api.shotgun.ServerCapabilities, 'host', {}) def test_bad_version(self): '''test_bad_meta tests passing bad meta data type''' - self.assertRaises(api.ShotgunError, api.ServerCapabilities, 'host', {'version':(0,0,0)}) + self.assertRaises(api.ShotgunError, api.shotgun.ServerCapabilities, 'host', {'version':(0,0,0)}) def test_dev_version(self): - serverCapabilities = api.ServerCapabilities('host', {'version':(3,4,0,'Dev')}) + serverCapabilities = api.shotgun.ServerCapabilities('host', {'version':(3,4,0,'Dev')}) self.assertEqual(serverCapabilities.version, (3,4,0)) self.assertTrue(serverCapabilities.is_dev) - serverCapabilities = api.ServerCapabilities('host', {'version':(2,4,0)}) + serverCapabilities = api.shotgun.ServerCapabilities('host', {'version':(2,4,0)}) self.assertEqual(serverCapabilities.version, (2,4,0)) self.assertFalse(serverCapabilities.is_dev) @@ -117,92 +116,33 @@ def test_windows(self): def test_linux(self): self.assert_platform('Linux', 'linux') - @patch('shotgun_api3.platform') + @patch('shotgun_api3.shotgun.platform') def assert_platform(self, sys_ret_val, expected, mock_platform): mock_platform.system.return_value = sys_ret_val expected_local_path_field = "local_path_%s" % expected - client_caps = api.ClientCapabilities() + client_caps = api.shotgun.ClientCapabilities() self.assertEquals(client_caps.platform, expected) self.assertEquals(client_caps.local_path_field, expected_local_path_field) - @patch('shotgun_api3.platform') + @patch('shotgun_api3.shotgun.platform') def test_no_platform(self, mock_platform): mock_platform.system.return_value = "unsupported" - client_caps = api.ClientCapabilities() + client_caps = api.shotgun.ClientCapabilities() self.assertIsNone(client_caps.platform) self.assertIsNone(client_caps.local_path_field) - @patch('shotgun_api3.sys') + @patch('shotgun_api3.shotgun.sys') def test_py_version(self, mock_sys): major = 2 minor = 7 micro = 3 mock_sys.version_info = (major, minor, micro, 'final', 0) expected_py_version = "%s.%s" % (major, minor) - client_caps = api.ClientCapabilities() + client_caps = api.shotgun.ClientCapabilities() self.assertEquals(client_caps.py_version, expected_py_version) -class TestSockSocket(unittest.TestCase): - def test_defaultproxy(self): - socks._defaultproxy = object() - ssocket = api.socksocket() - self.assertEquals(ssocket._socksocket__proxy, socks._defaultproxy) - - def test_connect_bad_destpair(self): - '''test_connect_bad_destpair tests various bad destpair parameters.''' - ssocket = api.socksocket() - #not list or tuple - self.assertRaises(api.GeneralProxyError, ssocket.connect, 'not a list or tuple') - #wrong length - self.assertRaises(api.GeneralProxyError, ssocket.connect, [1]) - #first item str - self.assertRaises(api.GeneralProxyError, ssocket.connect, [1, 2]) - #second item int - self.assertRaises(api.GeneralProxyError, ssocket.connect, ['1', '2']) - - def test_connect_sock5(self): - self._assert_connect_proxy_type(api.PROXY_TYPE_SOCKS5, sock5_calls=1) - - def test_connect_sock4(self): - self._assert_connect_proxy_type(api.PROXY_TYPE_SOCKS4, sock4_calls=1) - - def test_connect_http(self): - self._assert_connect_proxy_type(api.PROXY_TYPE_HTTP, http_calls=1) - - def test_connect_proxy_type_none(self): - self._assert_connect_proxy_type(None) - - def test_unkown_proxy_type(self): - ssocket = api.socksocket() - ssocket._socksocket__proxy = (99, 'address', 1080) - self.assertRaises(api.GeneralProxyError, ssocket.connect, ('dns',9999)) - - - @patch('socks._orgsocket.connect') - def _assert_connect_proxy_type(self, - proxy_type, - _orgsocket_connect, - sock5_calls=0, - sock4_calls=0, - http_calls=0): - ssocket = api.socksocket() - ssocket._socksocket__proxy = (proxy_type, 'address', 1080) - ssocket._socksocket__negotiatesocks5 = Mock(name='__negotiatesocks5') - ssocket._socksocket__negotiatesocks4 = Mock(name='__negotiatesocks4') - ssocket._socksocket__negotiatehttp = Mock(name='__negotiatehttp') - - self.assertEquals(ssocket._socksocket__negotiatesocks5.call_count, 0) - self.assertEquals(ssocket._socksocket__negotiatesocks4.call_count, 0) - self.assertEquals(ssocket._socksocket__negotiatehttp.call_count, 0) - self.assertEquals(_orgsocket_connect.call_count, 0) - - ssocket.connect(('dns', 8080)) - self.assertEquals(ssocket._socksocket__negotiatesocks5.call_count, sock5_calls) - self.assertEquals(ssocket._socksocket__negotiatesocks4.call_count, sock4_calls) - self.assertEquals(ssocket._socksocket__negotiatehttp.call_count, http_calls) - self.assertEquals(_orgsocket_connect.call_count, 1) From a8d33ec6491623eae4155444698de1835744d8e1 Mon Sep 17 00:00:00 2001 From: kennedy behrman Date: Mon, 25 Jul 2011 08:28:51 -0700 Subject: [PATCH 019/570] shared functionality extracted to _translate_filters; bugs: undeclared varialbe 'missing'; use of string.partition (not available in py 24; some cleanup; more tests; ticket#14276 --- run-tests-2.4 | 2 +- run-tests-2.5 | 2 +- shotgun_api3/shotgun.py | 347 +++++++++++++++++++++------------------- tests/tests_unit.py | 82 +++++++--- 4 files changed, 243 insertions(+), 190 deletions(-) diff --git a/run-tests-2.4 b/run-tests-2.4 index b670d87ab..f24bbc302 100755 --- a/run-tests-2.4 +++ b/run-tests-2.4 @@ -1 +1 @@ -clear && find ./ -name ".coverage" -delete && find ./ -name "*.pyc" -delete && nosetests-2.4 -vd --stop --with-cover --cover-package=shotgun_json \ No newline at end of file +clear && find ./ -name ".coverage" -delete && find ./ -name "*.pyc" -delete && nosetests-2.4 -vd --stop --with-cover --cover-package=shotgun_api3 diff --git a/run-tests-2.5 b/run-tests-2.5 index 5ed4104e9..20fdfe95e 100755 --- a/run-tests-2.5 +++ b/run-tests-2.5 @@ -1 +1 @@ -clear && find ./ -name ".coverage" -delete && find ./ -name "*.pyc" -delete && nosetests-2.5 -vd --stop --with-cover --cover-package=shotgun_json \ No newline at end of file +clear && find ./ -name ".coverage" -delete && find ./ -name "*.pyc" -delete && nosetests-2.5 -vd --stop --with-cover --cover-package=shotgun_api3 diff --git a/shotgun_api3/shotgun.py b/shotgun_api3/shotgun.py index ebb001664..60c42560b 100755 --- a/shotgun_api3/shotgun.py +++ b/shotgun_api3/shotgun.py @@ -1,31 +1,33 @@ #!/usr/bin/env python -# --------------------------------------------------------------------------------------------- -# Copyright (c) 2009-2011, Shotgun Software Inc -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# - Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# - Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# - Neither the name of the Shotgun Software Inc nor the names of its -# contributors may be used to endorse or promote products derived from this -# software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +''' + ----------------------------------------------------------------------------- + Copyright (c) 2009-2011, Shotgun Software Inc + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + - Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + - Neither the name of the Shotgun Software Inc nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +''' import base64 @@ -47,12 +49,13 @@ from lib.httplib2 import Http from lib.sgtimezone import SgTimezone -log = logging.getLogger("shotgun_api3") +LOG = logging.getLogger("shotgun_api3") +SG_TIMEZONE = SgTimezone() try: import simplejson as json except ImportError: - log.debug("simplejson not found, dropping back to json") + LOG.debug("simplejson not found, dropping back to json") import json as json # ---------------------------------------------------------------------------- @@ -67,16 +70,18 @@ class ShotgunError(Exception): pass class Fault(ShotgunError): + """Exception when server side exception detected.""" pass # ---------------------------------------------------------------------------- # API class ServerCapabilities(object): + """Container for the servers capabilities, such as version and paging. + """ def __init__(self, host, meta): - """Container for the servers capabilities, such as version and - paging. + """ServerCapabilities.__init__ :param host: Host name for the server excluding protocol. @@ -108,7 +113,7 @@ def _ensure_json_supported(self): :raises ShotgunError: The current server version does not support json """ - if not self.version or self.version < (2,4,0): + if not self.version or self.version < (2, 4, 0): raise ShotgunError("JSON API requires server version 2.4 or "\ "higher, server is %s" % (self.version,)) @@ -118,16 +123,15 @@ def __str__(self): % (self.host, self.version, self.is_dev) class ClientCapabilities(object): + """Container for the client capabilities. + + Detects the current client platform and works out the SG field + used for local data paths. + """ def __init__(self): - """Container for the client capabilities. - - Detects the current client platform and works out the SG field - used for local data paths. - """ - system = platform.system().lower() - if system =='darwin': + if system == 'darwin': self.platform = "mac" elif system in ('windows','linux'): self.platform = system @@ -147,10 +151,9 @@ def __str__(self): self.py_version) class _Config(object): + """Container for the client configuration.""" def __init__(self): - """Container for the client configuration.""" - self.max_rpc_attempts = 3 self.timeout_secs = None self.api_ver = 'api3' @@ -180,8 +183,13 @@ class Shotgun(object): "^(\d{4})\D?(0[1-9]|1[0-2])\D?([12]\d|0[1-9]|3[01])"\ "(\D?([01]\d|2[0-3])\D?([0-5]\d)\D?([0-5]\d)?\D?(\d{3})?)?$") - def __init__(self, base_url, script_name, api_key, - convert_datetimes_to_utc=True, http_proxy=None, connect=True): + def __init__(self, + base_url, + script_name, + api_key, + convert_datetimes_to_utc=True, + http_proxy=None, + connect=True): """Initialises a new instance of the Shotgun client. :param base_url: http or https url to the shotgun server. @@ -229,9 +237,8 @@ def __init__(self, base_url, script_name, api_key, self.config.authorization = "Basic " + auth.strip() if http_proxy: - _, proxy_netloc, _, _, _ = urlparse.urlsplit(http_proxy) - self.config.proxy_server, _, proxy_port = proxy_netloc.partition( - ":") + proxy_netloc = urlparse.urlsplit(http_proxy)[1] + self.config.proxy_server, proxy_port = proxy_netloc.split(":", 1) self.config.proxy_port = int(proxy_port or 8080) self.client_caps = ClientCapabilities() @@ -239,13 +246,14 @@ def __init__(self, base_url, script_name, api_key, #test to ensure the the server supports the json API #call to server will only be made once and will raise error if connect: - sc = self.server_caps + self.server_caps # ======================================================================== # API Functions @property def server_info(self): + """Returns server information.""" return self.server_caps.server_info @property @@ -290,7 +298,28 @@ def find_one(self, entity_type, filters, fields=None, order=None, filter_operator=None, retired_only=False): """Calls the find() method and returns the first result, or None. - The params are the same as for find(). + :param entity_type: Required, entity type (string) to find. + + :param filters: Required, list of filters to apply. + + :param fields: Optional list of fields from the matched entities to + return. Defaults to id. + + :param order: Optional list of fields to order the results by, list + has the form [{'field_name':'foo','direction':'asc or desc'},] + + :param filter_operator: Optional operator to apply to the filters, + supported values are 'all' and 'any'. Defaults to 'all'. + + :param limit: Optional, number of entities to return per page. + Defaults to 0 which returns all entities that match. + + :param page: Optional, page of results to return. By default all + results are returned. Use together with limit. + + :param retired_only: Optional, flag to return only entities that have + been retried. Defaults to False which returns only entities which + have not been retired. """ results = self.find(entity_type, filters, fields, order, @@ -332,53 +361,24 @@ def find(self, entity_type, filters, fields=None, order=None, """ if not isinstance(limit, int) or limit < 0: - raise ValueError("limit parameter must be a positive integer") + raise ValueError("limit parameter must be a positive integer") if not isinstance(page, int) or page < 0: raise ValueError("page parameter must be a positive integer") if isinstance(filters, (list, tuple)): - new_filters = {} - - if not filter_operator or filter_operator == "all": - new_filters["logical_operator"] = "and" - else: - new_filters["logical_operator"] = "or" - - new_filters["conditions"] = [{"path":f[0], "relation":f[1], "values":f[2:]} for f in filters ] - - filters = new_filters + filters = _translate_filters(filters, filter_operator) elif filter_operator: #TODO:Not sure if this test is correct, replicated from prev api raise ShotgunError("Deprecated: Use of filter_operator for find()" " is not valid any more. See the documentation on find()") - params = { - "type" : entity_type, - "return_fields": fields or ["id"], - "filters": filters, - "return_only" : (retired_only and 'retired') or "active", - "return_paging_info" : True, - "paging": { - "entities_per_page": self.config.records_per_page, - "current_page": 1 - } - } + params = self._construct_read_parameters(entity_type, + fields, + filters, + retired_only, + order) - - if order: - sort_list = [] - for sort in order: - if sort.has_key('column'): - # TODO: warn about deprecation of 'column' param name - sort['field_name'] = sort['column'] - sort.setdefault("direction", "asc") - sort_list.append({ - 'field_name': sort['field_name'], - 'direction' : sort['direction'] - }) - params['sorts'] = sort_list - if limit and limit <= self.config.records_per_page: params["paging"]["entities_per_page"] = limit # If page isn't set and the limit doesn't require pagination, @@ -410,7 +410,43 @@ def find(self, entity_type, filters, fields=None, order=None, return self._parse_records(records) - def summarize(self, entity_type, filters, summary_fields, filter_operator=None, grouping=None): + + + def _construct_read_parameters(self, + entity_type, + fields, + filters, + retired_only, + order): + params = {} + params["type"] = entity_type + params["return_fields"] = fields or ["id"] + params["filters"] = filters + params["return_only"] = (retired_only and 'retired') or "active" + params["return_paging_info"] = True + params["paging"] = { "entities_per_page": self.config.records_per_page, + "current_page": 1 } + + if order: + sort_list = [] + for sort in order: + if sort.has_key('column'): + # TODO: warn about deprecation of 'column' param name + sort['field_name'] = sort['column'] + sort.setdefault("direction", "asc") + sort_list.append({ + 'field_name': sort['field_name'], + 'direction' : sort['direction'] + }) + params['sorts'] = sort_list + return params + + def summarize(self, + entity_type, + filters, + summary_fields, + filter_operator=None, + grouping=None): """ Return group and summary information for entity_type for summary_fields based on the given filters. @@ -419,9 +455,16 @@ def summarize(self, entity_type, filters, summary_fields, filter_operator=None, raise ValueError("summarize() 'filters' parameter must be a list") if not isinstance(grouping, list) and grouping != None: - raise ValueError("summarize() 'grouping' parameter must be a list or None") + msg = "summarize() 'grouping' parameter must be a list or None" + raise ValueError(msg) + + filters = _translate_filters(filters, filter_operator) + params = {"type": entity_type, + "summaries": summary_fields, + "filters": filters} + if grouping != None: + params['grouping'] = grouping - params = _create_summary_request(entity_type, filters, summary_fields, filter_operator, grouping) records = self._call_rpc('summarize', params) return records @@ -531,48 +574,35 @@ def batch(self, requests): calls = [] def _required_keys(message, required_keys, data): - if set(required_keys) - set(data.keys()): + missing = set(required_keys) - set(data.keys()) + if missing: raise ShotgunError("%s missing required key: %s. "\ "Value was: %s." % (message, ", ".join(missing), data)) for req in requests: - _required_keys("Batched request", ['request_type','entity_type'], - req) + _required_keys("Batched request", + ['request_type','entity_type'], + req) + request_params = {'request_type': req['request_type'], + "type" : req["entity_type"]} if req["request_type"] == "create": _required_keys("Batched create request", ['data'], req) - - calls.append({ - "request_type" : "create", - "type" : req["entity_type"], - "fields" : self._dict_to_list(req["data"]), - "return_fields" : req.get("return_fields") or["id"] - }) - + request_params['fields'] = self._dict_to_list(req["data"]) + request_params["return_fields"] = req.get("return_fields") or["id"] elif req["request_type"] == "update": - _required_keys("Batched update request", ['entity_id','data'], - req) - - calls.append({ - "request_type" : "update", - "type" : req["entity_type"], - "id" : req["entity_id"], - "fields" : self._dict_to_list(req["data"]), - }) - + _required_keys("Batched update request", + ['entity_id','data'], + req) + request_params['id'] = req['entity_id'] + request_params['fields'] = self._dict_to_list(req["data"]) elif req["request_type"] == "delete": _required_keys("Batched delete request", ['entity_id'], req) - - calls.append({ - "request_type" : "delete", - "type" : req["entity_type"], - "id" : req["entity_id"], - }) - + request_params['id'] = req['entity_id'] else: raise ShotgunError("Invalid request_type '%s' for batch" % ( - req["request_type"])) - + req["request_type"])) + calls.append(request_params) records = self._call_rpc("batch", calls) return self._parse_records(records) @@ -777,13 +807,14 @@ def upload(self, entity_type, entity_id, path, field_name=None, "not sure why.\nPath: %s\nUrl: %s\nError: %s" % ( path, url, str(result))) + # TODO: # we changed the result string in the middle of 1.8 to return the id # remove once everyone is > 1.8.3 r = str(result).split(":") - id = 0 + attachment_id = 0 if len(r) > 1: - id = int(str(result).split(":")[1].split("\n")[0]) - return id + attachment_id = int(str(result).split(":")[1].split("\n")[0]) + return attachment_id def download_attachment(self, attachment_id): """Gets the returns binary content of the specified attachment. @@ -852,7 +883,7 @@ def schema(self, entity_type): "instead" % entity_type) def entity_types(self): - raise ShotgunError("Deprecated: use schema_entity_read() instead") + raise ShotgunError("Deprecated: use schema_entity_read() instead") # ======================================================================== # RPC Functions @@ -862,7 +893,7 @@ def _call_rpc(self, method, params, include_script_name=True, first=False): """ - log.debug("Starting rpc call to %s with params %s" % ( + LOG.debug("Starting rpc call to %s with params %s" % ( method, params)) params = self._transform_outbound(params) @@ -876,7 +907,7 @@ def _call_rpc(self, method, params, include_script_name=True, first=False): } http_status, resp_headers, body = self._make_call("POST", self.config.api_path, encoded_payload, req_headers) - log.info("Completed rpc call to %s" % (method)) + LOG.info("Completed rpc call to %s" % (method)) self._parse_http_status(http_status) response = self._decode_response(resp_headers, body) @@ -964,9 +995,9 @@ def _http_request(self, verb, path, body, headers): """ url = urlparse.urlunparse((self.config.scheme, self.config.server, path, None, None, None)) - log.debug("Request is %s:%s" % (verb, url)) - log.debug("Request headers are %s" % headers) - log.debug("Request body is %s" % body) + LOG.debug("Request is %s:%s" % (verb, url)) + LOG.debug("Request headers are %s" % headers) + LOG.debug("Request body is %s" % body) conn = self._get_connection() resp, content = conn.request(url,method=verb, body=body, @@ -979,9 +1010,9 @@ def _http_request(self, verb, path, body, headers): ) resp_body = content - log.debug("Response status is %s %s" % http_status) - log.debug("Response headers are %s" % resp_headers) - log.debug("Response body is %s" % resp_body) + LOG.debug("Response status is %s %s" % http_status) + LOG.debug("Response headers are %s" % resp_headers) + LOG.debug("Response body is %s" % resp_body) return (http_status, resp_headers, resp_body) @@ -1063,8 +1094,8 @@ def _transform_outbound(self, data): if self.config.convert_datetimes_to_utc: def _change_tz(value): if value.tzinfo == None: - value = value.replace(tzinfo=sg_timezone.local) - return value.astimezone(sg_timezone.utc) + value = value.replace(tzinfo=SG_TIMEZONE.local) + return value.astimezone(SG_TIMEZONE.utc) else: _change_tz = None @@ -1102,11 +1133,11 @@ def _transform_inbound(self, data): #to the local time, otherwise it will fail to compare to datetimes #that do not have a time zone. if self.config.convert_datetimes_to_utc: - _change_tz = lambda x: x.replace(tzinfo=sg_timezone.utc)\ - .astimezone(sg_timezone.local)\ + _change_tz = lambda x: x.replace(tzinfo=SG_TIMEZONE.utc)\ + .astimezone(SG_TIMEZONE.local)\ .replace(tzinfo=None) else: - _change_tz = None + _change_tz = None def _inbound_visitor(value): if isinstance(value, basestring): @@ -1216,11 +1247,12 @@ def _build_thumb_url(self, entity_type, entity_id): # curl "https://foo.com/upload/get_thumbnail_url?entity_type=Version&entity_id=1" # 1 # /files/0000/0000/0012/232/shot_thumb.jpg.jpg - - url = "/upload/get_thumbnail_url?entity_type=%s&entity_id=%s" % ( - urllib.quote(entity_type), urllib.quote(str(entity_id))) + entity_info = {'e_type':urllib.quote(entity_type), + 'e_id':urllib.quote(str(entity_id))} + url = ("/upload/get_thumbnail_url?" + + "entity_type=%(e_type)s&entity_id=%(e_id)s" % entity_info) - _, _, body = self._make_call("GET", url, None, None) + body = self._make_call("GET", url, None, None)[2] code, thumb_url = body.splitlines() code = int(code) @@ -1294,10 +1326,13 @@ def encode(self, params, files, boundary=None, buffer=None): buffer.write('\r\n\r\n%s\r\n' % value) for (key, fd) in files: filename = fd.name.split('/')[-1] - content_type = mimetypes.guess_type(filename)[0] or 'application/octet-stream' + content_type = mimetypes.guess_type(filename)[0] + content_type = content_type or 'application/octet-stream' file_size = os.fstat(fd.fileno())[stat.ST_SIZE] buffer.write('--%s\r\n' % boundary) - buffer.write('Content-Disposition: form-data; name="%s"; filename="%s"\r\n' % (key, filename)) + c_dis = 'Content-Disposition: form-data; name="%s"; filename="%s"%s' + content_disposition = c_dis % (key, filename, '\r\n') + buffer.write(content_disposition) buffer.write('Content-Type: %s\r\n' % content_type) buffer.write('Content-Length: %s\r\n' % file_size) fd.seek(0) @@ -1310,32 +1345,16 @@ def https_request(self, request): return self.http_request(request) - - -sg_timezone = SgTimezone() - - -def _create_summary_request(entity_type, filters, summary_fields, filter_operator, grouping): - '''_create_summary_request assembles a request based on input''' - #TODO make this part of summary method? +def _translate_filters(filters, filter_operator): + '''_translate_filters translates filters params into data structure + expected by rpc call.''' new_filters = {} + if not filter_operator or filter_operator == "all": new_filters["logical_operator"] = "and" else: new_filters["logical_operator"] = "or" - - new_filters["conditions"] = [] - for f in filters: - new_filters["conditions"].append( {"path":f[0],"relation":f[1],"values":f[2:]} ) - - filters = new_filters - - req = { - "type": entity_type, - "summaries": summary_fields, - "filters": filters, - } - if grouping != None: - req['grouping'] = grouping - - return req + conditions = [{"path":f[0], "relation":f[1], "values":f[2:]} + for f in filters] + new_filters["conditions"] = conditions + return new_filters diff --git a/tests/tests_unit.py b/tests/tests_unit.py index d35764bd1..f35102cf3 100644 --- a/tests/tests_unit.py +++ b/tests/tests_unit.py @@ -25,57 +25,62 @@ def test_http_proxy(self): self.assertEquals(sg.config.proxy_port, proxy_port) -class TestCreateSummaryRequest(unittest.TestCase): +class TestShotgunSummarize(unittest.TestCase): '''Test case for _create_summary_request function and parameter validation as it exists in Shotgun.summarize. Does not require database connection or test data.''' - def setUp(self): - server_path = 'http://server_path' - script_name = 'script_name' - api_key = 'api_key' - - self.sg = api.Shotgun(server_path, - script_name, - api_key, + self.sg = api.Shotgun('http://server_path', + 'script_name', + 'api_key', connect=False) def test_filter_operator_none(self): expected_logical_operator = 'and' filter_operator = None - result = api.shotgun._create_summary_request('',[],None,filter_operator, None) + self._assert_filter_operator(expected_logical_operator, filter_operator) + + def _assert_filter_operator(self, expected_logical_operator, filter_operator): + result = self.get_call_rpc_params(None, {'filter_operator':filter_operator}) actual_logical_operator = result['filters']['logical_operator'] self.assertEqual(expected_logical_operator, actual_logical_operator) def test_filter_operator_all(self): expected_logical_operator = 'and' filter_operator = 'all' - result = api.shotgun._create_summary_request('',[],None,filter_operator, None) - actual_logical_operator = result['filters']['logical_operator'] - self.assertEqual(expected_logical_operator, actual_logical_operator) + self._assert_filter_operator(expected_logical_operator, filter_operator) - def test_filter_operator_none(self): + def test_filter_operator_or(self): expected_logical_operator = 'or' filter_operator = 'or' - result = api.shotgun._create_summary_request('',[],None,filter_operator, None) - actual_logical_operator = result['filters']['logical_operator'] - self.assertEqual(expected_logical_operator, actual_logical_operator) + self._assert_filter_operator(expected_logical_operator, filter_operator) def test_filters(self): path = 'path' relation = 'relation' value = 'value' - expected_condition = {'path':path, 'relation':relation, 'value':value} - result = api.shotgun._create_summary_request('', [[path, relation, value]], None, None, None) + expected_condition = {'path':path, 'relation':relation, 'values':[value]} + args = ['',[[path, relation, value]],None] + result = self.get_call_rpc_params(args, {}) actual_condition = result['filters']['conditions'][0] + self.assertEquals(expected_condition, actual_condition) + + @patch('shotgun_api3.Shotgun._call_rpc') + def get_call_rpc_params(self, args, kws, call_rpc): + '''Return params sent to _call_rpc from summarize.''' + if not args: + args = [None, [], None] + self.sg.summarize(*args, **kws) + return call_rpc.call_args[0][1] def test_grouping(self): - result = api.shotgun._create_summary_request('', [], None, None, None) + result = self.get_call_rpc_params(None, {}) self.assertFalse(result.has_key('grouping')) grouping = ['something'] - result = api.shotgun._create_summary_request('', [], None, None, grouping) + kws = {'grouping':grouping} + result = self.get_call_rpc_params(None, kws) self.assertEqual(grouping, result['grouping']) def test_filters_type(self): @@ -86,6 +91,36 @@ def test_grouping_type(self): '''test_grouping_type tests that grouping parameter is a list or None''' self.assertRaises(ValueError, self.sg.summarize, '', [], [], grouping='Not a list') +class TestShotgunBatch(unittest.TestCase): + def setUp(self): + self.sg = api.Shotgun('http://server_path', + 'script_name', + 'api_key', + connect=False) + + def test_missing_required_key(self): + req = {} + # requires keys request_type and entity_type + self.assertRaises(api.ShotgunError, self.sg.batch, [req]) + req['entity_type'] = 'Entity' + self.assertRaises(api.ShotgunError, self.sg.batch, [req]) + req['request_type'] = 'not_real_type' + self.assertRaises(api.ShotgunError, self.sg.batch, [req]) + # create requires data key + req['request_type'] = 'create' + self.assertRaises(api.ShotgunError, self.sg.batch, [req]) + # update requires entity_id and data + req['request_type'] = 'update' + req['data'] = {} + self.assertRaises(api.ShotgunError, self.sg.batch, [req]) + del req['data'] + req['entity_id'] = 2334 + self.assertRaises(api.ShotgunError, self.sg.batch, [req]) + # delete requires entity_id + req['request_type'] = 'delete' + del req['entity_id'] + self.assertRaises(api.ShotgunError, self.sg.batch, [req]) + class TestServerCapabilities(unittest.TestCase): def test_no_server_version(self): @@ -129,9 +164,8 @@ def assert_platform(self, sys_ret_val, expected, mock_platform): def test_no_platform(self, mock_platform): mock_platform.system.return_value = "unsupported" client_caps = api.shotgun.ClientCapabilities() - self.assertIsNone(client_caps.platform) - self.assertIsNone(client_caps.local_path_field) - + self.assertEquals(client_caps.platform, None) + self.assertEquals(client_caps.local_path_field, None) @patch('shotgun_api3.shotgun.sys') def test_py_version(self, mock_sys): From fb14144c1d7ef32c872ad2abf430397c2e05eb46 Mon Sep 17 00:00:00 2001 From: kennedy behrman Date: Mon, 25 Jul 2011 15:06:06 -0700 Subject: [PATCH 020/570] added command line option for tests_unit --- tests/tests_unit.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/tests_unit.py b/tests/tests_unit.py index f35102cf3..15c433b56 100644 --- a/tests/tests_unit.py +++ b/tests/tests_unit.py @@ -177,7 +177,8 @@ def test_py_version(self, mock_sys): client_caps = api.shotgun.ClientCapabilities() self.assertEquals(client_caps.py_version, expected_py_version) - +if __name__ == '__main__': + unittest.main() From 8238e9238690bf04706089babd4e8e78e5bd8f3a Mon Sep 17 00:00:00 2001 From: kennedy behrman Date: Tue, 26 Jul 2011 22:46:02 -0700 Subject: [PATCH 021/570] adding example config --- tests/example_config | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 tests/example_config diff --git a/tests/example_config b/tests/example_config new file mode 100644 index 000000000..87d8b51bc --- /dev/null +++ b/tests/example_config @@ -0,0 +1,33 @@ +# Example server and login information to use for tests. This file should be renamed +# to config and the appropriate values should be added. +# +# The unit tests can either connect to a server or use a mock server connection. +# When connected to a server they use the entity id's listed below to add, +# update and delete values. They also add, update and delete schema fields. +# +# For more details see the test_client.py file. + +[TEST_RUN_OPTIONS] +mock:True + +[SERVER_INFO] + +# Full url to the Shotgun server server +# e.g. http://my-company.shotgunstudio.com +server_url : http://0.0.0.0:3000 +# script name as key as listed in admin panel for your server +script_name : test script name +api_key : test script api key +http_proxy : +session_uuid : + +[TEST_DATA] +project_name : SG unittest project + +human_name : Sg unittest human +human_login : sgunittesthuman +human_password : human password + +asset_code : Sg unittest asset +version_code : Sg unittest version +shot_code : Sg unittest shot From c8e3bb9efa6f36cf4b5fc181133d7878a6cca989 Mon Sep 17 00:00:00 2001 From: kennedy behrman Date: Wed, 27 Jul 2011 11:24:26 -0700 Subject: [PATCH 022/570] added ensure_ascii flag --- README.mdown | 16 +++++++++++-- shotgun_api3/shotgun.py | 50 ++++++++++++++++++++++++++++++----------- tests/base.py | 6 +++++ tests/example_config | 13 ++--------- tests/test_api.py | 32 ++++++++++++++++++++++++++ 5 files changed, 91 insertions(+), 26 deletions(-) diff --git a/README.mdown b/README.mdown index 7d6ee42ac..1bb814b69 100644 --- a/README.mdown +++ b/README.mdown @@ -13,7 +13,6 @@ For Python 2.6 and higher, we will use the built-in json module... but installin ## Known Issues -Does not yet support the new summarize method from the summary_grouping branch. @@ -31,7 +30,7 @@ Shotgun provides a simple Python-based API for accessing Shotgun and integrating - Python v2.4 - v2.7. (We do have plans to eventually support Python 3) ## Installing -To use Shotgun's Python API module, you need to place it in one of the directories specified by the environment variable PYTHONPATH. For more information on PYTHONPATH and using modules in Python, see http://docs.python.org/tutorial/modules.html +To use Shotgun's Python API module, you need to place the package shotgun_api3 in one of the directories specified by the environment variable PYTHONPATH. For more information on PYTHONPATH and using modules in Python, see http://docs.python.org/tutorial/modules.html ## Documentation Tutorials and detailed documentation about the Python API are available on the [Shotgun Support website](https://support.shotgunsoftware.com/forums/48807-developer-api-info). @@ -42,10 +41,23 @@ Some useful direct links within this section are below: * [Reference: Data Types](https://github.com/shotgunsoftware/python-api/wiki/Reference%3A-Data-Types) * [Reference: Filter Syntax](https://github.com/shotgunsoftware/python-api/wiki/Reference%3A-Filter-Syntax) +## Tests +Integration and unit tests are provided. +- test_client and tests_unit mock server interaction and do not require a shotgun instance to be available. +- test_api and test_api_long do require a shotgun instance, with a script key available for the tests. These tests rely on a tests/config file, which can be created by renaming example_config and supplying the server and script user values. The tests will set up test data on the server based on the data set forth in the config. This data will be manipulated by the tests, and should not be used for other purposes. +- To run all of the tests, use the shell script run-tests. This script require nose to be installed. + ## Changelog (Note that many of these items may be specific to the XML-RPC version of the API and may not apply to the Python JSON API) +**v3.0.8 - 2011 July 27** + + + summarize available in json api + + refactored single file into package + + tests added (Thanks to Aaron Morton https://github.com/amorton) + + added ensure_ascii parameter, insuring results are ascii + **v3.0.6 - 2010 Jan 25** + optimization: don't request paging_info unless required (and server support is available) diff --git a/shotgun_api3/shotgun.py b/shotgun_api3/shotgun.py index 60c42560b..84acaa689 100755 --- a/shotgun_api3/shotgun.py +++ b/shotgun_api3/shotgun.py @@ -60,7 +60,7 @@ # ---------------------------------------------------------------------------- # Version -__version__ = "3.1a1" +__version__ = "3.0.8" # ---------------------------------------------------------------------------- # Errors @@ -189,6 +189,7 @@ def __init__(self, api_key, convert_datetimes_to_utc=True, http_proxy=None, + ensure_ascii=True, connect=True): """Initialises a new instance of the Shotgun client. @@ -216,7 +217,6 @@ def __init__(self, self.config.script_name = script_name self.config.convert_datetimes_to_utc = convert_datetimes_to_utc self.config.proxy_info = http_proxy - self._connection = None base_url = (base_url or "").lower() @@ -241,6 +241,9 @@ def __init__(self, self.config.proxy_server, proxy_port = proxy_netloc.split(":", 1) self.config.proxy_port = int(proxy_port or 8080) + if ensure_ascii: + self._json_loads = self._json_loads_ascii + self.client_caps = ClientCapabilities() self._server_caps = None #test to ensure the the server supports the json API @@ -1045,12 +1048,39 @@ def _decode_response(self, headers, body): ct = (headers.get("content-type") or "application/json").lower() - if ct.startswith("application/json") or \ - ct.startswith("text/javascript"): - return json.loads(body) - + if ct.startswith("application/json") or ct.startswith("text/javascript"): + return self._json_loads(body) return body + def _json_loads(self, body): + return json.loads(body) + + def _json_loads_ascii(self, body): + '''See http://stackoverflow.com/questions/956867''' + def _decode_list(lst): + newlist = [] + for i in lst: + if isinstance(i, unicode): + i = i.encode('utf-8') + elif isinstance(i, list): + i = _decode_list(i) + newlist.append(i) + return newlist + + def _decode_dict(dct): + newdict = {} + for k, v in dct.iteritems(): + if isinstance(k, unicode): + k = k.encode('utf-8') + if isinstance(v, unicode): + v = v.encode('utf-8') + elif isinstance(v, list): + v = _decode_list(v) + newdict[k] = v + return newdict + return json.loads(body, object_hook=_decode_dict) + + def _response_errors(self, sg_response): """Raises any API errors specified in the response. @@ -1280,13 +1310,7 @@ def _dict_to_list(self, d, key_name="field_name", value_name="value"): for k,v in (d or {}).iteritems() ] - - - - - - -# ---------------------------------------------------------------------------- + # Helpers from the previous API, left as is. # Based on http://code.activestate.com/recipes/146306/ diff --git a/tests/base.py b/tests/base.py index 7d0f38e3c..d2793b3a6 100644 --- a/tests/base.py +++ b/tests/base.py @@ -186,6 +186,12 @@ def _setup_db(self, config): 'project':self.project} self.shot = _find_or_create_entity(self.sg, 'Shot', data, keys) + keys = ['project','user'] + data = {'project':self.project, + 'user':self.human_user, + 'content':'anything'} + self.note = _find_or_create_entity(self.sg, 'Note', data, keys) + class SgTestConfig(object): '''Reads test config and holds values''' diff --git a/tests/example_config b/tests/example_config index 87d8b51bc..3d6439e15 100644 --- a/tests/example_config +++ b/tests/example_config @@ -1,14 +1,5 @@ -# Example server and login information to use for tests. This file should be renamed -# to config and the appropriate values should be added. -# -# The unit tests can either connect to a server or use a mock server connection. -# When connected to a server they use the entity id's listed below to add, -# update and delete values. They also add, update and delete schema fields. -# -# For more details see the test_client.py file. - -[TEST_RUN_OPTIONS] -mock:True +# Example server and login information to use for tests which require a live server. +# This file should be renamed to config and the appropriate values should be added. [SERVER_INFO] diff --git a/tests/test_api.py b/tests/test_api.py index 645ce31af..7c1e538ef 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -14,6 +14,8 @@ class TestShotgunApi(base.LiveTestBase): def setUp(self): super(TestShotgunApi, self).setUp() + # give note unicode content + self.sg.update('Note', self.note['id'], {'content':u'La Pe\xf1a'}) def test_info(self): """Called info""" @@ -174,3 +176,33 @@ def test_simple_summary(self): assert(result['groups'][0]['summaries']) assert(result['summaries']) + def test_ensure_ascii(self): + '''test_ensure_ascii tests ensure_unicode flag.''' + sg_ascii = api.Shotgun(self.config.server_url, + self.config.script_name, + self.config.api_key, + ensure_ascii=True) + + result = sg_ascii.find_one('Note', [['id','is',self.note['id']]], fields=['content']) + self.assertFalse(_has_unicode(result)) + + + def test_ensure_unicode(self): + '''test_ensure_unicode tests ensure_unicode flag.''' + sg_unicode = api.Shotgun(self.config.server_url, + self.config.script_name, + self.config.api_key, + ensure_ascii=False) + result = sg_unicode.find_one('Note', [['id','is',self.note['id']]], fields=['content']) + print result + self.assertTrue(_has_unicode(result)) + +def _has_unicode(data): + for k, v in data.items(): + if (isinstance(k, unicode)): + return True + if (isinstance(v, unicode)): + return True + return False + + From 8efd618481a9c9eece8c528a8916ad863257d72a Mon Sep 17 00:00:00 2001 From: kennedy behrman Date: Wed, 27 Jul 2011 12:04:31 -0700 Subject: [PATCH 023/570] updated decode test to handle ascii or unicode setting --- tests/test_client.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/tests/test_client.py b/tests/test_client.py index 5fe79e7df..328420f7a 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -296,23 +296,37 @@ def test_encode_payload(self): j = self.sg._encode_payload(d) self.assertTrue(isinstance(j, str)) - def test_decode_response(self): + def test_decode_response_ascii(self): + self._assert_decode_resonse(True, u"my data \u00E0".encode('utf8')) + + def test_decode_response_unicode(self): + self._assert_decode_resonse(False, u"my data \u00E0") + + def _assert_decode_resonse(self, ensure_ascii, data): """HTTP Response is decoded as JSON or text""" headers = { "content-type" : "application/json;charset=utf-8" } d = { - "this is " : u"my data \u00E0" + "this is " : data } - j = json.dumps(d, ensure_ascii=False, encoding="utf-8") - self.assertEqual(d, self.sg._decode_response(headers, j)) + sg = api.Shotgun(self.config.server_url, + self.config.script_name, + self.config.api_key, + http_proxy=self.config.http_proxy, + ensure_ascii = ensure_ascii, + connect=False) + + j = json.dumps(d, ensure_ascii=ensure_ascii, encoding="utf-8") + self.assertEqual(d, sg._decode_response(headers, j)) headers["content-type"] = "text/javascript" - self.assertEqual(d, self.sg._decode_response(headers, j)) + self.assertEqual(d, sg._decode_response(headers, j)) headers["content-type"] = "text/foo" - self.assertEqual(j, self.sg._decode_response(headers, j)) + self.assertEqual(j, sg._decode_response(headers, j)) + def test_parse_records(self): """Parse records to replace thumbnail and local paths""" From d35782ea0877a2d73d7c0ec92679404703be4d2f Mon Sep 17 00:00:00 2001 From: kbehrman Date: Tue, 2 Aug 2011 15:55:07 -0700 Subject: [PATCH 024/570] added tests comparing data field set/get to result from xmlrpc; conformed utc/datetime behaviour to that of the xmlrpc wrapper --- README.mdown | 3 +- shotgun_api3/shotgun.py | 4 +- tests/base.py | 14 +++ tests/example_config | 1 + tests/test_api.py | 213 +++++++++++++++++++++++++++++++++++++++- tests/test_client.py | 4 +- 6 files changed, 231 insertions(+), 8 deletions(-) diff --git a/README.mdown b/README.mdown index 1bb814b69..6d73ec5b3 100644 --- a/README.mdown +++ b/README.mdown @@ -12,8 +12,7 @@ For Python 2.6 and higher, we will use the built-in json module... but installin ## Known Issues - - +- Some older versions of shotgun 2.4 may not allow the setting of float fields using this api. # Shotgun Python API diff --git a/shotgun_api3/shotgun.py b/shotgun_api3/shotgun.py index 84acaa689..7a7cb3e3a 100755 --- a/shotgun_api3/shotgun.py +++ b/shotgun_api3/shotgun.py @@ -1158,14 +1158,12 @@ def _outbound_visitor(value): def _transform_inbound(self, data): """Transforms data types or values after they are received from the server.""" - #NOTE: The time zone is removed from the time after it is transformed #to the local time, otherwise it will fail to compare to datetimes #that do not have a time zone. if self.config.convert_datetimes_to_utc: _change_tz = lambda x: x.replace(tzinfo=SG_TIMEZONE.utc)\ - .astimezone(SG_TIMEZONE.local)\ - .replace(tzinfo=None) + .astimezone(SG_TIMEZONE.local) else: _change_tz = None diff --git a/tests/base.py b/tests/base.py index d2793b3a6..4b5ead44c 100644 --- a/tests/base.py +++ b/tests/base.py @@ -25,6 +25,8 @@ def __init__(self, *args, **kws): self.shot = None self.asset = None self.version = None + self.note = None + self.task = None self.human_password = None self.server_url = None self.connect = False @@ -158,6 +160,7 @@ class LiveTestBase(TestBase): '''Test base for tests relying on connection to server.''' def setUp(self): super(LiveTestBase, self).setUp() + self.sg_version = self.sg.info()['version'][:3] self._setup_db(self.config) def _setup_db(self, config): @@ -167,6 +170,10 @@ def _setup_db(self, config): data = {'name':self.config.human_name, 'login':self.config.human_login, 'password_proxy':self.config.human_password} + if self.sg_version >= (3, 0, 0): + data['locked_until'] = None + + self.human_user = _find_or_create_entity(self.sg, 'HumanUser', data) data = {'code':self.config.asset_code, @@ -192,6 +199,12 @@ def _setup_db(self, config): 'content':'anything'} self.note = _find_or_create_entity(self.sg, 'Note', data, keys) + keys = ['project', 'entity', 'content'] + data = {'project':self.project, + 'entity':self.asset, + 'content':self.config.task_content} + self.task = _find_or_create_entity(self.sg, 'Task', data, keys) + class SgTestConfig(object): '''Reads test config and holds values''' @@ -209,6 +222,7 @@ def __init__(self): self.asset_code = None self.version_code = None self.shot_code = None + self.task_content = None def read_config(self, config_path): diff --git a/tests/example_config b/tests/example_config index 3d6439e15..e034ac02e 100644 --- a/tests/example_config +++ b/tests/example_config @@ -22,3 +22,4 @@ human_password : human password asset_code : Sg unittest asset version_code : Sg unittest version shot_code : Sg unittest shot +task_content : Sg unittest task diff --git a/tests/test_api.py b/tests/test_api.py index 7c1e538ef..81320b96b 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -197,7 +197,218 @@ def test_ensure_unicode(self): print result self.assertTrue(_has_unicode(result)) -def _has_unicode(data): + +class TestDataTypes(base.LiveTestBase): + '''Test fields representing the different data types mapped on the server side. + + Untested data types: password, percent, pivot_column, serializable, image, currency + multi_entity, system_task_type, timecode, url, uuid + ''' + def setUp(self): + super(TestDataTypes, self).setUp() + + def test_set_checkbox(self): + entity = 'HumanUser' + entity_id = self.human_user['id'] + field_name = 'email_notes' + pos_values = [False, True] + expected, actual = self.assert_set_field(entity, + entity_id, + field_name, + pos_values) + self.assertEqual(expected, actual) + + + def test_set_color(self): + entity = 'Task' + entity_id = self.task['id'] + field_name = 'color' + pos_values = ['pipeline_step', '222,0,0'] + expected, actual = self.assert_set_field(entity, + entity_id, + field_name, + pos_values) + self.assertEqual(expected, actual) + + + def test_set_date(self): + entity = 'Task' + entity_id = self.task['id'] + field_name = 'due_date' + pos_values = ['2008-05-08', '2011-05-05'] + expected, actual = self.assert_set_field(entity, + entity_id, + field_name, + pos_values) + self.assertEqual(expected, actual) + + def test_set_date_time(self): + entity = 'HumanUser' + entity_id = self.human_user['id'] + field_name = 'locked_until' + local = api.shotgun.SG_TIMEZONE.local + dt_1 = datetime.datetime(2008, 10, 13, 23, 10, tzinfo=local) + dt_2 = datetime.datetime(2009, 10, 13, 23, 10, tzinfo=local) + pos_values = [dt_1, dt_2] + expected, actual = self.assert_set_field(entity, + entity_id, + field_name, + pos_values) + self.assertEqual(expected, actual) + + def test_set_duration(self): + entity = 'Task' + entity_id = self.task['id'] + field_name = 'duration' + pos_values = [2100, 1300] + expected, actual = self.assert_set_field(entity, + entity_id, + field_name, + pos_values) + self.assertEqual(expected, actual) + + def test_set_entity(self): + entity = 'Task' + entity_id = self.task['id'] + field_name = 'entity' + pos_values = [self.asset, self.shot] + expected, actual = self.assert_set_field(entity, + entity_id, + field_name, + pos_values) + self.assertEqual(expected['id'], actual['id']) + + def test_set_float(self): + entity = 'Version' + entity_id = self.version['id'] + field_name = 'sg_movie_aspect_ratio' + pos_values = [2.0, 3.0] + expected, actual = self.assert_set_field(entity, + entity_id, + field_name, + pos_values) + self.assertEqual(expected, actual) + + + def test_set_list(self): + entity = 'Note' + entity_id = self.note['id'] + field_name = 'read_by_current_user' + pos_values = ['read','unread'] + expected, actual = self.assert_set_field(entity, + entity_id, + field_name, + pos_values) + self.assertEqual(expected, actual) + + + def test_set_number(self): + entity = 'Shot' + entity_id = self.shot['id'] + field_name = 'head_in' + pos_values = [2300, 1300] + expected, actual = self.assert_set_field(entity, + entity_id, + field_name, + pos_values) + self.assertEqual(expected, actual) + + def test_set_status_list(self): + entity = 'Task' + entity_id = self.task['id'] + field_name = 'sg_status_list' + pos_values = ['rdy','fin'] + expected, actual = self.assert_set_field(entity, + entity_id, + field_name, + pos_values) + self.assertEqual(expected, actual) + + def test_set_status_list(self): + entity = 'Task' + entity_id = self.task['id'] + field_name = 'sg_status_list' + pos_values = ['rdy','fin'] + expected, actual = self.assert_set_field(entity, + entity_id, + field_name, + pos_values) + self.assertEqual(expected, actual) + + def test_set_tag_list(self): + entity = 'Task' + entity_id = self.task['id'] + field_name = 'tag_list' + pos_values = [['a','b'],['c']] + expected, actual = self.assert_set_field(entity, + entity_id, + field_name, + pos_values) + self.assertEqual(expected, actual) + + def test_set_text(self): + entity = 'Note' + entity_id = self.note['id'] + field_name = 'content' + pos_values = ['this content', 'that content'] + expected, actual = self.assert_set_field(entity, + entity_id, + field_name, + pos_values) + self.assertEqual(expected, actual) + + def assert_set_field(self, entity, entity_id, field_name, pos_values): + query_result = self.sg.find_one(entity, + [['id', 'is', entity_id]], + [field_name]) + initial_value = query_result[field_name] + new_value = (initial_value == pos_values[0] and pos_values[1]) or pos_values[0] + self.sg.update(entity, entity_id, {field_name:new_value}) + new_values = self.sg.find_one(entity, + [['id', 'is', entity_id]], + [field_name]) + return new_value, new_values[field_name] + +class TestUtc(base.LiveTestBase): + '''Test utc options''' + + def setUp(self): + super(TestUtc, self).setUp() + utc = api.shotgun.SG_TIMEZONE.utc + self.datetime_utc = datetime.datetime(2008, 10, 13, 23, 10, tzinfo=utc) + local = api.shotgun.SG_TIMEZONE.local + self.datetime_local = datetime.datetime(2008, 10, 13, 23, 10, tzinfo=local) + self.datetime_none = datetime.datetime(2008, 10, 13, 23, 10) + + def test_convert_to_utc(self): + sg_utc= api.Shotgun(self.config.server_url, + self.config.script_name, + self.config.api_key, + http_proxy=self.config.http_proxy, + convert_datetimes_to_utc=True) + self._assert_expected(sg_utc, self.datetime_none, self.datetime_local) + self._assert_expected(sg_utc, self.datetime_local, self.datetime_local) + + def test_no_convert_to_utc(self): + sg_no_utc= api.Shotgun(self.config.server_url, + self.config.script_name, + self.config.api_key, + http_proxy=self.config.http_proxy, + convert_datetimes_to_utc=False) + self._assert_expected(sg_no_utc, self.datetime_none, self.datetime_none) + self._assert_expected(sg_no_utc, self.datetime_utc, self.datetime_none) + + def _assert_expected(self, sg, date_time, expected): + entity_name = 'HumanUser' + entity_id = self.human_user['id'] + field_name = 'locked_until' + sg.update(entity_name, entity_id, {field_name:date_time}) + result = sg.find_one(entity_name, [['id','is',entity_id]],[field_name]) + self.assertEqual(result[field_name], expected) + + + +def _has_unicode(data): for k, v in data.items(): if (isinstance(k, unicode)): return True diff --git a/tests/test_client.py b/tests/test_client.py index 328420f7a..2fb0a50dc 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -17,7 +17,7 @@ import shotgun_api3.lib.httplib2 as httplib2 import shotgun_api3 as api -from shotgun_api3.shotgun import ServerCapabilities +from shotgun_api3.shotgun import ServerCapabilities, SG_TIMEZONE import base class TestShotgunClient(base.MockTestBase): @@ -231,7 +231,7 @@ def test_transform_data(self): timestamp = time.time() #microseconds will be last during transforms now = datetime.datetime.fromtimestamp(timestamp).replace( - microsecond=0) + microsecond=0, tzinfo=SG_TIMEZONE.local) utc_now = datetime.datetime.utcfromtimestamp(timestamp).replace( microsecond=0) local = { From 1a66e9672fe776badbc375a7769a0a168a7f3177 Mon Sep 17 00:00:00 2001 From: kbehrman Date: Wed, 3 Aug 2011 08:20:47 -0700 Subject: [PATCH 025/570] exposed exceptions from xmlrpclib --- shotgun_api3/__init__.py | 4 +- shotgun_api3/lib/xmlrpclib.py | 341 ++++++++++++++++++++++++++++++++++ shotgun_api3/shotgun.py | 10 +- tests/test_api.py | 48 +++-- tests/test_client.py | 2 +- tests/tests_unit.py | 1 - 6 files changed, 390 insertions(+), 16 deletions(-) create mode 100644 shotgun_api3/lib/xmlrpclib.py diff --git a/shotgun_api3/__init__.py b/shotgun_api3/__init__.py index e1dbf1128..ebb4dd10a 100644 --- a/shotgun_api3/__init__.py +++ b/shotgun_api3/__init__.py @@ -1,2 +1,2 @@ -from shotgun import Shotgun, ShotgunError, Fault - +from shotgun import (Shotgun, ShotgunError, Fault, ProtocolError, ResponseError, + Error) diff --git a/shotgun_api3/lib/xmlrpclib.py b/shotgun_api3/lib/xmlrpclib.py new file mode 100644 index 000000000..ed2a90c1c --- /dev/null +++ b/shotgun_api3/lib/xmlrpclib.py @@ -0,0 +1,341 @@ +#! /opt/local/bin/python + +# XML-RPC CLIENT LIBRARY +# $Id: xmlrpclib.py 41594 2005-12-04 19:11:17Z andrew.kuchling $ +# +# an XML-RPC client interface for Python. +# +# the marshalling and response parser code can also be used to +# implement XML-RPC servers. +# +# Notes: +# this version is designed to work with Python 2.1 or newer. +# +# History: +# 1999-01-14 fl Created +# 1999-01-15 fl Changed dateTime to use localtime +# 1999-01-16 fl Added Binary/base64 element, default to RPC2 service +# 1999-01-19 fl Fixed array data element (from Skip Montanaro) +# 1999-01-21 fl Fixed dateTime constructor, etc. +# 1999-02-02 fl Added fault handling, handle empty sequences, etc. +# 1999-02-10 fl Fixed problem with empty responses (from Skip Montanaro) +# 1999-06-20 fl Speed improvements, pluggable parsers/transports (0.9.8) +# 2000-11-28 fl Changed boolean to check the truth value of its argument +# 2001-02-24 fl Added encoding/Unicode/SafeTransport patches +# 2001-02-26 fl Added compare support to wrappers (0.9.9/1.0b1) +# 2001-03-28 fl Make sure response tuple is a singleton +# 2001-03-29 fl Don't require empty params element (from Nicholas Riley) +# 2001-06-10 fl Folded in _xmlrpclib accelerator support (1.0b2) +# 2001-08-20 fl Base xmlrpclib.Error on built-in Exception (from Paul Prescod) +# 2001-09-03 fl Allow Transport subclass to override getparser +# 2001-09-10 fl Lazy import of urllib, cgi, xmllib (20x import speedup) +# 2001-10-01 fl Remove containers from memo cache when done with them +# 2001-10-01 fl Use faster escape method (80% dumps speedup) +# 2001-10-02 fl More dumps microtuning +# 2001-10-04 fl Make sure import expat gets a parser (from Guido van Rossum) +# 2001-10-10 sm Allow long ints to be passed as ints if they don't overflow +# 2001-10-17 sm Test for int and long overflow (allows use on 64-bit systems) +# 2001-11-12 fl Use repr() to marshal doubles (from Paul Felix) +# 2002-03-17 fl Avoid buffered read when possible (from James Rucker) +# 2002-04-07 fl Added pythondoc comments +# 2002-04-16 fl Added __str__ methods to datetime/binary wrappers +# 2002-05-15 fl Added error constants (from Andrew Kuchling) +# 2002-06-27 fl Merged with Python CVS version +# 2002-10-22 fl Added basic authentication (based on code from Phillip Eby) +# 2003-01-22 sm Add support for the bool type +# 2003-02-27 gvr Remove apply calls +# 2003-04-24 sm Use cStringIO if available +# 2003-04-25 ak Add support for nil +# 2003-06-15 gn Add support for time.struct_time +# 2003-07-12 gp Correct marshalling of Faults +# 2003-10-31 mvl Add multicall support +# 2004-08-20 mvl Bump minimum supported Python version to 2.1 +# +# Copyright (c) 1999-2002 by Secret Labs AB. +# Copyright (c) 1999-2002 by Fredrik Lundh. +# +# info@pythonware.com +# http://www.pythonware.com +# +# -------------------------------------------------------------------- +# The XML-RPC client interface is +# +# Copyright (c) 1999-2002 by Secret Labs AB +# Copyright (c) 1999-2002 by Fredrik Lundh +# +# By obtaining, using, and/or copying this software and/or its +# associated documentation, you agree that you have read, understood, +# and will comply with the following terms and conditions: +# +# Permission to use, copy, modify, and distribute this software and +# its associated documentation for any purpose and without fee is +# hereby granted, provided that the above copyright notice appears in +# all copies, and that both that copyright notice and this permission +# notice appear in supporting documentation, and that the name of +# Secret Labs AB or the author not be used in advertising or publicity +# pertaining to distribution of the software without specific, written +# prior permission. +# +# SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD +# TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANT- +# ABILITY AND FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR +# BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY +# DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS +# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE +# OF THIS SOFTWARE. +# -------------------------------------------------------------------- + +# +# things to look into some day: + +# TODO: sort out True/False/boolean issues for Python 2.3 + +""" +An XML-RPC client interface for Python. + +The marshalling and response parser code can also be used to +implement XML-RPC servers. + +Exported exceptions: + + Error Base class for client errors + ProtocolError Indicates an HTTP protocol error + ResponseError Indicates a broken response package + Fault Indicates an XML-RPC fault package + +Exported classes: + + ServerProxy Represents a logical connection to an XML-RPC server + + MultiCall Executor of boxcared xmlrpc requests + Boolean boolean wrapper to generate a "boolean" XML-RPC value + DateTime dateTime wrapper for an ISO 8601 string or time tuple or + localtime integer value to generate a "dateTime.iso8601" + XML-RPC value + Binary binary data wrapper + + SlowParser Slow but safe standard parser (based on xmllib) + Marshaller Generate an XML-RPC params chunk from a Python data structure + Unmarshaller Unmarshal an XML-RPC response from incoming XML event message + Transport Handles an HTTP transaction to an XML-RPC server + SafeTransport Handles an HTTPS transaction to an XML-RPC server + +Exported constants: + + True + False + +Exported functions: + + boolean Convert any Python value to an XML-RPC boolean + getparser Create instance of the fastest available parser & attach + to an unmarshalling object + dumps Convert an argument tuple or a Fault instance to an XML-RPC + request (or response, if the methodresponse option is used). + loads Convert an XML-RPC packet to unmarshalled data plus a method + name (None if not present). +""" + +import re, string, time, operator + +from types import * +import socket +import errno +import httplib + +# -------------------------------------------------------------------- +# Internal stuff + +try: + unicode +except NameError: + unicode = None # unicode support not available + +try: + import datetime + #import sg_timezone +except ImportError: + datetime = None + +try: + _bool_is_builtin = False.__class__.__name__ == "bool" +except NameError: + _bool_is_builtin = 0 + +def _decode(data, encoding, is8bit=re.compile("[\x80-\xff]").search): + # decode non-ascii string (if possible) + if unicode and encoding and is8bit(data): + data = unicode(data, encoding) + return data + +def escape(s, replace=string.replace): + s = replace(s, "&", "&") + s = replace(s, "<", "<") + return replace(s, ">", ">",) + +if unicode: + def _stringify(string): + # convert to 7-bit ascii if possible + try: + return string.encode("ascii") + except UnicodeError: + return string +else: + def _stringify(string): + return string + +#__version__ = "1.0.1" + +# xmlrpc integer limits +MAXINT = 2L**31-1 +MININT = -2L**31 + +# -------------------------------------------------------------------- +# Error constants (from Dan Libby's specification at +# http://xmlrpc-epi.sourceforge.net/specs/rfc.fault_codes.php) + +# Ranges of errors +PARSE_ERROR = -32700 +SERVER_ERROR = -32600 +APPLICATION_ERROR = -32500 +SYSTEM_ERROR = -32400 +TRANSPORT_ERROR = -32300 + +# Specific errors +NOT_WELLFORMED_ERROR = -32700 +UNSUPPORTED_ENCODING = -32701 +INVALID_ENCODING_CHAR = -32702 +INVALID_XMLRPC = -32600 +METHOD_NOT_FOUND = -32601 +INVALID_METHOD_PARAMS = -32602 +INTERNAL_ERROR = -32603 + +# -------------------------------------------------------------------- +# Exceptions + +## +# Base class for all kinds of client-side errors. + +class Error(Exception): + """Base class for client errors.""" + def __str__(self): + return repr(self) + +## +# Indicates an HTTP-level protocol error. This is raised by the HTTP +# transport layer, if the server returns an error code other than 200 +# (OK). +# +# @param url The target URL. +# @param errcode The HTTP error code. +# @param errmsg The HTTP error message. +# @param headers The HTTP header dictionary. + +class ProtocolError(Error): + """Indicates an HTTP protocol error.""" + def __init__(self, url, errcode, errmsg, headers): + Error.__init__(self) + self.url = url + self.errcode = errcode + self.errmsg = errmsg + self.headers = headers + def __repr__(self): + return ( + "" % + (self.url, self.errcode, self.errmsg) + ) + +## +# Indicates a broken XML-RPC response package. This exception is +# raised by the unmarshalling layer, if the XML-RPC response is +# malformed. + +class ResponseError(Error): + """Indicates a broken response package.""" + pass + +## +# Indicates an XML-RPC fault response package. This exception is +# raised by the unmarshalling layer, if the XML-RPC response contains +# a fault string. This exception can also used as a class, to +# generate a fault XML-RPC message. +# +# @param faultCode The XML-RPC fault code. +# @param faultString The XML-RPC fault string. + +class Fault(Error): + """Indicates an XML-RPC fault package.""" + def __init__(self, faultCode, faultString, **extra): + Error.__init__(self) + self.faultCode = faultCode + self.faultString = faultString + def __repr__(self): + return ( + "" % + (self.faultCode, repr(self.faultString)) + ) + +# -------------------------------------------------------------------- +# Special values + +## +# Wrapper for XML-RPC boolean values. Use the xmlrpclib.True and +# xmlrpclib.False constants, or the xmlrpclib.boolean() function, to +# generate boolean XML-RPC values. +# +# @param value A boolean value. Any true value is interpreted as True, +# all other values are interpreted as False. + +if _bool_is_builtin: + boolean = Boolean = bool + # to avoid breaking code which references xmlrpclib.{True,False} + True, False = True, False +else: + class Boolean: + """Boolean-value wrapper. + + Use True or False to generate a "boolean" XML-RPC value. + """ + + def __init__(self, value = 0): + self.value = operator.truth(value) + + def encode(self, out): + out.write("%d\n" % self.value) + + def __cmp__(self, other): + if isinstance(other, Boolean): + other = other.value + return cmp(self.value, other) + + def __repr__(self): + if self.value: + return "" % id(self) + else: + return "" % id(self) + + def __int__(self): + return self.value + + def __nonzero__(self): + return self.value + + True, False = Boolean(1), Boolean(0) + + ## + # Map true or false value to XML-RPC boolean values. + # + # @def boolean(value) + # @param value A boolean value. Any true value is mapped to True, + # all other values are mapped to False. + # @return xmlrpclib.True or xmlrpclib.False. + # @see Boolean + # @see True + # @see False + + def boolean(value, _truefalse=(False, True)): + """Convert any Python value to XML-RPC 'boolean'.""" + return _truefalse[operator.truth(value)] + + diff --git a/shotgun_api3/shotgun.py b/shotgun_api3/shotgun.py index 7a7cb3e3a..7b7f18c20 100755 --- a/shotgun_api3/shotgun.py +++ b/shotgun_api3/shotgun.py @@ -48,6 +48,7 @@ import urlparse from lib.httplib2 import Http from lib.sgtimezone import SgTimezone +from lib.xmlrpclib import Error, ProtocolError, ResponseError LOG = logging.getLogger("shotgun_api3") SG_TIMEZONE = SgTimezone() @@ -1026,9 +1027,16 @@ def _parse_http_status(self, status): :param status: Tuple of (code, reason). """ + error_code = status[0] + reason = status[1] if status[0] >= 300: - raise RuntimeError("HTTP error from server %s %s" % status) + msg = "HTTP error from server" + raise ProtocolError(self.config.server, + error_code, + reason, + msg) + return def _decode_response(self, headers, body): diff --git a/tests/test_api.py b/tests/test_api.py index 81320b96b..5af1991b8 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -6,9 +6,9 @@ import datetime import os +from mock import patch, Mock, MagicMock -import shotgun_api3 as api - +import shotgun_api3 import base class TestShotgunApi(base.LiveTestBase): @@ -157,8 +157,8 @@ def test_upload_download(self): def test_deprecated_functions(self): """Deprecated functions raise errors""" - self.assertRaises(api.ShotgunError, self.sg.schema, "foo") - self.assertRaises(api.ShotgunError, self.sg.entity_types) + self.assertRaises(shotgun_api3.ShotgunError, self.sg.schema, "foo") + self.assertRaises(shotgun_api3.ShotgunError, self.sg.entity_types) def test_simple_summary(self): @@ -178,7 +178,7 @@ def test_simple_summary(self): def test_ensure_ascii(self): '''test_ensure_ascii tests ensure_unicode flag.''' - sg_ascii = api.Shotgun(self.config.server_url, + sg_ascii = shotgun_api3.Shotgun(self.config.server_url, self.config.script_name, self.config.api_key, ensure_ascii=True) @@ -189,7 +189,7 @@ def test_ensure_ascii(self): def test_ensure_unicode(self): '''test_ensure_unicode tests ensure_unicode flag.''' - sg_unicode = api.Shotgun(self.config.server_url, + sg_unicode = shotgun_api3.Shotgun(self.config.server_url, self.config.script_name, self.config.api_key, ensure_ascii=False) @@ -246,7 +246,7 @@ def test_set_date_time(self): entity = 'HumanUser' entity_id = self.human_user['id'] field_name = 'locked_until' - local = api.shotgun.SG_TIMEZONE.local + local = shotgun_api3.shotgun.SG_TIMEZONE.local dt_1 = datetime.datetime(2008, 10, 13, 23, 10, tzinfo=local) dt_2 = datetime.datetime(2009, 10, 13, 23, 10, tzinfo=local) pos_values = [dt_1, dt_2] @@ -374,14 +374,14 @@ class TestUtc(base.LiveTestBase): def setUp(self): super(TestUtc, self).setUp() - utc = api.shotgun.SG_TIMEZONE.utc + utc = shotgun_api3.shotgun.SG_TIMEZONE.utc self.datetime_utc = datetime.datetime(2008, 10, 13, 23, 10, tzinfo=utc) - local = api.shotgun.SG_TIMEZONE.local + local = shotgun_api3.shotgun.SG_TIMEZONE.local self.datetime_local = datetime.datetime(2008, 10, 13, 23, 10, tzinfo=local) self.datetime_none = datetime.datetime(2008, 10, 13, 23, 10) def test_convert_to_utc(self): - sg_utc= api.Shotgun(self.config.server_url, + sg_utc= shotgun_api3.Shotgun(self.config.server_url, self.config.script_name, self.config.api_key, http_proxy=self.config.http_proxy, @@ -390,7 +390,7 @@ def test_convert_to_utc(self): self._assert_expected(sg_utc, self.datetime_local, self.datetime_local) def test_no_convert_to_utc(self): - sg_no_utc= api.Shotgun(self.config.server_url, + sg_no_utc= shotgun_api3.Shotgun(self.config.server_url, self.config.script_name, self.config.api_key, http_proxy=self.config.http_proxy, @@ -407,6 +407,32 @@ def _assert_expected(self, sg, date_time, expected): self.assertEqual(result[field_name], expected) +class TestErrors(base.TestBase): + def test_bad_auth(self): + '''test_bad_auth invalid script name or api key raises fault''' + server_url = self.config.server_url + script_name = 'not_real_script_name' + api_key = self.config.api_key + sg = shotgun_api3.Shotgun(server_url, script_name, api_key) + self.assertRaises(shotgun_api3.Fault, sg.find_one, 'Shot',[]) + + script_name = self.config.script_name + api_key = 'notrealapikey' + sg = shotgun_api3.Shotgun(server_url, script_name, api_key) + self.assertRaises(shotgun_api3.Fault, sg.find_one, 'Shot',[]) + + @patch('shotgun_api3.shotgun.Http.request') + def test_status_not_200(self, mock_request): + response = MagicMock(name="response mock", spec=dict) + response.status = 300 + response.reason = 'reason' + mock_request.return_value = (response, {}) + self.assertRaises(shotgun_api3.ProtocolError, self.sg.find_one, 'Shot', []) + +# def test_malformed_response(self): +# #TODO ResponseError +# pass + def _has_unicode(data): for k, v in data.items(): diff --git a/tests/test_client.py b/tests/test_client.py index 2fb0a50dc..b3ea2bce5 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -175,7 +175,7 @@ def test_http_error(self): self._mock_http( "big old error string", status=(500, "Internal Server Error")) - self.assertRaises(RuntimeError, self.sg.info) + self.assertRaises(api.ProtocolError, self.sg.info) self.assertEqual(1, self.sg._http_request.call_count, "Call is not repeated") diff --git a/tests/tests_unit.py b/tests/tests_unit.py index 15c433b56..1db521d0d 100644 --- a/tests/tests_unit.py +++ b/tests/tests_unit.py @@ -1,6 +1,5 @@ #! /opt/local/bin/python import unittest -from tests import base from mock import patch, Mock import shotgun_api3 as api From beaf2edc8fec19a9c5255dbcfa9984cdf1e1da1f Mon Sep 17 00:00:00 2001 From: kbehrman Date: Wed, 3 Aug 2011 11:09:39 -0700 Subject: [PATCH 026/570] added bundled simplejson as fallback for python versions not including json --- shotgun_api3/lib/simplejson/__init__.py | 438 +++ shotgun_api3/lib/simplejson/_speedups.c | 2652 +++++++++++++++++++ shotgun_api3/lib/simplejson/decoder.py | 421 +++ shotgun_api3/lib/simplejson/encoder.py | 503 ++++ shotgun_api3/lib/simplejson/ordered_dict.py | 119 + shotgun_api3/lib/simplejson/scanner.py | 77 + shotgun_api3/lib/simplejson/tool.py | 39 + shotgun_api3/shotgun.py | 9 +- 8 files changed, 4257 insertions(+), 1 deletion(-) create mode 100644 shotgun_api3/lib/simplejson/__init__.py create mode 100644 shotgun_api3/lib/simplejson/_speedups.c create mode 100644 shotgun_api3/lib/simplejson/decoder.py create mode 100644 shotgun_api3/lib/simplejson/encoder.py create mode 100644 shotgun_api3/lib/simplejson/ordered_dict.py create mode 100644 shotgun_api3/lib/simplejson/scanner.py create mode 100644 shotgun_api3/lib/simplejson/tool.py diff --git a/shotgun_api3/lib/simplejson/__init__.py b/shotgun_api3/lib/simplejson/__init__.py new file mode 100644 index 000000000..210b957a9 --- /dev/null +++ b/shotgun_api3/lib/simplejson/__init__.py @@ -0,0 +1,438 @@ +r"""JSON (JavaScript Object Notation) is a subset of +JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data +interchange format. + +:mod:`simplejson` exposes an API familiar to users of the standard library +:mod:`marshal` and :mod:`pickle` modules. It is the externally maintained +version of the :mod:`json` library contained in Python 2.6, but maintains +compatibility with Python 2.4 and Python 2.5 and (currently) has +significant performance advantages, even without using the optional C +extension for speedups. + +Encoding basic Python object hierarchies:: + + >>> import simplejson as json + >>> json.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}]) + '["foo", {"bar": ["baz", null, 1.0, 2]}]' + >>> print json.dumps("\"foo\bar") + "\"foo\bar" + >>> print json.dumps(u'\u1234') + "\u1234" + >>> print json.dumps('\\') + "\\" + >>> print json.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True) + {"a": 0, "b": 0, "c": 0} + >>> from StringIO import StringIO + >>> io = StringIO() + >>> json.dump(['streaming API'], io) + >>> io.getvalue() + '["streaming API"]' + +Compact encoding:: + + >>> import simplejson as json + >>> json.dumps([1,2,3,{'4': 5, '6': 7}], separators=(',',':')) + '[1,2,3,{"4":5,"6":7}]' + +Pretty printing:: + + >>> import simplejson as json + >>> s = json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=' ') + >>> print '\n'.join([l.rstrip() for l in s.splitlines()]) + { + "4": 5, + "6": 7 + } + +Decoding JSON:: + + >>> import simplejson as json + >>> obj = [u'foo', {u'bar': [u'baz', None, 1.0, 2]}] + >>> json.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') == obj + True + >>> json.loads('"\\"foo\\bar"') == u'"foo\x08ar' + True + >>> from StringIO import StringIO + >>> io = StringIO('["streaming API"]') + >>> json.load(io)[0] == 'streaming API' + True + +Specializing JSON object decoding:: + + >>> import simplejson as json + >>> def as_complex(dct): + ... if '__complex__' in dct: + ... return complex(dct['real'], dct['imag']) + ... return dct + ... + >>> json.loads('{"__complex__": true, "real": 1, "imag": 2}', + ... object_hook=as_complex) + (1+2j) + >>> from decimal import Decimal + >>> json.loads('1.1', parse_float=Decimal) == Decimal('1.1') + True + +Specializing JSON object encoding:: + + >>> import simplejson as json + >>> def encode_complex(obj): + ... if isinstance(obj, complex): + ... return [obj.real, obj.imag] + ... raise TypeError(repr(o) + " is not JSON serializable") + ... + >>> json.dumps(2 + 1j, default=encode_complex) + '[2.0, 1.0]' + >>> json.JSONEncoder(default=encode_complex).encode(2 + 1j) + '[2.0, 1.0]' + >>> ''.join(json.JSONEncoder(default=encode_complex).iterencode(2 + 1j)) + '[2.0, 1.0]' + + +Using simplejson.tool from the shell to validate and pretty-print:: + + $ echo '{"json":"obj"}' | python -m simplejson.tool + { + "json": "obj" + } + $ echo '{ 1.2:3.4}' | python -m simplejson.tool + Expecting property name: line 1 column 2 (char 2) +""" +__version__ = '2.1.6' +__all__ = [ + 'dump', 'dumps', 'load', 'loads', + 'JSONDecoder', 'JSONDecodeError', 'JSONEncoder', + 'OrderedDict', +] + +__author__ = 'Bob Ippolito ' + +from decimal import Decimal + +from decoder import JSONDecoder, JSONDecodeError +from encoder import JSONEncoder +def _import_OrderedDict(): + import collections + try: + return collections.OrderedDict + except AttributeError: + import ordered_dict + return ordered_dict.OrderedDict +OrderedDict = _import_OrderedDict() + +def _import_c_make_encoder(): + try: + from simplejson._speedups import make_encoder + return make_encoder + except ImportError: + return None + +_default_encoder = JSONEncoder( + skipkeys=False, + ensure_ascii=True, + check_circular=True, + allow_nan=True, + indent=None, + separators=None, + encoding='utf-8', + default=None, + use_decimal=False, +) + +def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True, + allow_nan=True, cls=None, indent=None, separators=None, + encoding='utf-8', default=None, use_decimal=False, **kw): + """Serialize ``obj`` as a JSON formatted stream to ``fp`` (a + ``.write()``-supporting file-like object). + + If ``skipkeys`` is true then ``dict`` keys that are not basic types + (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) + will be skipped instead of raising a ``TypeError``. + + If ``ensure_ascii`` is false, then the some chunks written to ``fp`` + may be ``unicode`` instances, subject to normal Python ``str`` to + ``unicode`` coercion rules. Unless ``fp.write()`` explicitly + understands ``unicode`` (as in ``codecs.getwriter()``) this is likely + to cause an error. + + If ``check_circular`` is false, then the circular reference check + for container types will be skipped and a circular reference will + result in an ``OverflowError`` (or worse). + + If ``allow_nan`` is false, then it will be a ``ValueError`` to + serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) + in strict compliance of the JSON specification, instead of using the + JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). + + If *indent* is a string, then JSON array elements and object members + will be pretty-printed with a newline followed by that string repeated + for each level of nesting. ``None`` (the default) selects the most compact + representation without any newlines. For backwards compatibility with + versions of simplejson earlier than 2.1.0, an integer is also accepted + and is converted to a string with that many spaces. + + If ``separators`` is an ``(item_separator, dict_separator)`` tuple + then it will be used instead of the default ``(', ', ': ')`` separators. + ``(',', ':')`` is the most compact JSON representation. + + ``encoding`` is the character encoding for str instances, default is UTF-8. + + ``default(obj)`` is a function that should return a serializable version + of obj or raise TypeError. The default simply raises TypeError. + + If *use_decimal* is true (default: ``False``) then decimal.Decimal + will be natively serialized to JSON with full precision. + + To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the + ``.default()`` method to serialize additional types), specify it with + the ``cls`` kwarg. + + """ + # cached encoder + if (not skipkeys and ensure_ascii and + check_circular and allow_nan and + cls is None and indent is None and separators is None and + encoding == 'utf-8' and default is None and not use_decimal + and not kw): + iterable = _default_encoder.iterencode(obj) + else: + if cls is None: + cls = JSONEncoder + iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii, + check_circular=check_circular, allow_nan=allow_nan, indent=indent, + separators=separators, encoding=encoding, + default=default, use_decimal=use_decimal, **kw).iterencode(obj) + # could accelerate with writelines in some versions of Python, at + # a debuggability cost + for chunk in iterable: + fp.write(chunk) + + +def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True, + allow_nan=True, cls=None, indent=None, separators=None, + encoding='utf-8', default=None, use_decimal=False, **kw): + """Serialize ``obj`` to a JSON formatted ``str``. + + If ``skipkeys`` is false then ``dict`` keys that are not basic types + (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) + will be skipped instead of raising a ``TypeError``. + + If ``ensure_ascii`` is false, then the return value will be a + ``unicode`` instance subject to normal Python ``str`` to ``unicode`` + coercion rules instead of being escaped to an ASCII ``str``. + + If ``check_circular`` is false, then the circular reference check + for container types will be skipped and a circular reference will + result in an ``OverflowError`` (or worse). + + If ``allow_nan`` is false, then it will be a ``ValueError`` to + serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in + strict compliance of the JSON specification, instead of using the + JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). + + If ``indent`` is a string, then JSON array elements and object members + will be pretty-printed with a newline followed by that string repeated + for each level of nesting. ``None`` (the default) selects the most compact + representation without any newlines. For backwards compatibility with + versions of simplejson earlier than 2.1.0, an integer is also accepted + and is converted to a string with that many spaces. + + If ``separators`` is an ``(item_separator, dict_separator)`` tuple + then it will be used instead of the default ``(', ', ': ')`` separators. + ``(',', ':')`` is the most compact JSON representation. + + ``encoding`` is the character encoding for str instances, default is UTF-8. + + ``default(obj)`` is a function that should return a serializable version + of obj or raise TypeError. The default simply raises TypeError. + + If *use_decimal* is true (default: ``False``) then decimal.Decimal + will be natively serialized to JSON with full precision. + + To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the + ``.default()`` method to serialize additional types), specify it with + the ``cls`` kwarg. + + """ + # cached encoder + if (not skipkeys and ensure_ascii and + check_circular and allow_nan and + cls is None and indent is None and separators is None and + encoding == 'utf-8' and default is None and not use_decimal + and not kw): + return _default_encoder.encode(obj) + if cls is None: + cls = JSONEncoder + return cls( + skipkeys=skipkeys, ensure_ascii=ensure_ascii, + check_circular=check_circular, allow_nan=allow_nan, indent=indent, + separators=separators, encoding=encoding, default=default, + use_decimal=use_decimal, **kw).encode(obj) + + +_default_decoder = JSONDecoder(encoding=None, object_hook=None, + object_pairs_hook=None) + + +def load(fp, encoding=None, cls=None, object_hook=None, parse_float=None, + parse_int=None, parse_constant=None, object_pairs_hook=None, + use_decimal=False, **kw): + """Deserialize ``fp`` (a ``.read()``-supporting file-like object containing + a JSON document) to a Python object. + + *encoding* determines the encoding used to interpret any + :class:`str` objects decoded by this instance (``'utf-8'`` by + default). It has no effect when decoding :class:`unicode` objects. + + Note that currently only encodings that are a superset of ASCII work, + strings of other encodings should be passed in as :class:`unicode`. + + *object_hook*, if specified, will be called with the result of every + JSON object decoded and its return value will be used in place of the + given :class:`dict`. This can be used to provide custom + deserializations (e.g. to support JSON-RPC class hinting). + + *object_pairs_hook* is an optional function that will be called with + the result of any object literal decode with an ordered list of pairs. + The return value of *object_pairs_hook* will be used instead of the + :class:`dict`. This feature can be used to implement custom decoders + that rely on the order that the key and value pairs are decoded (for + example, :func:`collections.OrderedDict` will remember the order of + insertion). If *object_hook* is also defined, the *object_pairs_hook* + takes priority. + + *parse_float*, if specified, will be called with the string of every + JSON float to be decoded. By default, this is equivalent to + ``float(num_str)``. This can be used to use another datatype or parser + for JSON floats (e.g. :class:`decimal.Decimal`). + + *parse_int*, if specified, will be called with the string of every + JSON int to be decoded. By default, this is equivalent to + ``int(num_str)``. This can be used to use another datatype or parser + for JSON integers (e.g. :class:`float`). + + *parse_constant*, if specified, will be called with one of the + following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This + can be used to raise an exception if invalid JSON numbers are + encountered. + + If *use_decimal* is true (default: ``False``) then it implies + parse_float=decimal.Decimal for parity with ``dump``. + + To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` + kwarg. + + """ + return loads(fp.read(), + encoding=encoding, cls=cls, object_hook=object_hook, + parse_float=parse_float, parse_int=parse_int, + parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, + use_decimal=use_decimal, **kw) + + +def loads(s, encoding=None, cls=None, object_hook=None, parse_float=None, + parse_int=None, parse_constant=None, object_pairs_hook=None, + use_decimal=False, **kw): + """Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON + document) to a Python object. + + *encoding* determines the encoding used to interpret any + :class:`str` objects decoded by this instance (``'utf-8'`` by + default). It has no effect when decoding :class:`unicode` objects. + + Note that currently only encodings that are a superset of ASCII work, + strings of other encodings should be passed in as :class:`unicode`. + + *object_hook*, if specified, will be called with the result of every + JSON object decoded and its return value will be used in place of the + given :class:`dict`. This can be used to provide custom + deserializations (e.g. to support JSON-RPC class hinting). + + *object_pairs_hook* is an optional function that will be called with + the result of any object literal decode with an ordered list of pairs. + The return value of *object_pairs_hook* will be used instead of the + :class:`dict`. This feature can be used to implement custom decoders + that rely on the order that the key and value pairs are decoded (for + example, :func:`collections.OrderedDict` will remember the order of + insertion). If *object_hook* is also defined, the *object_pairs_hook* + takes priority. + + *parse_float*, if specified, will be called with the string of every + JSON float to be decoded. By default, this is equivalent to + ``float(num_str)``. This can be used to use another datatype or parser + for JSON floats (e.g. :class:`decimal.Decimal`). + + *parse_int*, if specified, will be called with the string of every + JSON int to be decoded. By default, this is equivalent to + ``int(num_str)``. This can be used to use another datatype or parser + for JSON integers (e.g. :class:`float`). + + *parse_constant*, if specified, will be called with one of the + following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This + can be used to raise an exception if invalid JSON numbers are + encountered. + + If *use_decimal* is true (default: ``False``) then it implies + parse_float=decimal.Decimal for parity with ``dump``. + + To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` + kwarg. + + """ + if (cls is None and encoding is None and object_hook is None and + parse_int is None and parse_float is None and + parse_constant is None and object_pairs_hook is None + and not use_decimal and not kw): + return _default_decoder.decode(s) + if cls is None: + cls = JSONDecoder + if object_hook is not None: + kw['object_hook'] = object_hook + if object_pairs_hook is not None: + kw['object_pairs_hook'] = object_pairs_hook + if parse_float is not None: + kw['parse_float'] = parse_float + if parse_int is not None: + kw['parse_int'] = parse_int + if parse_constant is not None: + kw['parse_constant'] = parse_constant + if use_decimal: + if parse_float is not None: + raise TypeError("use_decimal=True implies parse_float=Decimal") + kw['parse_float'] = Decimal + return cls(encoding=encoding, **kw).decode(s) + + +def _toggle_speedups(enabled): + import simplejson.decoder as dec + import simplejson.encoder as enc + import simplejson.scanner as scan + c_make_encoder = _import_c_make_encoder() + if enabled: + dec.scanstring = dec.c_scanstring or dec.py_scanstring + enc.c_make_encoder = c_make_encoder + enc.encode_basestring_ascii = (enc.c_encode_basestring_ascii or + enc.py_encode_basestring_ascii) + scan.make_scanner = scan.c_make_scanner or scan.py_make_scanner + else: + dec.scanstring = dec.py_scanstring + enc.c_make_encoder = None + enc.encode_basestring_ascii = enc.py_encode_basestring_ascii + scan.make_scanner = scan.py_make_scanner + dec.make_scanner = scan.make_scanner + global _default_decoder + _default_decoder = JSONDecoder( + encoding=None, + object_hook=None, + object_pairs_hook=None, + ) + global _default_encoder + _default_encoder = JSONEncoder( + skipkeys=False, + ensure_ascii=True, + check_circular=True, + allow_nan=True, + indent=None, + separators=None, + encoding='utf-8', + default=None, + ) diff --git a/shotgun_api3/lib/simplejson/_speedups.c b/shotgun_api3/lib/simplejson/_speedups.c new file mode 100644 index 000000000..8b3474702 --- /dev/null +++ b/shotgun_api3/lib/simplejson/_speedups.c @@ -0,0 +1,2652 @@ +#include "Python.h" +#include "structmember.h" +#if PY_VERSION_HEX < 0x02070000 && !defined(PyOS_string_to_double) +#define PyOS_string_to_double json_PyOS_string_to_double +static double +json_PyOS_string_to_double(const char *s, char **endptr, PyObject *overflow_exception); +static double +json_PyOS_string_to_double(const char *s, char **endptr, PyObject *overflow_exception) { + double x; + assert(endptr == NULL); + assert(overflow_exception == NULL); + PyFPE_START_PROTECT("json_PyOS_string_to_double", return -1.0;) + x = PyOS_ascii_atof(s); + PyFPE_END_PROTECT(x) + return x; +} +#endif +#if PY_VERSION_HEX < 0x02060000 && !defined(Py_TYPE) +#define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) +#endif +#if PY_VERSION_HEX < 0x02060000 && !defined(Py_SIZE) +#define Py_SIZE(ob) (((PyVarObject*)(ob))->ob_size) +#endif +#if PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN) +typedef int Py_ssize_t; +#define PY_SSIZE_T_MAX INT_MAX +#define PY_SSIZE_T_MIN INT_MIN +#define PyInt_FromSsize_t PyInt_FromLong +#define PyInt_AsSsize_t PyInt_AsLong +#endif +#ifndef Py_IS_FINITE +#define Py_IS_FINITE(X) (!Py_IS_INFINITY(X) && !Py_IS_NAN(X)) +#endif + +#ifdef __GNUC__ +#define UNUSED __attribute__((__unused__)) +#else +#define UNUSED +#endif + +#define DEFAULT_ENCODING "utf-8" + +#define PyScanner_Check(op) PyObject_TypeCheck(op, &PyScannerType) +#define PyScanner_CheckExact(op) (Py_TYPE(op) == &PyScannerType) +#define PyEncoder_Check(op) PyObject_TypeCheck(op, &PyEncoderType) +#define PyEncoder_CheckExact(op) (Py_TYPE(op) == &PyEncoderType) +#define Decimal_Check(op) (PyObject_TypeCheck(op, DecimalTypePtr)) + +static PyTypeObject PyScannerType; +static PyTypeObject PyEncoderType; +static PyTypeObject *DecimalTypePtr; + +typedef struct _PyScannerObject { + PyObject_HEAD + PyObject *encoding; + PyObject *strict; + PyObject *object_hook; + PyObject *pairs_hook; + PyObject *parse_float; + PyObject *parse_int; + PyObject *parse_constant; + PyObject *memo; +} PyScannerObject; + +static PyMemberDef scanner_members[] = { + {"encoding", T_OBJECT, offsetof(PyScannerObject, encoding), READONLY, "encoding"}, + {"strict", T_OBJECT, offsetof(PyScannerObject, strict), READONLY, "strict"}, + {"object_hook", T_OBJECT, offsetof(PyScannerObject, object_hook), READONLY, "object_hook"}, + {"object_pairs_hook", T_OBJECT, offsetof(PyScannerObject, pairs_hook), READONLY, "object_pairs_hook"}, + {"parse_float", T_OBJECT, offsetof(PyScannerObject, parse_float), READONLY, "parse_float"}, + {"parse_int", T_OBJECT, offsetof(PyScannerObject, parse_int), READONLY, "parse_int"}, + {"parse_constant", T_OBJECT, offsetof(PyScannerObject, parse_constant), READONLY, "parse_constant"}, + {NULL} +}; + +typedef struct _PyEncoderObject { + PyObject_HEAD + PyObject *markers; + PyObject *defaultfn; + PyObject *encoder; + PyObject *indent; + PyObject *key_separator; + PyObject *item_separator; + PyObject *sort_keys; + PyObject *skipkeys; + PyObject *key_memo; + int fast_encode; + int allow_nan; + int use_decimal; +} PyEncoderObject; + +static PyMemberDef encoder_members[] = { + {"markers", T_OBJECT, offsetof(PyEncoderObject, markers), READONLY, "markers"}, + {"default", T_OBJECT, offsetof(PyEncoderObject, defaultfn), READONLY, "default"}, + {"encoder", T_OBJECT, offsetof(PyEncoderObject, encoder), READONLY, "encoder"}, + {"indent", T_OBJECT, offsetof(PyEncoderObject, indent), READONLY, "indent"}, + {"key_separator", T_OBJECT, offsetof(PyEncoderObject, key_separator), READONLY, "key_separator"}, + {"item_separator", T_OBJECT, offsetof(PyEncoderObject, item_separator), READONLY, "item_separator"}, + {"sort_keys", T_OBJECT, offsetof(PyEncoderObject, sort_keys), READONLY, "sort_keys"}, + {"skipkeys", T_OBJECT, offsetof(PyEncoderObject, skipkeys), READONLY, "skipkeys"}, + {"key_memo", T_OBJECT, offsetof(PyEncoderObject, key_memo), READONLY, "key_memo"}, + {NULL} +}; + +static Py_ssize_t +ascii_escape_char(Py_UNICODE c, char *output, Py_ssize_t chars); +static PyObject * +ascii_escape_unicode(PyObject *pystr); +static PyObject * +ascii_escape_str(PyObject *pystr); +static PyObject * +py_encode_basestring_ascii(PyObject* self UNUSED, PyObject *pystr); +void init_speedups(void); +static PyObject * +scan_once_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr); +static PyObject * +scan_once_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr); +static PyObject * +_build_rval_index_tuple(PyObject *rval, Py_ssize_t idx); +static PyObject * +scanner_new(PyTypeObject *type, PyObject *args, PyObject *kwds); +static int +scanner_init(PyObject *self, PyObject *args, PyObject *kwds); +static void +scanner_dealloc(PyObject *self); +static int +scanner_clear(PyObject *self); +static PyObject * +encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds); +static int +encoder_init(PyObject *self, PyObject *args, PyObject *kwds); +static void +encoder_dealloc(PyObject *self); +static int +encoder_clear(PyObject *self); +static int +encoder_listencode_list(PyEncoderObject *s, PyObject *rval, PyObject *seq, Py_ssize_t indent_level); +static int +encoder_listencode_obj(PyEncoderObject *s, PyObject *rval, PyObject *obj, Py_ssize_t indent_level); +static int +encoder_listencode_dict(PyEncoderObject *s, PyObject *rval, PyObject *dct, Py_ssize_t indent_level); +static PyObject * +_encoded_const(PyObject *obj); +static void +raise_errmsg(char *msg, PyObject *s, Py_ssize_t end); +static PyObject * +encoder_encode_string(PyEncoderObject *s, PyObject *obj); +static int +_convertPyInt_AsSsize_t(PyObject *o, Py_ssize_t *size_ptr); +static PyObject * +_convertPyInt_FromSsize_t(Py_ssize_t *size_ptr); +static PyObject * +encoder_encode_float(PyEncoderObject *s, PyObject *obj); + +#define S_CHAR(c) (c >= ' ' && c <= '~' && c != '\\' && c != '"') +#define IS_WHITESPACE(c) (((c) == ' ') || ((c) == '\t') || ((c) == '\n') || ((c) == '\r')) + +#define MIN_EXPANSION 6 +#ifdef Py_UNICODE_WIDE +#define MAX_EXPANSION (2 * MIN_EXPANSION) +#else +#define MAX_EXPANSION MIN_EXPANSION +#endif + +static int +_convertPyInt_AsSsize_t(PyObject *o, Py_ssize_t *size_ptr) +{ + /* PyObject to Py_ssize_t converter */ + *size_ptr = PyInt_AsSsize_t(o); + if (*size_ptr == -1 && PyErr_Occurred()) + return 0; + return 1; +} + +static PyObject * +_convertPyInt_FromSsize_t(Py_ssize_t *size_ptr) +{ + /* Py_ssize_t to PyObject converter */ + return PyInt_FromSsize_t(*size_ptr); +} + +static Py_ssize_t +ascii_escape_char(Py_UNICODE c, char *output, Py_ssize_t chars) +{ + /* Escape unicode code point c to ASCII escape sequences + in char *output. output must have at least 12 bytes unused to + accommodate an escaped surrogate pair "\uXXXX\uXXXX" */ + output[chars++] = '\\'; + switch (c) { + case '\\': output[chars++] = (char)c; break; + case '"': output[chars++] = (char)c; break; + case '\b': output[chars++] = 'b'; break; + case '\f': output[chars++] = 'f'; break; + case '\n': output[chars++] = 'n'; break; + case '\r': output[chars++] = 'r'; break; + case '\t': output[chars++] = 't'; break; + default: +#ifdef Py_UNICODE_WIDE + if (c >= 0x10000) { + /* UTF-16 surrogate pair */ + Py_UNICODE v = c - 0x10000; + c = 0xd800 | ((v >> 10) & 0x3ff); + output[chars++] = 'u'; + output[chars++] = "0123456789abcdef"[(c >> 12) & 0xf]; + output[chars++] = "0123456789abcdef"[(c >> 8) & 0xf]; + output[chars++] = "0123456789abcdef"[(c >> 4) & 0xf]; + output[chars++] = "0123456789abcdef"[(c ) & 0xf]; + c = 0xdc00 | (v & 0x3ff); + output[chars++] = '\\'; + } +#endif + output[chars++] = 'u'; + output[chars++] = "0123456789abcdef"[(c >> 12) & 0xf]; + output[chars++] = "0123456789abcdef"[(c >> 8) & 0xf]; + output[chars++] = "0123456789abcdef"[(c >> 4) & 0xf]; + output[chars++] = "0123456789abcdef"[(c ) & 0xf]; + } + return chars; +} + +static PyObject * +ascii_escape_unicode(PyObject *pystr) +{ + /* Take a PyUnicode pystr and return a new ASCII-only escaped PyString */ + Py_ssize_t i; + Py_ssize_t input_chars; + Py_ssize_t output_size; + Py_ssize_t max_output_size; + Py_ssize_t chars; + PyObject *rval; + char *output; + Py_UNICODE *input_unicode; + + input_chars = PyUnicode_GET_SIZE(pystr); + input_unicode = PyUnicode_AS_UNICODE(pystr); + + /* One char input can be up to 6 chars output, estimate 4 of these */ + output_size = 2 + (MIN_EXPANSION * 4) + input_chars; + max_output_size = 2 + (input_chars * MAX_EXPANSION); + rval = PyString_FromStringAndSize(NULL, output_size); + if (rval == NULL) { + return NULL; + } + output = PyString_AS_STRING(rval); + chars = 0; + output[chars++] = '"'; + for (i = 0; i < input_chars; i++) { + Py_UNICODE c = input_unicode[i]; + if (S_CHAR(c)) { + output[chars++] = (char)c; + } + else { + chars = ascii_escape_char(c, output, chars); + } + if (output_size - chars < (1 + MAX_EXPANSION)) { + /* There's more than four, so let's resize by a lot */ + Py_ssize_t new_output_size = output_size * 2; + /* This is an upper bound */ + if (new_output_size > max_output_size) { + new_output_size = max_output_size; + } + /* Make sure that the output size changed before resizing */ + if (new_output_size != output_size) { + output_size = new_output_size; + if (_PyString_Resize(&rval, output_size) == -1) { + return NULL; + } + output = PyString_AS_STRING(rval); + } + } + } + output[chars++] = '"'; + if (_PyString_Resize(&rval, chars) == -1) { + return NULL; + } + return rval; +} + +static PyObject * +ascii_escape_str(PyObject *pystr) +{ + /* Take a PyString pystr and return a new ASCII-only escaped PyString */ + Py_ssize_t i; + Py_ssize_t input_chars; + Py_ssize_t output_size; + Py_ssize_t chars; + PyObject *rval; + char *output; + char *input_str; + + input_chars = PyString_GET_SIZE(pystr); + input_str = PyString_AS_STRING(pystr); + + /* Fast path for a string that's already ASCII */ + for (i = 0; i < input_chars; i++) { + Py_UNICODE c = (Py_UNICODE)(unsigned char)input_str[i]; + if (!S_CHAR(c)) { + /* If we have to escape something, scan the string for unicode */ + Py_ssize_t j; + for (j = i; j < input_chars; j++) { + c = (Py_UNICODE)(unsigned char)input_str[j]; + if (c > 0x7f) { + /* We hit a non-ASCII character, bail to unicode mode */ + PyObject *uni; + uni = PyUnicode_DecodeUTF8(input_str, input_chars, "strict"); + if (uni == NULL) { + return NULL; + } + rval = ascii_escape_unicode(uni); + Py_DECREF(uni); + return rval; + } + } + break; + } + } + + if (i == input_chars) { + /* Input is already ASCII */ + output_size = 2 + input_chars; + } + else { + /* One char input can be up to 6 chars output, estimate 4 of these */ + output_size = 2 + (MIN_EXPANSION * 4) + input_chars; + } + rval = PyString_FromStringAndSize(NULL, output_size); + if (rval == NULL) { + return NULL; + } + output = PyString_AS_STRING(rval); + output[0] = '"'; + + /* We know that everything up to i is ASCII already */ + chars = i + 1; + memcpy(&output[1], input_str, i); + + for (; i < input_chars; i++) { + Py_UNICODE c = (Py_UNICODE)(unsigned char)input_str[i]; + if (S_CHAR(c)) { + output[chars++] = (char)c; + } + else { + chars = ascii_escape_char(c, output, chars); + } + /* An ASCII char can't possibly expand to a surrogate! */ + if (output_size - chars < (1 + MIN_EXPANSION)) { + /* There's more than four, so let's resize by a lot */ + output_size *= 2; + if (output_size > 2 + (input_chars * MIN_EXPANSION)) { + output_size = 2 + (input_chars * MIN_EXPANSION); + } + if (_PyString_Resize(&rval, output_size) == -1) { + return NULL; + } + output = PyString_AS_STRING(rval); + } + } + output[chars++] = '"'; + if (_PyString_Resize(&rval, chars) == -1) { + return NULL; + } + return rval; +} + +static void +raise_errmsg(char *msg, PyObject *s, Py_ssize_t end) +{ + /* Use the Python function simplejson.decoder.errmsg to raise a nice + looking ValueError exception */ + static PyObject *JSONDecodeError = NULL; + PyObject *exc; + if (JSONDecodeError == NULL) { + PyObject *decoder = PyImport_ImportModule("simplejson.decoder"); + if (decoder == NULL) + return; + JSONDecodeError = PyObject_GetAttrString(decoder, "JSONDecodeError"); + Py_DECREF(decoder); + if (JSONDecodeError == NULL) + return; + } + exc = PyObject_CallFunction(JSONDecodeError, "(zOO&)", msg, s, _convertPyInt_FromSsize_t, &end); + if (exc) { + PyErr_SetObject(JSONDecodeError, exc); + Py_DECREF(exc); + } +} + +static PyObject * +join_list_unicode(PyObject *lst) +{ + /* return u''.join(lst) */ + static PyObject *joinfn = NULL; + if (joinfn == NULL) { + PyObject *ustr = PyUnicode_FromUnicode(NULL, 0); + if (ustr == NULL) + return NULL; + + joinfn = PyObject_GetAttrString(ustr, "join"); + Py_DECREF(ustr); + if (joinfn == NULL) + return NULL; + } + return PyObject_CallFunctionObjArgs(joinfn, lst, NULL); +} + +static PyObject * +join_list_string(PyObject *lst) +{ + /* return ''.join(lst) */ + static PyObject *joinfn = NULL; + if (joinfn == NULL) { + PyObject *ustr = PyString_FromStringAndSize(NULL, 0); + if (ustr == NULL) + return NULL; + + joinfn = PyObject_GetAttrString(ustr, "join"); + Py_DECREF(ustr); + if (joinfn == NULL) + return NULL; + } + return PyObject_CallFunctionObjArgs(joinfn, lst, NULL); +} + +static PyObject * +_build_rval_index_tuple(PyObject *rval, Py_ssize_t idx) { + /* return (rval, idx) tuple, stealing reference to rval */ + PyObject *tpl; + PyObject *pyidx; + /* + steal a reference to rval, returns (rval, idx) + */ + if (rval == NULL) { + return NULL; + } + pyidx = PyInt_FromSsize_t(idx); + if (pyidx == NULL) { + Py_DECREF(rval); + return NULL; + } + tpl = PyTuple_New(2); + if (tpl == NULL) { + Py_DECREF(pyidx); + Py_DECREF(rval); + return NULL; + } + PyTuple_SET_ITEM(tpl, 0, rval); + PyTuple_SET_ITEM(tpl, 1, pyidx); + return tpl; +} + +#define APPEND_OLD_CHUNK \ + if (chunk != NULL) { \ + if (chunks == NULL) { \ + chunks = PyList_New(0); \ + if (chunks == NULL) { \ + goto bail; \ + } \ + } \ + if (PyList_Append(chunks, chunk)) { \ + goto bail; \ + } \ + Py_CLEAR(chunk); \ + } + +static PyObject * +scanstring_str(PyObject *pystr, Py_ssize_t end, char *encoding, int strict, Py_ssize_t *next_end_ptr) +{ + /* Read the JSON string from PyString pystr. + end is the index of the first character after the quote. + encoding is the encoding of pystr (must be an ASCII superset) + if strict is zero then literal control characters are allowed + *next_end_ptr is a return-by-reference index of the character + after the end quote + + Return value is a new PyString (if ASCII-only) or PyUnicode + */ + PyObject *rval; + Py_ssize_t len = PyString_GET_SIZE(pystr); + Py_ssize_t begin = end - 1; + Py_ssize_t next = begin; + int has_unicode = 0; + char *buf = PyString_AS_STRING(pystr); + PyObject *chunks = NULL; + PyObject *chunk = NULL; + + if (end < 0 || len <= end) { + PyErr_SetString(PyExc_ValueError, "end is out of bounds"); + goto bail; + } + while (1) { + /* Find the end of the string or the next escape */ + Py_UNICODE c = 0; + for (next = end; next < len; next++) { + c = (unsigned char)buf[next]; + if (c == '"' || c == '\\') { + break; + } + else if (strict && c <= 0x1f) { + raise_errmsg("Invalid control character at", pystr, next); + goto bail; + } + else if (c > 0x7f) { + has_unicode = 1; + } + } + if (!(c == '"' || c == '\\')) { + raise_errmsg("Unterminated string starting at", pystr, begin); + goto bail; + } + /* Pick up this chunk if it's not zero length */ + if (next != end) { + PyObject *strchunk; + APPEND_OLD_CHUNK + strchunk = PyString_FromStringAndSize(&buf[end], next - end); + if (strchunk == NULL) { + goto bail; + } + if (has_unicode) { + chunk = PyUnicode_FromEncodedObject(strchunk, encoding, NULL); + Py_DECREF(strchunk); + if (chunk == NULL) { + goto bail; + } + } + else { + chunk = strchunk; + } + } + next++; + if (c == '"') { + end = next; + break; + } + if (next == len) { + raise_errmsg("Unterminated string starting at", pystr, begin); + goto bail; + } + c = buf[next]; + if (c != 'u') { + /* Non-unicode backslash escapes */ + end = next + 1; + switch (c) { + case '"': break; + case '\\': break; + case '/': break; + case 'b': c = '\b'; break; + case 'f': c = '\f'; break; + case 'n': c = '\n'; break; + case 'r': c = '\r'; break; + case 't': c = '\t'; break; + default: c = 0; + } + if (c == 0) { + raise_errmsg("Invalid \\escape", pystr, end - 2); + goto bail; + } + } + else { + c = 0; + next++; + end = next + 4; + if (end >= len) { + raise_errmsg("Invalid \\uXXXX escape", pystr, next - 1); + goto bail; + } + /* Decode 4 hex digits */ + for (; next < end; next++) { + Py_UNICODE digit = buf[next]; + c <<= 4; + switch (digit) { + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + c |= (digit - '0'); break; + case 'a': case 'b': case 'c': case 'd': case 'e': + case 'f': + c |= (digit - 'a' + 10); break; + case 'A': case 'B': case 'C': case 'D': case 'E': + case 'F': + c |= (digit - 'A' + 10); break; + default: + raise_errmsg("Invalid \\uXXXX escape", pystr, end - 5); + goto bail; + } + } +#ifdef Py_UNICODE_WIDE + /* Surrogate pair */ + if ((c & 0xfc00) == 0xd800) { + Py_UNICODE c2 = 0; + if (end + 6 >= len) { + raise_errmsg("Unpaired high surrogate", pystr, end - 5); + goto bail; + } + if (buf[next++] != '\\' || buf[next++] != 'u') { + raise_errmsg("Unpaired high surrogate", pystr, end - 5); + goto bail; + } + end += 6; + /* Decode 4 hex digits */ + for (; next < end; next++) { + c2 <<= 4; + Py_UNICODE digit = buf[next]; + switch (digit) { + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + c2 |= (digit - '0'); break; + case 'a': case 'b': case 'c': case 'd': case 'e': + case 'f': + c2 |= (digit - 'a' + 10); break; + case 'A': case 'B': case 'C': case 'D': case 'E': + case 'F': + c2 |= (digit - 'A' + 10); break; + default: + raise_errmsg("Invalid \\uXXXX escape", pystr, end - 5); + goto bail; + } + } + if ((c2 & 0xfc00) != 0xdc00) { + raise_errmsg("Unpaired high surrogate", pystr, end - 5); + goto bail; + } + c = 0x10000 + (((c - 0xd800) << 10) | (c2 - 0xdc00)); + } + else if ((c & 0xfc00) == 0xdc00) { + raise_errmsg("Unpaired low surrogate", pystr, end - 5); + goto bail; + } +#endif + } + if (c > 0x7f) { + has_unicode = 1; + } + APPEND_OLD_CHUNK + if (has_unicode) { + chunk = PyUnicode_FromUnicode(&c, 1); + if (chunk == NULL) { + goto bail; + } + } + else { + char c_char = Py_CHARMASK(c); + chunk = PyString_FromStringAndSize(&c_char, 1); + if (chunk == NULL) { + goto bail; + } + } + } + + if (chunks == NULL) { + if (chunk != NULL) + rval = chunk; + else + rval = PyString_FromStringAndSize("", 0); + } + else { + APPEND_OLD_CHUNK + rval = join_list_string(chunks); + if (rval == NULL) { + goto bail; + } + Py_CLEAR(chunks); + } + + *next_end_ptr = end; + return rval; +bail: + *next_end_ptr = -1; + Py_XDECREF(chunk); + Py_XDECREF(chunks); + return NULL; +} + + +static PyObject * +scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next_end_ptr) +{ + /* Read the JSON string from PyUnicode pystr. + end is the index of the first character after the quote. + if strict is zero then literal control characters are allowed + *next_end_ptr is a return-by-reference index of the character + after the end quote + + Return value is a new PyUnicode + */ + PyObject *rval; + Py_ssize_t len = PyUnicode_GET_SIZE(pystr); + Py_ssize_t begin = end - 1; + Py_ssize_t next = begin; + const Py_UNICODE *buf = PyUnicode_AS_UNICODE(pystr); + PyObject *chunks = NULL; + PyObject *chunk = NULL; + + if (end < 0 || len <= end) { + PyErr_SetString(PyExc_ValueError, "end is out of bounds"); + goto bail; + } + while (1) { + /* Find the end of the string or the next escape */ + Py_UNICODE c = 0; + for (next = end; next < len; next++) { + c = buf[next]; + if (c == '"' || c == '\\') { + break; + } + else if (strict && c <= 0x1f) { + raise_errmsg("Invalid control character at", pystr, next); + goto bail; + } + } + if (!(c == '"' || c == '\\')) { + raise_errmsg("Unterminated string starting at", pystr, begin); + goto bail; + } + /* Pick up this chunk if it's not zero length */ + if (next != end) { + APPEND_OLD_CHUNK + chunk = PyUnicode_FromUnicode(&buf[end], next - end); + if (chunk == NULL) { + goto bail; + } + } + next++; + if (c == '"') { + end = next; + break; + } + if (next == len) { + raise_errmsg("Unterminated string starting at", pystr, begin); + goto bail; + } + c = buf[next]; + if (c != 'u') { + /* Non-unicode backslash escapes */ + end = next + 1; + switch (c) { + case '"': break; + case '\\': break; + case '/': break; + case 'b': c = '\b'; break; + case 'f': c = '\f'; break; + case 'n': c = '\n'; break; + case 'r': c = '\r'; break; + case 't': c = '\t'; break; + default: c = 0; + } + if (c == 0) { + raise_errmsg("Invalid \\escape", pystr, end - 2); + goto bail; + } + } + else { + c = 0; + next++; + end = next + 4; + if (end >= len) { + raise_errmsg("Invalid \\uXXXX escape", pystr, next - 1); + goto bail; + } + /* Decode 4 hex digits */ + for (; next < end; next++) { + Py_UNICODE digit = buf[next]; + c <<= 4; + switch (digit) { + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + c |= (digit - '0'); break; + case 'a': case 'b': case 'c': case 'd': case 'e': + case 'f': + c |= (digit - 'a' + 10); break; + case 'A': case 'B': case 'C': case 'D': case 'E': + case 'F': + c |= (digit - 'A' + 10); break; + default: + raise_errmsg("Invalid \\uXXXX escape", pystr, end - 5); + goto bail; + } + } +#ifdef Py_UNICODE_WIDE + /* Surrogate pair */ + if ((c & 0xfc00) == 0xd800) { + Py_UNICODE c2 = 0; + if (end + 6 >= len) { + raise_errmsg("Unpaired high surrogate", pystr, end - 5); + goto bail; + } + if (buf[next++] != '\\' || buf[next++] != 'u') { + raise_errmsg("Unpaired high surrogate", pystr, end - 5); + goto bail; + } + end += 6; + /* Decode 4 hex digits */ + for (; next < end; next++) { + c2 <<= 4; + Py_UNICODE digit = buf[next]; + switch (digit) { + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + c2 |= (digit - '0'); break; + case 'a': case 'b': case 'c': case 'd': case 'e': + case 'f': + c2 |= (digit - 'a' + 10); break; + case 'A': case 'B': case 'C': case 'D': case 'E': + case 'F': + c2 |= (digit - 'A' + 10); break; + default: + raise_errmsg("Invalid \\uXXXX escape", pystr, end - 5); + goto bail; + } + } + if ((c2 & 0xfc00) != 0xdc00) { + raise_errmsg("Unpaired high surrogate", pystr, end - 5); + goto bail; + } + c = 0x10000 + (((c - 0xd800) << 10) | (c2 - 0xdc00)); + } + else if ((c & 0xfc00) == 0xdc00) { + raise_errmsg("Unpaired low surrogate", pystr, end - 5); + goto bail; + } +#endif + } + APPEND_OLD_CHUNK + chunk = PyUnicode_FromUnicode(&c, 1); + if (chunk == NULL) { + goto bail; + } + } + + if (chunks == NULL) { + if (chunk != NULL) + rval = chunk; + else + rval = PyUnicode_FromUnicode(NULL, 0); + } + else { + APPEND_OLD_CHUNK + rval = join_list_unicode(chunks); + if (rval == NULL) { + goto bail; + } + Py_CLEAR(chunks); + } + *next_end_ptr = end; + return rval; +bail: + *next_end_ptr = -1; + Py_XDECREF(chunk); + Py_XDECREF(chunks); + return NULL; +} + +PyDoc_STRVAR(pydoc_scanstring, + "scanstring(basestring, end, encoding, strict=True) -> (str, end)\n" + "\n" + "Scan the string s for a JSON string. End is the index of the\n" + "character in s after the quote that started the JSON string.\n" + "Unescapes all valid JSON string escape sequences and raises ValueError\n" + "on attempt to decode an invalid string. If strict is False then literal\n" + "control characters are allowed in the string.\n" + "\n" + "Returns a tuple of the decoded string and the index of the character in s\n" + "after the end quote." +); + +static PyObject * +py_scanstring(PyObject* self UNUSED, PyObject *args) +{ + PyObject *pystr; + PyObject *rval; + Py_ssize_t end; + Py_ssize_t next_end = -1; + char *encoding = NULL; + int strict = 1; + if (!PyArg_ParseTuple(args, "OO&|zi:scanstring", &pystr, _convertPyInt_AsSsize_t, &end, &encoding, &strict)) { + return NULL; + } + if (encoding == NULL) { + encoding = DEFAULT_ENCODING; + } + if (PyString_Check(pystr)) { + rval = scanstring_str(pystr, end, encoding, strict, &next_end); + } + else if (PyUnicode_Check(pystr)) { + rval = scanstring_unicode(pystr, end, strict, &next_end); + } + else { + PyErr_Format(PyExc_TypeError, + "first argument must be a string, not %.80s", + Py_TYPE(pystr)->tp_name); + return NULL; + } + return _build_rval_index_tuple(rval, next_end); +} + +PyDoc_STRVAR(pydoc_encode_basestring_ascii, + "encode_basestring_ascii(basestring) -> str\n" + "\n" + "Return an ASCII-only JSON representation of a Python string" +); + +static PyObject * +py_encode_basestring_ascii(PyObject* self UNUSED, PyObject *pystr) +{ + /* Return an ASCII-only JSON representation of a Python string */ + /* METH_O */ + if (PyString_Check(pystr)) { + return ascii_escape_str(pystr); + } + else if (PyUnicode_Check(pystr)) { + return ascii_escape_unicode(pystr); + } + else { + PyErr_Format(PyExc_TypeError, + "first argument must be a string, not %.80s", + Py_TYPE(pystr)->tp_name); + return NULL; + } +} + +static void +scanner_dealloc(PyObject *self) +{ + /* Deallocate scanner object */ + scanner_clear(self); + Py_TYPE(self)->tp_free(self); +} + +static int +scanner_traverse(PyObject *self, visitproc visit, void *arg) +{ + PyScannerObject *s; + assert(PyScanner_Check(self)); + s = (PyScannerObject *)self; + Py_VISIT(s->encoding); + Py_VISIT(s->strict); + Py_VISIT(s->object_hook); + Py_VISIT(s->pairs_hook); + Py_VISIT(s->parse_float); + Py_VISIT(s->parse_int); + Py_VISIT(s->parse_constant); + Py_VISIT(s->memo); + return 0; +} + +static int +scanner_clear(PyObject *self) +{ + PyScannerObject *s; + assert(PyScanner_Check(self)); + s = (PyScannerObject *)self; + Py_CLEAR(s->encoding); + Py_CLEAR(s->strict); + Py_CLEAR(s->object_hook); + Py_CLEAR(s->pairs_hook); + Py_CLEAR(s->parse_float); + Py_CLEAR(s->parse_int); + Py_CLEAR(s->parse_constant); + Py_CLEAR(s->memo); + return 0; +} + +static PyObject * +_parse_object_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) { + /* Read a JSON object from PyString pystr. + idx is the index of the first character after the opening curly brace. + *next_idx_ptr is a return-by-reference index to the first character after + the closing curly brace. + + Returns a new PyObject (usually a dict, but object_hook or + object_pairs_hook can change that) + */ + char *str = PyString_AS_STRING(pystr); + Py_ssize_t end_idx = PyString_GET_SIZE(pystr) - 1; + PyObject *rval = NULL; + PyObject *pairs = NULL; + PyObject *item; + PyObject *key = NULL; + PyObject *val = NULL; + char *encoding = PyString_AS_STRING(s->encoding); + int strict = PyObject_IsTrue(s->strict); + int has_pairs_hook = (s->pairs_hook != Py_None); + Py_ssize_t next_idx; + if (has_pairs_hook) { + pairs = PyList_New(0); + if (pairs == NULL) + return NULL; + } + else { + rval = PyDict_New(); + if (rval == NULL) + return NULL; + } + + /* skip whitespace after { */ + while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + + /* only loop if the object is non-empty */ + if (idx <= end_idx && str[idx] != '}') { + while (idx <= end_idx) { + PyObject *memokey; + + /* read key */ + if (str[idx] != '"') { + raise_errmsg("Expecting property name", pystr, idx); + goto bail; + } + key = scanstring_str(pystr, idx + 1, encoding, strict, &next_idx); + if (key == NULL) + goto bail; + memokey = PyDict_GetItem(s->memo, key); + if (memokey != NULL) { + Py_INCREF(memokey); + Py_DECREF(key); + key = memokey; + } + else { + if (PyDict_SetItem(s->memo, key, key) < 0) + goto bail; + } + idx = next_idx; + + /* skip whitespace between key and : delimiter, read :, skip whitespace */ + while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + if (idx > end_idx || str[idx] != ':') { + raise_errmsg("Expecting : delimiter", pystr, idx); + goto bail; + } + idx++; + while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + + /* read any JSON data type */ + val = scan_once_str(s, pystr, idx, &next_idx); + if (val == NULL) + goto bail; + + if (has_pairs_hook) { + item = PyTuple_Pack(2, key, val); + if (item == NULL) + goto bail; + Py_CLEAR(key); + Py_CLEAR(val); + if (PyList_Append(pairs, item) == -1) { + Py_DECREF(item); + goto bail; + } + Py_DECREF(item); + } + else { + if (PyDict_SetItem(rval, key, val) < 0) + goto bail; + Py_CLEAR(key); + Py_CLEAR(val); + } + idx = next_idx; + + /* skip whitespace before } or , */ + while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + + /* bail if the object is closed or we didn't get the , delimiter */ + if (idx > end_idx) break; + if (str[idx] == '}') { + break; + } + else if (str[idx] != ',') { + raise_errmsg("Expecting , delimiter", pystr, idx); + goto bail; + } + idx++; + + /* skip whitespace after , delimiter */ + while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + } + } + /* verify that idx < end_idx, str[idx] should be '}' */ + if (idx > end_idx || str[idx] != '}') { + raise_errmsg("Expecting object", pystr, end_idx); + goto bail; + } + + /* if pairs_hook is not None: rval = object_pairs_hook(pairs) */ + if (s->pairs_hook != Py_None) { + val = PyObject_CallFunctionObjArgs(s->pairs_hook, pairs, NULL); + if (val == NULL) + goto bail; + Py_DECREF(pairs); + *next_idx_ptr = idx + 1; + return val; + } + + /* if object_hook is not None: rval = object_hook(rval) */ + if (s->object_hook != Py_None) { + val = PyObject_CallFunctionObjArgs(s->object_hook, rval, NULL); + if (val == NULL) + goto bail; + Py_DECREF(rval); + rval = val; + val = NULL; + } + *next_idx_ptr = idx + 1; + return rval; +bail: + Py_XDECREF(rval); + Py_XDECREF(key); + Py_XDECREF(val); + Py_XDECREF(pairs); + return NULL; +} + +static PyObject * +_parse_object_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) { + /* Read a JSON object from PyUnicode pystr. + idx is the index of the first character after the opening curly brace. + *next_idx_ptr is a return-by-reference index to the first character after + the closing curly brace. + + Returns a new PyObject (usually a dict, but object_hook can change that) + */ + Py_UNICODE *str = PyUnicode_AS_UNICODE(pystr); + Py_ssize_t end_idx = PyUnicode_GET_SIZE(pystr) - 1; + PyObject *rval = NULL; + PyObject *pairs = NULL; + PyObject *item; + PyObject *key = NULL; + PyObject *val = NULL; + int strict = PyObject_IsTrue(s->strict); + int has_pairs_hook = (s->pairs_hook != Py_None); + Py_ssize_t next_idx; + + if (has_pairs_hook) { + pairs = PyList_New(0); + if (pairs == NULL) + return NULL; + } + else { + rval = PyDict_New(); + if (rval == NULL) + return NULL; + } + + /* skip whitespace after { */ + while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + + /* only loop if the object is non-empty */ + if (idx <= end_idx && str[idx] != '}') { + while (idx <= end_idx) { + PyObject *memokey; + + /* read key */ + if (str[idx] != '"') { + raise_errmsg("Expecting property name", pystr, idx); + goto bail; + } + key = scanstring_unicode(pystr, idx + 1, strict, &next_idx); + if (key == NULL) + goto bail; + memokey = PyDict_GetItem(s->memo, key); + if (memokey != NULL) { + Py_INCREF(memokey); + Py_DECREF(key); + key = memokey; + } + else { + if (PyDict_SetItem(s->memo, key, key) < 0) + goto bail; + } + idx = next_idx; + + /* skip whitespace between key and : delimiter, read :, skip whitespace */ + while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + if (idx > end_idx || str[idx] != ':') { + raise_errmsg("Expecting : delimiter", pystr, idx); + goto bail; + } + idx++; + while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + + /* read any JSON term */ + val = scan_once_unicode(s, pystr, idx, &next_idx); + if (val == NULL) + goto bail; + + if (has_pairs_hook) { + item = PyTuple_Pack(2, key, val); + if (item == NULL) + goto bail; + Py_CLEAR(key); + Py_CLEAR(val); + if (PyList_Append(pairs, item) == -1) { + Py_DECREF(item); + goto bail; + } + Py_DECREF(item); + } + else { + if (PyDict_SetItem(rval, key, val) < 0) + goto bail; + Py_CLEAR(key); + Py_CLEAR(val); + } + idx = next_idx; + + /* skip whitespace before } or , */ + while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + + /* bail if the object is closed or we didn't get the , delimiter */ + if (idx > end_idx) break; + if (str[idx] == '}') { + break; + } + else if (str[idx] != ',') { + raise_errmsg("Expecting , delimiter", pystr, idx); + goto bail; + } + idx++; + + /* skip whitespace after , delimiter */ + while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + } + } + + /* verify that idx < end_idx, str[idx] should be '}' */ + if (idx > end_idx || str[idx] != '}') { + raise_errmsg("Expecting object", pystr, end_idx); + goto bail; + } + + /* if pairs_hook is not None: rval = object_pairs_hook(pairs) */ + if (s->pairs_hook != Py_None) { + val = PyObject_CallFunctionObjArgs(s->pairs_hook, pairs, NULL); + if (val == NULL) + goto bail; + Py_DECREF(pairs); + *next_idx_ptr = idx + 1; + return val; + } + + /* if object_hook is not None: rval = object_hook(rval) */ + if (s->object_hook != Py_None) { + val = PyObject_CallFunctionObjArgs(s->object_hook, rval, NULL); + if (val == NULL) + goto bail; + Py_DECREF(rval); + rval = val; + val = NULL; + } + *next_idx_ptr = idx + 1; + return rval; +bail: + Py_XDECREF(rval); + Py_XDECREF(key); + Py_XDECREF(val); + Py_XDECREF(pairs); + return NULL; +} + +static PyObject * +_parse_array_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) { + /* Read a JSON array from PyString pystr. + idx is the index of the first character after the opening brace. + *next_idx_ptr is a return-by-reference index to the first character after + the closing brace. + + Returns a new PyList + */ + char *str = PyString_AS_STRING(pystr); + Py_ssize_t end_idx = PyString_GET_SIZE(pystr) - 1; + PyObject *val = NULL; + PyObject *rval = PyList_New(0); + Py_ssize_t next_idx; + if (rval == NULL) + return NULL; + + /* skip whitespace after [ */ + while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + + /* only loop if the array is non-empty */ + if (idx <= end_idx && str[idx] != ']') { + while (idx <= end_idx) { + + /* read any JSON term and de-tuplefy the (rval, idx) */ + val = scan_once_str(s, pystr, idx, &next_idx); + if (val == NULL) { + if (PyErr_ExceptionMatches(PyExc_StopIteration)) { + PyErr_Clear(); + raise_errmsg("Expecting object", pystr, idx); + } + goto bail; + } + + if (PyList_Append(rval, val) == -1) + goto bail; + + Py_CLEAR(val); + idx = next_idx; + + /* skip whitespace between term and , */ + while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + + /* bail if the array is closed or we didn't get the , delimiter */ + if (idx > end_idx) break; + if (str[idx] == ']') { + break; + } + else if (str[idx] != ',') { + raise_errmsg("Expecting , delimiter", pystr, idx); + goto bail; + } + idx++; + + /* skip whitespace after , */ + while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + } + } + + /* verify that idx < end_idx, str[idx] should be ']' */ + if (idx > end_idx || str[idx] != ']') { + raise_errmsg("Expecting object", pystr, end_idx); + goto bail; + } + *next_idx_ptr = idx + 1; + return rval; +bail: + Py_XDECREF(val); + Py_DECREF(rval); + return NULL; +} + +static PyObject * +_parse_array_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) { + /* Read a JSON array from PyString pystr. + idx is the index of the first character after the opening brace. + *next_idx_ptr is a return-by-reference index to the first character after + the closing brace. + + Returns a new PyList + */ + Py_UNICODE *str = PyUnicode_AS_UNICODE(pystr); + Py_ssize_t end_idx = PyUnicode_GET_SIZE(pystr) - 1; + PyObject *val = NULL; + PyObject *rval = PyList_New(0); + Py_ssize_t next_idx; + if (rval == NULL) + return NULL; + + /* skip whitespace after [ */ + while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + + /* only loop if the array is non-empty */ + if (idx <= end_idx && str[idx] != ']') { + while (idx <= end_idx) { + + /* read any JSON term */ + val = scan_once_unicode(s, pystr, idx, &next_idx); + if (val == NULL) { + if (PyErr_ExceptionMatches(PyExc_StopIteration)) { + PyErr_Clear(); + raise_errmsg("Expecting object", pystr, idx); + } + goto bail; + } + + if (PyList_Append(rval, val) == -1) + goto bail; + + Py_CLEAR(val); + idx = next_idx; + + /* skip whitespace between term and , */ + while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + + /* bail if the array is closed or we didn't get the , delimiter */ + if (idx > end_idx) break; + if (str[idx] == ']') { + break; + } + else if (str[idx] != ',') { + raise_errmsg("Expecting , delimiter", pystr, idx); + goto bail; + } + idx++; + + /* skip whitespace after , */ + while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + } + } + + /* verify that idx < end_idx, str[idx] should be ']' */ + if (idx > end_idx || str[idx] != ']') { + raise_errmsg("Expecting object", pystr, end_idx); + goto bail; + } + *next_idx_ptr = idx + 1; + return rval; +bail: + Py_XDECREF(val); + Py_DECREF(rval); + return NULL; +} + +static PyObject * +_parse_constant(PyScannerObject *s, char *constant, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) { + /* Read a JSON constant from PyString pystr. + constant is the constant string that was found + ("NaN", "Infinity", "-Infinity"). + idx is the index of the first character of the constant + *next_idx_ptr is a return-by-reference index to the first character after + the constant. + + Returns the result of parse_constant + */ + PyObject *cstr; + PyObject *rval; + /* constant is "NaN", "Infinity", or "-Infinity" */ + cstr = PyString_InternFromString(constant); + if (cstr == NULL) + return NULL; + + /* rval = parse_constant(constant) */ + rval = PyObject_CallFunctionObjArgs(s->parse_constant, cstr, NULL); + idx += PyString_GET_SIZE(cstr); + Py_DECREF(cstr); + *next_idx_ptr = idx; + return rval; +} + +static PyObject * +_match_number_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_ssize_t *next_idx_ptr) { + /* Read a JSON number from PyString pystr. + idx is the index of the first character of the number + *next_idx_ptr is a return-by-reference index to the first character after + the number. + + Returns a new PyObject representation of that number: + PyInt, PyLong, or PyFloat. + May return other types if parse_int or parse_float are set + */ + char *str = PyString_AS_STRING(pystr); + Py_ssize_t end_idx = PyString_GET_SIZE(pystr) - 1; + Py_ssize_t idx = start; + int is_float = 0; + PyObject *rval; + PyObject *numstr; + + /* read a sign if it's there, make sure it's not the end of the string */ + if (str[idx] == '-') { + idx++; + if (idx > end_idx) { + PyErr_SetNone(PyExc_StopIteration); + return NULL; + } + } + + /* read as many integer digits as we find as long as it doesn't start with 0 */ + if (str[idx] >= '1' && str[idx] <= '9') { + idx++; + while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; + } + /* if it starts with 0 we only expect one integer digit */ + else if (str[idx] == '0') { + idx++; + } + /* no integer digits, error */ + else { + PyErr_SetNone(PyExc_StopIteration); + return NULL; + } + + /* if the next char is '.' followed by a digit then read all float digits */ + if (idx < end_idx && str[idx] == '.' && str[idx + 1] >= '0' && str[idx + 1] <= '9') { + is_float = 1; + idx += 2; + while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; + } + + /* if the next char is 'e' or 'E' then maybe read the exponent (or backtrack) */ + if (idx < end_idx && (str[idx] == 'e' || str[idx] == 'E')) { + + /* save the index of the 'e' or 'E' just in case we need to backtrack */ + Py_ssize_t e_start = idx; + idx++; + + /* read an exponent sign if present */ + if (idx < end_idx && (str[idx] == '-' || str[idx] == '+')) idx++; + + /* read all digits */ + while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; + + /* if we got a digit, then parse as float. if not, backtrack */ + if (str[idx - 1] >= '0' && str[idx - 1] <= '9') { + is_float = 1; + } + else { + idx = e_start; + } + } + + /* copy the section we determined to be a number */ + numstr = PyString_FromStringAndSize(&str[start], idx - start); + if (numstr == NULL) + return NULL; + if (is_float) { + /* parse as a float using a fast path if available, otherwise call user defined method */ + if (s->parse_float != (PyObject *)&PyFloat_Type) { + rval = PyObject_CallFunctionObjArgs(s->parse_float, numstr, NULL); + } + else { + /* rval = PyFloat_FromDouble(PyOS_ascii_atof(PyString_AS_STRING(numstr))); */ + double d = PyOS_string_to_double(PyString_AS_STRING(numstr), + NULL, NULL); + if (d == -1.0 && PyErr_Occurred()) + return NULL; + rval = PyFloat_FromDouble(d); + } + } + else { + /* parse as an int using a fast path if available, otherwise call user defined method */ + if (s->parse_int != (PyObject *)&PyInt_Type) { + rval = PyObject_CallFunctionObjArgs(s->parse_int, numstr, NULL); + } + else { + rval = PyInt_FromString(PyString_AS_STRING(numstr), NULL, 10); + } + } + Py_DECREF(numstr); + *next_idx_ptr = idx; + return rval; +} + +static PyObject * +_match_number_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_ssize_t *next_idx_ptr) { + /* Read a JSON number from PyUnicode pystr. + idx is the index of the first character of the number + *next_idx_ptr is a return-by-reference index to the first character after + the number. + + Returns a new PyObject representation of that number: + PyInt, PyLong, or PyFloat. + May return other types if parse_int or parse_float are set + */ + Py_UNICODE *str = PyUnicode_AS_UNICODE(pystr); + Py_ssize_t end_idx = PyUnicode_GET_SIZE(pystr) - 1; + Py_ssize_t idx = start; + int is_float = 0; + PyObject *rval; + PyObject *numstr; + + /* read a sign if it's there, make sure it's not the end of the string */ + if (str[idx] == '-') { + idx++; + if (idx > end_idx) { + PyErr_SetNone(PyExc_StopIteration); + return NULL; + } + } + + /* read as many integer digits as we find as long as it doesn't start with 0 */ + if (str[idx] >= '1' && str[idx] <= '9') { + idx++; + while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; + } + /* if it starts with 0 we only expect one integer digit */ + else if (str[idx] == '0') { + idx++; + } + /* no integer digits, error */ + else { + PyErr_SetNone(PyExc_StopIteration); + return NULL; + } + + /* if the next char is '.' followed by a digit then read all float digits */ + if (idx < end_idx && str[idx] == '.' && str[idx + 1] >= '0' && str[idx + 1] <= '9') { + is_float = 1; + idx += 2; + while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; + } + + /* if the next char is 'e' or 'E' then maybe read the exponent (or backtrack) */ + if (idx < end_idx && (str[idx] == 'e' || str[idx] == 'E')) { + Py_ssize_t e_start = idx; + idx++; + + /* read an exponent sign if present */ + if (idx < end_idx && (str[idx] == '-' || str[idx] == '+')) idx++; + + /* read all digits */ + while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; + + /* if we got a digit, then parse as float. if not, backtrack */ + if (str[idx - 1] >= '0' && str[idx - 1] <= '9') { + is_float = 1; + } + else { + idx = e_start; + } + } + + /* copy the section we determined to be a number */ + numstr = PyUnicode_FromUnicode(&str[start], idx - start); + if (numstr == NULL) + return NULL; + if (is_float) { + /* parse as a float using a fast path if available, otherwise call user defined method */ + if (s->parse_float != (PyObject *)&PyFloat_Type) { + rval = PyObject_CallFunctionObjArgs(s->parse_float, numstr, NULL); + } + else { + rval = PyFloat_FromString(numstr, NULL); + } + } + else { + /* no fast path for unicode -> int, just call */ + rval = PyObject_CallFunctionObjArgs(s->parse_int, numstr, NULL); + } + Py_DECREF(numstr); + *next_idx_ptr = idx; + return rval; +} + +static PyObject * +scan_once_str(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) +{ + /* Read one JSON term (of any kind) from PyString pystr. + idx is the index of the first character of the term + *next_idx_ptr is a return-by-reference index to the first character after + the number. + + Returns a new PyObject representation of the term. + */ + char *str = PyString_AS_STRING(pystr); + Py_ssize_t length = PyString_GET_SIZE(pystr); + PyObject *rval = NULL; + int fallthrough = 0; + if (idx >= length) { + PyErr_SetNone(PyExc_StopIteration); + return NULL; + } + if (Py_EnterRecursiveCall(" while decoding a JSON document")) + return NULL; + switch (str[idx]) { + case '"': + /* string */ + rval = scanstring_str(pystr, idx + 1, + PyString_AS_STRING(s->encoding), + PyObject_IsTrue(s->strict), + next_idx_ptr); + break; + case '{': + /* object */ + rval = _parse_object_str(s, pystr, idx + 1, next_idx_ptr); + break; + case '[': + /* array */ + rval = _parse_array_str(s, pystr, idx + 1, next_idx_ptr); + break; + case 'n': + /* null */ + if ((idx + 3 < length) && str[idx + 1] == 'u' && str[idx + 2] == 'l' && str[idx + 3] == 'l') { + Py_INCREF(Py_None); + *next_idx_ptr = idx + 4; + rval = Py_None; + } + else + fallthrough = 1; + break; + case 't': + /* true */ + if ((idx + 3 < length) && str[idx + 1] == 'r' && str[idx + 2] == 'u' && str[idx + 3] == 'e') { + Py_INCREF(Py_True); + *next_idx_ptr = idx + 4; + rval = Py_True; + } + else + fallthrough = 1; + break; + case 'f': + /* false */ + if ((idx + 4 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'l' && str[idx + 3] == 's' && str[idx + 4] == 'e') { + Py_INCREF(Py_False); + *next_idx_ptr = idx + 5; + rval = Py_False; + } + else + fallthrough = 1; + break; + case 'N': + /* NaN */ + if ((idx + 2 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'N') { + rval = _parse_constant(s, "NaN", idx, next_idx_ptr); + } + else + fallthrough = 1; + break; + case 'I': + /* Infinity */ + if ((idx + 7 < length) && str[idx + 1] == 'n' && str[idx + 2] == 'f' && str[idx + 3] == 'i' && str[idx + 4] == 'n' && str[idx + 5] == 'i' && str[idx + 6] == 't' && str[idx + 7] == 'y') { + rval = _parse_constant(s, "Infinity", idx, next_idx_ptr); + } + else + fallthrough = 1; + break; + case '-': + /* -Infinity */ + if ((idx + 8 < length) && str[idx + 1] == 'I' && str[idx + 2] == 'n' && str[idx + 3] == 'f' && str[idx + 4] == 'i' && str[idx + 5] == 'n' && str[idx + 6] == 'i' && str[idx + 7] == 't' && str[idx + 8] == 'y') { + rval = _parse_constant(s, "-Infinity", idx, next_idx_ptr); + } + else + fallthrough = 1; + break; + default: + fallthrough = 1; + } + /* Didn't find a string, object, array, or named constant. Look for a number. */ + if (fallthrough) + rval = _match_number_str(s, pystr, idx, next_idx_ptr); + Py_LeaveRecursiveCall(); + return rval; +} + +static PyObject * +scan_once_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr) +{ + /* Read one JSON term (of any kind) from PyUnicode pystr. + idx is the index of the first character of the term + *next_idx_ptr is a return-by-reference index to the first character after + the number. + + Returns a new PyObject representation of the term. + */ + Py_UNICODE *str = PyUnicode_AS_UNICODE(pystr); + Py_ssize_t length = PyUnicode_GET_SIZE(pystr); + PyObject *rval = NULL; + int fallthrough = 0; + if (idx >= length) { + PyErr_SetNone(PyExc_StopIteration); + return NULL; + } + if (Py_EnterRecursiveCall(" while decoding a JSON document")) + return NULL; + switch (str[idx]) { + case '"': + /* string */ + rval = scanstring_unicode(pystr, idx + 1, + PyObject_IsTrue(s->strict), + next_idx_ptr); + break; + case '{': + /* object */ + rval = _parse_object_unicode(s, pystr, idx + 1, next_idx_ptr); + break; + case '[': + /* array */ + rval = _parse_array_unicode(s, pystr, idx + 1, next_idx_ptr); + break; + case 'n': + /* null */ + if ((idx + 3 < length) && str[idx + 1] == 'u' && str[idx + 2] == 'l' && str[idx + 3] == 'l') { + Py_INCREF(Py_None); + *next_idx_ptr = idx + 4; + rval = Py_None; + } + else + fallthrough = 1; + break; + case 't': + /* true */ + if ((idx + 3 < length) && str[idx + 1] == 'r' && str[idx + 2] == 'u' && str[idx + 3] == 'e') { + Py_INCREF(Py_True); + *next_idx_ptr = idx + 4; + rval = Py_True; + } + else + fallthrough = 1; + break; + case 'f': + /* false */ + if ((idx + 4 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'l' && str[idx + 3] == 's' && str[idx + 4] == 'e') { + Py_INCREF(Py_False); + *next_idx_ptr = idx + 5; + rval = Py_False; + } + else + fallthrough = 1; + break; + case 'N': + /* NaN */ + if ((idx + 2 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'N') { + rval = _parse_constant(s, "NaN", idx, next_idx_ptr); + } + else + fallthrough = 1; + break; + case 'I': + /* Infinity */ + if ((idx + 7 < length) && str[idx + 1] == 'n' && str[idx + 2] == 'f' && str[idx + 3] == 'i' && str[idx + 4] == 'n' && str[idx + 5] == 'i' && str[idx + 6] == 't' && str[idx + 7] == 'y') { + rval = _parse_constant(s, "Infinity", idx, next_idx_ptr); + } + else + fallthrough = 1; + break; + case '-': + /* -Infinity */ + if ((idx + 8 < length) && str[idx + 1] == 'I' && str[idx + 2] == 'n' && str[idx + 3] == 'f' && str[idx + 4] == 'i' && str[idx + 5] == 'n' && str[idx + 6] == 'i' && str[idx + 7] == 't' && str[idx + 8] == 'y') { + rval = _parse_constant(s, "-Infinity", idx, next_idx_ptr); + } + else + fallthrough = 1; + break; + default: + fallthrough = 1; + } + /* Didn't find a string, object, array, or named constant. Look for a number. */ + if (fallthrough) + rval = _match_number_unicode(s, pystr, idx, next_idx_ptr); + Py_LeaveRecursiveCall(); + return rval; +} + +static PyObject * +scanner_call(PyObject *self, PyObject *args, PyObject *kwds) +{ + /* Python callable interface to scan_once_{str,unicode} */ + PyObject *pystr; + PyObject *rval; + Py_ssize_t idx; + Py_ssize_t next_idx = -1; + static char *kwlist[] = {"string", "idx", NULL}; + PyScannerObject *s; + assert(PyScanner_Check(self)); + s = (PyScannerObject *)self; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO&:scan_once", kwlist, &pystr, _convertPyInt_AsSsize_t, &idx)) + return NULL; + + if (PyString_Check(pystr)) { + rval = scan_once_str(s, pystr, idx, &next_idx); + } + else if (PyUnicode_Check(pystr)) { + rval = scan_once_unicode(s, pystr, idx, &next_idx); + } + else { + PyErr_Format(PyExc_TypeError, + "first argument must be a string, not %.80s", + Py_TYPE(pystr)->tp_name); + return NULL; + } + PyDict_Clear(s->memo); + return _build_rval_index_tuple(rval, next_idx); +} + +static PyObject * +scanner_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyScannerObject *s; + s = (PyScannerObject *)type->tp_alloc(type, 0); + if (s != NULL) { + s->encoding = NULL; + s->strict = NULL; + s->object_hook = NULL; + s->pairs_hook = NULL; + s->parse_float = NULL; + s->parse_int = NULL; + s->parse_constant = NULL; + } + return (PyObject *)s; +} + +static int +scanner_init(PyObject *self, PyObject *args, PyObject *kwds) +{ + /* Initialize Scanner object */ + PyObject *ctx; + static char *kwlist[] = {"context", NULL}; + PyScannerObject *s; + + assert(PyScanner_Check(self)); + s = (PyScannerObject *)self; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O:make_scanner", kwlist, &ctx)) + return -1; + + if (s->memo == NULL) { + s->memo = PyDict_New(); + if (s->memo == NULL) + goto bail; + } + + /* PyString_AS_STRING is used on encoding */ + s->encoding = PyObject_GetAttrString(ctx, "encoding"); + if (s->encoding == NULL) + goto bail; + if (s->encoding == Py_None) { + Py_DECREF(Py_None); + s->encoding = PyString_InternFromString(DEFAULT_ENCODING); + } + else if (PyUnicode_Check(s->encoding)) { + PyObject *tmp = PyUnicode_AsEncodedString(s->encoding, NULL, NULL); + Py_DECREF(s->encoding); + s->encoding = tmp; + } + if (s->encoding == NULL || !PyString_Check(s->encoding)) + goto bail; + + /* All of these will fail "gracefully" so we don't need to verify them */ + s->strict = PyObject_GetAttrString(ctx, "strict"); + if (s->strict == NULL) + goto bail; + s->object_hook = PyObject_GetAttrString(ctx, "object_hook"); + if (s->object_hook == NULL) + goto bail; + s->pairs_hook = PyObject_GetAttrString(ctx, "object_pairs_hook"); + if (s->pairs_hook == NULL) + goto bail; + s->parse_float = PyObject_GetAttrString(ctx, "parse_float"); + if (s->parse_float == NULL) + goto bail; + s->parse_int = PyObject_GetAttrString(ctx, "parse_int"); + if (s->parse_int == NULL) + goto bail; + s->parse_constant = PyObject_GetAttrString(ctx, "parse_constant"); + if (s->parse_constant == NULL) + goto bail; + + return 0; + +bail: + Py_CLEAR(s->encoding); + Py_CLEAR(s->strict); + Py_CLEAR(s->object_hook); + Py_CLEAR(s->pairs_hook); + Py_CLEAR(s->parse_float); + Py_CLEAR(s->parse_int); + Py_CLEAR(s->parse_constant); + return -1; +} + +PyDoc_STRVAR(scanner_doc, "JSON scanner object"); + +static +PyTypeObject PyScannerType = { + PyObject_HEAD_INIT(NULL) + 0, /* tp_internal */ + "simplejson._speedups.Scanner", /* tp_name */ + sizeof(PyScannerObject), /* tp_basicsize */ + 0, /* tp_itemsize */ + scanner_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + scanner_call, /* tp_call */ + 0, /* tp_str */ + 0,/* PyObject_GenericGetAttr, */ /* tp_getattro */ + 0,/* PyObject_GenericSetAttr, */ /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ + scanner_doc, /* tp_doc */ + scanner_traverse, /* tp_traverse */ + scanner_clear, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + scanner_members, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + scanner_init, /* tp_init */ + 0,/* PyType_GenericAlloc, */ /* tp_alloc */ + scanner_new, /* tp_new */ + 0,/* PyObject_GC_Del, */ /* tp_free */ +}; + +static PyObject * +encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyEncoderObject *s; + s = (PyEncoderObject *)type->tp_alloc(type, 0); + if (s != NULL) { + s->markers = NULL; + s->defaultfn = NULL; + s->encoder = NULL; + s->indent = NULL; + s->key_separator = NULL; + s->item_separator = NULL; + s->sort_keys = NULL; + s->skipkeys = NULL; + s->key_memo = NULL; + } + return (PyObject *)s; +} + +static int +encoder_init(PyObject *self, PyObject *args, PyObject *kwds) +{ + /* initialize Encoder object */ + static char *kwlist[] = {"markers", "default", "encoder", "indent", "key_separator", "item_separator", "sort_keys", "skipkeys", "allow_nan", "key_memo", "use_decimal", NULL}; + + PyEncoderObject *s; + PyObject *markers, *defaultfn, *encoder, *indent, *key_separator; + PyObject *item_separator, *sort_keys, *skipkeys, *allow_nan, *key_memo, *use_decimal; + + assert(PyEncoder_Check(self)); + s = (PyEncoderObject *)self; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOOOOOOOOOO:make_encoder", kwlist, + &markers, &defaultfn, &encoder, &indent, &key_separator, &item_separator, + &sort_keys, &skipkeys, &allow_nan, &key_memo, &use_decimal)) + return -1; + + s->markers = markers; + s->defaultfn = defaultfn; + s->encoder = encoder; + s->indent = indent; + s->key_separator = key_separator; + s->item_separator = item_separator; + s->sort_keys = sort_keys; + s->skipkeys = skipkeys; + s->key_memo = key_memo; + s->fast_encode = (PyCFunction_Check(s->encoder) && PyCFunction_GetFunction(s->encoder) == (PyCFunction)py_encode_basestring_ascii); + s->allow_nan = PyObject_IsTrue(allow_nan); + s->use_decimal = PyObject_IsTrue(use_decimal); + + Py_INCREF(s->markers); + Py_INCREF(s->defaultfn); + Py_INCREF(s->encoder); + Py_INCREF(s->indent); + Py_INCREF(s->key_separator); + Py_INCREF(s->item_separator); + Py_INCREF(s->sort_keys); + Py_INCREF(s->skipkeys); + Py_INCREF(s->key_memo); + return 0; +} + +static PyObject * +encoder_call(PyObject *self, PyObject *args, PyObject *kwds) +{ + /* Python callable interface to encode_listencode_obj */ + static char *kwlist[] = {"obj", "_current_indent_level", NULL}; + PyObject *obj; + PyObject *rval; + Py_ssize_t indent_level; + PyEncoderObject *s; + assert(PyEncoder_Check(self)); + s = (PyEncoderObject *)self; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO&:_iterencode", kwlist, + &obj, _convertPyInt_AsSsize_t, &indent_level)) + return NULL; + rval = PyList_New(0); + if (rval == NULL) + return NULL; + if (encoder_listencode_obj(s, rval, obj, indent_level)) { + Py_DECREF(rval); + return NULL; + } + return rval; +} + +static PyObject * +_encoded_const(PyObject *obj) +{ + /* Return the JSON string representation of None, True, False */ + if (obj == Py_None) { + static PyObject *s_null = NULL; + if (s_null == NULL) { + s_null = PyString_InternFromString("null"); + } + Py_INCREF(s_null); + return s_null; + } + else if (obj == Py_True) { + static PyObject *s_true = NULL; + if (s_true == NULL) { + s_true = PyString_InternFromString("true"); + } + Py_INCREF(s_true); + return s_true; + } + else if (obj == Py_False) { + static PyObject *s_false = NULL; + if (s_false == NULL) { + s_false = PyString_InternFromString("false"); + } + Py_INCREF(s_false); + return s_false; + } + else { + PyErr_SetString(PyExc_ValueError, "not a const"); + return NULL; + } +} + +static PyObject * +encoder_encode_float(PyEncoderObject *s, PyObject *obj) +{ + /* Return the JSON representation of a PyFloat */ + double i = PyFloat_AS_DOUBLE(obj); + if (!Py_IS_FINITE(i)) { + if (!s->allow_nan) { + PyErr_SetString(PyExc_ValueError, "Out of range float values are not JSON compliant"); + return NULL; + } + if (i > 0) { + return PyString_FromString("Infinity"); + } + else if (i < 0) { + return PyString_FromString("-Infinity"); + } + else { + return PyString_FromString("NaN"); + } + } + /* Use a better float format here? */ + return PyObject_Repr(obj); +} + +static PyObject * +encoder_encode_string(PyEncoderObject *s, PyObject *obj) +{ + /* Return the JSON representation of a string */ + if (s->fast_encode) + return py_encode_basestring_ascii(NULL, obj); + else + return PyObject_CallFunctionObjArgs(s->encoder, obj, NULL); +} + +static int +_steal_list_append(PyObject *lst, PyObject *stolen) +{ + /* Append stolen and then decrement its reference count */ + int rval = PyList_Append(lst, stolen); + Py_DECREF(stolen); + return rval; +} + +static int +encoder_listencode_obj(PyEncoderObject *s, PyObject *rval, PyObject *obj, Py_ssize_t indent_level) +{ + /* Encode Python object obj to a JSON term, rval is a PyList */ + int rv = -1; + if (Py_EnterRecursiveCall(" while encoding a JSON document")) + return rv; + do { + if (obj == Py_None || obj == Py_True || obj == Py_False) { + PyObject *cstr = _encoded_const(obj); + if (cstr != NULL) + rv = _steal_list_append(rval, cstr); + } + else if (PyString_Check(obj) || PyUnicode_Check(obj)) + { + PyObject *encoded = encoder_encode_string(s, obj); + if (encoded != NULL) + rv = _steal_list_append(rval, encoded); + } + else if (PyInt_Check(obj) || PyLong_Check(obj)) { + PyObject *encoded = PyObject_Str(obj); + if (encoded != NULL) + rv = _steal_list_append(rval, encoded); + } + else if (PyFloat_Check(obj)) { + PyObject *encoded = encoder_encode_float(s, obj); + if (encoded != NULL) + rv = _steal_list_append(rval, encoded); + } + else if (PyList_Check(obj) || PyTuple_Check(obj)) { + rv = encoder_listencode_list(s, rval, obj, indent_level); + } + else if (PyDict_Check(obj)) { + rv = encoder_listencode_dict(s, rval, obj, indent_level); + } + else if (s->use_decimal && Decimal_Check(obj)) { + PyObject *encoded = PyObject_Str(obj); + if (encoded != NULL) + rv = _steal_list_append(rval, encoded); + } + else { + PyObject *ident = NULL; + PyObject *newobj; + if (s->markers != Py_None) { + int has_key; + ident = PyLong_FromVoidPtr(obj); + if (ident == NULL) + break; + has_key = PyDict_Contains(s->markers, ident); + if (has_key) { + if (has_key != -1) + PyErr_SetString(PyExc_ValueError, "Circular reference detected"); + Py_DECREF(ident); + break; + } + if (PyDict_SetItem(s->markers, ident, obj)) { + Py_DECREF(ident); + break; + } + } + newobj = PyObject_CallFunctionObjArgs(s->defaultfn, obj, NULL); + if (newobj == NULL) { + Py_XDECREF(ident); + break; + } + rv = encoder_listencode_obj(s, rval, newobj, indent_level); + Py_DECREF(newobj); + if (rv) { + Py_XDECREF(ident); + rv = -1; + } + else if (ident != NULL) { + if (PyDict_DelItem(s->markers, ident)) { + Py_XDECREF(ident); + rv = -1; + } + Py_XDECREF(ident); + } + } + } while (0); + Py_LeaveRecursiveCall(); + return rv; +} + +static int +encoder_listencode_dict(PyEncoderObject *s, PyObject *rval, PyObject *dct, Py_ssize_t indent_level) +{ + /* Encode Python dict dct a JSON term, rval is a PyList */ + static PyObject *open_dict = NULL; + static PyObject *close_dict = NULL; + static PyObject *empty_dict = NULL; + static PyObject *iteritems = NULL; + PyObject *kstr = NULL; + PyObject *ident = NULL; + PyObject *iter = NULL; + PyObject *item = NULL; + PyObject *items = NULL; + PyObject *encoded = NULL; + int skipkeys; + Py_ssize_t idx; + + if (open_dict == NULL || close_dict == NULL || empty_dict == NULL || iteritems == NULL) { + open_dict = PyString_InternFromString("{"); + close_dict = PyString_InternFromString("}"); + empty_dict = PyString_InternFromString("{}"); + iteritems = PyString_InternFromString("iteritems"); + if (open_dict == NULL || close_dict == NULL || empty_dict == NULL || iteritems == NULL) + return -1; + } + if (PyDict_Size(dct) == 0) + return PyList_Append(rval, empty_dict); + + if (s->markers != Py_None) { + int has_key; + ident = PyLong_FromVoidPtr(dct); + if (ident == NULL) + goto bail; + has_key = PyDict_Contains(s->markers, ident); + if (has_key) { + if (has_key != -1) + PyErr_SetString(PyExc_ValueError, "Circular reference detected"); + goto bail; + } + if (PyDict_SetItem(s->markers, ident, dct)) { + goto bail; + } + } + + if (PyList_Append(rval, open_dict)) + goto bail; + + if (s->indent != Py_None) { + /* TODO: DOES NOT RUN */ + indent_level += 1; + /* + newline_indent = '\n' + (_indent * _current_indent_level) + separator = _item_separator + newline_indent + buf += newline_indent + */ + } + + if (PyObject_IsTrue(s->sort_keys)) { + /* First sort the keys then replace them with (key, value) tuples. */ + Py_ssize_t i, nitems; + if (PyDict_CheckExact(dct)) + items = PyDict_Keys(dct); + else + items = PyMapping_Keys(dct); + if (items == NULL) + goto bail; + if (!PyList_Check(items)) { + PyErr_SetString(PyExc_ValueError, "keys must return list"); + goto bail; + } + if (PyList_Sort(items) < 0) + goto bail; + nitems = PyList_GET_SIZE(items); + for (i = 0; i < nitems; i++) { + PyObject *key, *value; + key = PyList_GET_ITEM(items, i); + value = PyDict_GetItem(dct, key); + item = PyTuple_Pack(2, key, value); + if (item == NULL) + goto bail; + PyList_SET_ITEM(items, i, item); + Py_DECREF(key); + } + } + else { + if (PyDict_CheckExact(dct)) + items = PyDict_Items(dct); + else + items = PyMapping_Items(dct); + } + if (items == NULL) + goto bail; + iter = PyObject_GetIter(items); + Py_DECREF(items); + if (iter == NULL) + goto bail; + + skipkeys = PyObject_IsTrue(s->skipkeys); + idx = 0; + while ((item = PyIter_Next(iter))) { + PyObject *encoded, *key, *value; + if (!PyTuple_Check(item) || Py_SIZE(item) != 2) { + PyErr_SetString(PyExc_ValueError, "items must return 2-tuples"); + goto bail; + } + key = PyTuple_GET_ITEM(item, 0); + if (key == NULL) + goto bail; + value = PyTuple_GET_ITEM(item, 1); + if (value == NULL) + goto bail; + + encoded = PyDict_GetItem(s->key_memo, key); + if (encoded != NULL) { + Py_INCREF(encoded); + } + else if (PyString_Check(key) || PyUnicode_Check(key)) { + Py_INCREF(key); + kstr = key; + } + else if (PyFloat_Check(key)) { + kstr = encoder_encode_float(s, key); + if (kstr == NULL) + goto bail; + } + else if (key == Py_True || key == Py_False || key == Py_None) { + /* This must come before the PyInt_Check because + True and False are also 1 and 0.*/ + kstr = _encoded_const(key); + if (kstr == NULL) + goto bail; + } + else if (PyInt_Check(key) || PyLong_Check(key)) { + kstr = PyObject_Str(key); + if (kstr == NULL) + goto bail; + } + else if (skipkeys) { + Py_DECREF(item); + continue; + } + else { + /* TODO: include repr of key */ + PyErr_SetString(PyExc_TypeError, "keys must be a string"); + goto bail; + } + + if (idx) { + if (PyList_Append(rval, s->item_separator)) + goto bail; + } + + if (encoded == NULL) { + encoded = encoder_encode_string(s, kstr); + Py_CLEAR(kstr); + if (encoded == NULL) + goto bail; + if (PyDict_SetItem(s->key_memo, key, encoded)) + goto bail; + } + if (PyList_Append(rval, encoded)) { + goto bail; + } + Py_CLEAR(encoded); + if (PyList_Append(rval, s->key_separator)) + goto bail; + if (encoder_listencode_obj(s, rval, value, indent_level)) + goto bail; + Py_CLEAR(item); + idx += 1; + } + Py_CLEAR(iter); + if (PyErr_Occurred()) + goto bail; + if (ident != NULL) { + if (PyDict_DelItem(s->markers, ident)) + goto bail; + Py_CLEAR(ident); + } + if (s->indent != Py_None) { + /* TODO: DOES NOT RUN */ + indent_level -= 1; + /* + yield '\n' + (_indent * _current_indent_level) + */ + } + if (PyList_Append(rval, close_dict)) + goto bail; + return 0; + +bail: + Py_XDECREF(encoded); + Py_XDECREF(items); + Py_XDECREF(iter); + Py_XDECREF(kstr); + Py_XDECREF(ident); + return -1; +} + + +static int +encoder_listencode_list(PyEncoderObject *s, PyObject *rval, PyObject *seq, Py_ssize_t indent_level) +{ + /* Encode Python list seq to a JSON term, rval is a PyList */ + static PyObject *open_array = NULL; + static PyObject *close_array = NULL; + static PyObject *empty_array = NULL; + PyObject *ident = NULL; + PyObject *iter = NULL; + PyObject *obj = NULL; + int is_true; + int i = 0; + + if (open_array == NULL || close_array == NULL || empty_array == NULL) { + open_array = PyString_InternFromString("["); + close_array = PyString_InternFromString("]"); + empty_array = PyString_InternFromString("[]"); + if (open_array == NULL || close_array == NULL || empty_array == NULL) + return -1; + } + ident = NULL; + is_true = PyObject_IsTrue(seq); + if (is_true == -1) + return -1; + else if (is_true == 0) + return PyList_Append(rval, empty_array); + + if (s->markers != Py_None) { + int has_key; + ident = PyLong_FromVoidPtr(seq); + if (ident == NULL) + goto bail; + has_key = PyDict_Contains(s->markers, ident); + if (has_key) { + if (has_key != -1) + PyErr_SetString(PyExc_ValueError, "Circular reference detected"); + goto bail; + } + if (PyDict_SetItem(s->markers, ident, seq)) { + goto bail; + } + } + + iter = PyObject_GetIter(seq); + if (iter == NULL) + goto bail; + + if (PyList_Append(rval, open_array)) + goto bail; + if (s->indent != Py_None) { + /* TODO: DOES NOT RUN */ + indent_level += 1; + /* + newline_indent = '\n' + (_indent * _current_indent_level) + separator = _item_separator + newline_indent + buf += newline_indent + */ + } + while ((obj = PyIter_Next(iter))) { + if (i) { + if (PyList_Append(rval, s->item_separator)) + goto bail; + } + if (encoder_listencode_obj(s, rval, obj, indent_level)) + goto bail; + i++; + Py_CLEAR(obj); + } + Py_CLEAR(iter); + if (PyErr_Occurred()) + goto bail; + if (ident != NULL) { + if (PyDict_DelItem(s->markers, ident)) + goto bail; + Py_CLEAR(ident); + } + if (s->indent != Py_None) { + /* TODO: DOES NOT RUN */ + indent_level -= 1; + /* + yield '\n' + (_indent * _current_indent_level) + */ + } + if (PyList_Append(rval, close_array)) + goto bail; + return 0; + +bail: + Py_XDECREF(obj); + Py_XDECREF(iter); + Py_XDECREF(ident); + return -1; +} + +static void +encoder_dealloc(PyObject *self) +{ + /* Deallocate Encoder */ + encoder_clear(self); + Py_TYPE(self)->tp_free(self); +} + +static int +encoder_traverse(PyObject *self, visitproc visit, void *arg) +{ + PyEncoderObject *s; + assert(PyEncoder_Check(self)); + s = (PyEncoderObject *)self; + Py_VISIT(s->markers); + Py_VISIT(s->defaultfn); + Py_VISIT(s->encoder); + Py_VISIT(s->indent); + Py_VISIT(s->key_separator); + Py_VISIT(s->item_separator); + Py_VISIT(s->sort_keys); + Py_VISIT(s->skipkeys); + Py_VISIT(s->key_memo); + return 0; +} + +static int +encoder_clear(PyObject *self) +{ + /* Deallocate Encoder */ + PyEncoderObject *s; + assert(PyEncoder_Check(self)); + s = (PyEncoderObject *)self; + Py_CLEAR(s->markers); + Py_CLEAR(s->defaultfn); + Py_CLEAR(s->encoder); + Py_CLEAR(s->indent); + Py_CLEAR(s->key_separator); + Py_CLEAR(s->item_separator); + Py_CLEAR(s->sort_keys); + Py_CLEAR(s->skipkeys); + Py_CLEAR(s->key_memo); + return 0; +} + +PyDoc_STRVAR(encoder_doc, "_iterencode(obj, _current_indent_level) -> iterable"); + +static +PyTypeObject PyEncoderType = { + PyObject_HEAD_INIT(NULL) + 0, /* tp_internal */ + "simplejson._speedups.Encoder", /* tp_name */ + sizeof(PyEncoderObject), /* tp_basicsize */ + 0, /* tp_itemsize */ + encoder_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + encoder_call, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ + encoder_doc, /* tp_doc */ + encoder_traverse, /* tp_traverse */ + encoder_clear, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + encoder_members, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + encoder_init, /* tp_init */ + 0, /* tp_alloc */ + encoder_new, /* tp_new */ + 0, /* tp_free */ +}; + +static PyMethodDef speedups_methods[] = { + {"encode_basestring_ascii", + (PyCFunction)py_encode_basestring_ascii, + METH_O, + pydoc_encode_basestring_ascii}, + {"scanstring", + (PyCFunction)py_scanstring, + METH_VARARGS, + pydoc_scanstring}, + {NULL, NULL, 0, NULL} +}; + +PyDoc_STRVAR(module_doc, +"simplejson speedups\n"); + +void +init_speedups(void) +{ + PyObject *m, *decimal; + PyScannerType.tp_new = PyType_GenericNew; + if (PyType_Ready(&PyScannerType) < 0) + return; + PyEncoderType.tp_new = PyType_GenericNew; + if (PyType_Ready(&PyEncoderType) < 0) + return; + + decimal = PyImport_ImportModule("decimal"); + if (decimal == NULL) + return; + DecimalTypePtr = (PyTypeObject*)PyObject_GetAttrString(decimal, "Decimal"); + Py_DECREF(decimal); + if (DecimalTypePtr == NULL) + return; + + m = Py_InitModule3("_speedups", speedups_methods, module_doc); + Py_INCREF((PyObject*)&PyScannerType); + PyModule_AddObject(m, "make_scanner", (PyObject*)&PyScannerType); + Py_INCREF((PyObject*)&PyEncoderType); + PyModule_AddObject(m, "make_encoder", (PyObject*)&PyEncoderType); +} diff --git a/shotgun_api3/lib/simplejson/decoder.py b/shotgun_api3/lib/simplejson/decoder.py new file mode 100644 index 000000000..e5496d6e7 --- /dev/null +++ b/shotgun_api3/lib/simplejson/decoder.py @@ -0,0 +1,421 @@ +"""Implementation of JSONDecoder +""" +import re +import sys +import struct + +from simplejson.scanner import make_scanner +def _import_c_scanstring(): + try: + from simplejson._speedups import scanstring + return scanstring + except ImportError: + return None +c_scanstring = _import_c_scanstring() + +__all__ = ['JSONDecoder'] + +FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL + +def _floatconstants(): + _BYTES = '7FF80000000000007FF0000000000000'.decode('hex') + # The struct module in Python 2.4 would get frexp() out of range here + # when an endian is specified in the format string. Fixed in Python 2.5+ + if sys.byteorder != 'big': + _BYTES = _BYTES[:8][::-1] + _BYTES[8:][::-1] + nan, inf = struct.unpack('dd', _BYTES) + return nan, inf, -inf + +NaN, PosInf, NegInf = _floatconstants() + + +class JSONDecodeError(ValueError): + """Subclass of ValueError with the following additional properties: + + msg: The unformatted error message + doc: The JSON document being parsed + pos: The start index of doc where parsing failed + end: The end index of doc where parsing failed (may be None) + lineno: The line corresponding to pos + colno: The column corresponding to pos + endlineno: The line corresponding to end (may be None) + endcolno: The column corresponding to end (may be None) + + """ + def __init__(self, msg, doc, pos, end=None): + ValueError.__init__(self, errmsg(msg, doc, pos, end=end)) + self.msg = msg + self.doc = doc + self.pos = pos + self.end = end + self.lineno, self.colno = linecol(doc, pos) + if end is not None: + self.endlineno, self.endcolno = linecol(doc, end) + else: + self.endlineno, self.endcolno = None, None + + +def linecol(doc, pos): + lineno = doc.count('\n', 0, pos) + 1 + if lineno == 1: + colno = pos + else: + colno = pos - doc.rindex('\n', 0, pos) + return lineno, colno + + +def errmsg(msg, doc, pos, end=None): + # Note that this function is called from _speedups + lineno, colno = linecol(doc, pos) + if end is None: + #fmt = '{0}: line {1} column {2} (char {3})' + #return fmt.format(msg, lineno, colno, pos) + fmt = '%s: line %d column %d (char %d)' + return fmt % (msg, lineno, colno, pos) + endlineno, endcolno = linecol(doc, end) + #fmt = '{0}: line {1} column {2} - line {3} column {4} (char {5} - {6})' + #return fmt.format(msg, lineno, colno, endlineno, endcolno, pos, end) + fmt = '%s: line %d column %d - line %d column %d (char %d - %d)' + return fmt % (msg, lineno, colno, endlineno, endcolno, pos, end) + + +_CONSTANTS = { + '-Infinity': NegInf, + 'Infinity': PosInf, + 'NaN': NaN, +} + +STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS) +BACKSLASH = { + '"': u'"', '\\': u'\\', '/': u'/', + 'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t', +} + +DEFAULT_ENCODING = "utf-8" + +def py_scanstring(s, end, encoding=None, strict=True, + _b=BACKSLASH, _m=STRINGCHUNK.match): + """Scan the string s for a JSON string. End is the index of the + character in s after the quote that started the JSON string. + Unescapes all valid JSON string escape sequences and raises ValueError + on attempt to decode an invalid string. If strict is False then literal + control characters are allowed in the string. + + Returns a tuple of the decoded string and the index of the character in s + after the end quote.""" + if encoding is None: + encoding = DEFAULT_ENCODING + chunks = [] + _append = chunks.append + begin = end - 1 + while 1: + chunk = _m(s, end) + if chunk is None: + raise JSONDecodeError( + "Unterminated string starting at", s, begin) + end = chunk.end() + content, terminator = chunk.groups() + # Content is contains zero or more unescaped string characters + if content: + if not isinstance(content, unicode): + content = unicode(content, encoding) + _append(content) + # Terminator is the end of string, a literal control character, + # or a backslash denoting that an escape sequence follows + if terminator == '"': + break + elif terminator != '\\': + if strict: + msg = "Invalid control character %r at" % (terminator,) + #msg = "Invalid control character {0!r} at".format(terminator) + raise JSONDecodeError(msg, s, end) + else: + _append(terminator) + continue + try: + esc = s[end] + except IndexError: + raise JSONDecodeError( + "Unterminated string starting at", s, begin) + # If not a unicode escape sequence, must be in the lookup table + if esc != 'u': + try: + char = _b[esc] + except KeyError: + msg = "Invalid \\escape: " + repr(esc) + raise JSONDecodeError(msg, s, end) + end += 1 + else: + # Unicode escape sequence + esc = s[end + 1:end + 5] + next_end = end + 5 + if len(esc) != 4: + msg = "Invalid \\uXXXX escape" + raise JSONDecodeError(msg, s, end) + uni = int(esc, 16) + # Check for surrogate pair on UCS-4 systems + if 0xd800 <= uni <= 0xdbff and sys.maxunicode > 65535: + msg = "Invalid \\uXXXX\\uXXXX surrogate pair" + if not s[end + 5:end + 7] == '\\u': + raise JSONDecodeError(msg, s, end) + esc2 = s[end + 7:end + 11] + if len(esc2) != 4: + raise JSONDecodeError(msg, s, end) + uni2 = int(esc2, 16) + uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00)) + next_end += 6 + char = unichr(uni) + end = next_end + # Append the unescaped character + _append(char) + return u''.join(chunks), end + + +# Use speedup if available +scanstring = c_scanstring or py_scanstring + +WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS) +WHITESPACE_STR = ' \t\n\r' + +def JSONObject((s, end), encoding, strict, scan_once, object_hook, + object_pairs_hook, memo=None, + _w=WHITESPACE.match, _ws=WHITESPACE_STR): + # Backwards compatibility + if memo is None: + memo = {} + memo_get = memo.setdefault + pairs = [] + # Use a slice to prevent IndexError from being raised, the following + # check will raise a more specific ValueError if the string is empty + nextchar = s[end:end + 1] + # Normally we expect nextchar == '"' + if nextchar != '"': + if nextchar in _ws: + end = _w(s, end).end() + nextchar = s[end:end + 1] + # Trivial empty object + if nextchar == '}': + if object_pairs_hook is not None: + result = object_pairs_hook(pairs) + return result, end + 1 + pairs = {} + if object_hook is not None: + pairs = object_hook(pairs) + return pairs, end + 1 + elif nextchar != '"': + raise JSONDecodeError("Expecting property name", s, end) + end += 1 + while True: + key, end = scanstring(s, end, encoding, strict) + key = memo_get(key, key) + + # To skip some function call overhead we optimize the fast paths where + # the JSON key separator is ": " or just ":". + if s[end:end + 1] != ':': + end = _w(s, end).end() + if s[end:end + 1] != ':': + raise JSONDecodeError("Expecting : delimiter", s, end) + + end += 1 + + try: + if s[end] in _ws: + end += 1 + if s[end] in _ws: + end = _w(s, end + 1).end() + except IndexError: + pass + + try: + value, end = scan_once(s, end) + except StopIteration: + raise JSONDecodeError("Expecting object", s, end) + pairs.append((key, value)) + + try: + nextchar = s[end] + if nextchar in _ws: + end = _w(s, end + 1).end() + nextchar = s[end] + except IndexError: + nextchar = '' + end += 1 + + if nextchar == '}': + break + elif nextchar != ',': + raise JSONDecodeError("Expecting , delimiter", s, end - 1) + + try: + nextchar = s[end] + if nextchar in _ws: + end += 1 + nextchar = s[end] + if nextchar in _ws: + end = _w(s, end + 1).end() + nextchar = s[end] + except IndexError: + nextchar = '' + + end += 1 + if nextchar != '"': + raise JSONDecodeError("Expecting property name", s, end - 1) + + if object_pairs_hook is not None: + result = object_pairs_hook(pairs) + return result, end + pairs = dict(pairs) + if object_hook is not None: + pairs = object_hook(pairs) + return pairs, end + +def JSONArray((s, end), scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR): + values = [] + nextchar = s[end:end + 1] + if nextchar in _ws: + end = _w(s, end + 1).end() + nextchar = s[end:end + 1] + # Look-ahead for trivial empty array + if nextchar == ']': + return values, end + 1 + _append = values.append + while True: + try: + value, end = scan_once(s, end) + except StopIteration: + raise JSONDecodeError("Expecting object", s, end) + _append(value) + nextchar = s[end:end + 1] + if nextchar in _ws: + end = _w(s, end + 1).end() + nextchar = s[end:end + 1] + end += 1 + if nextchar == ']': + break + elif nextchar != ',': + raise JSONDecodeError("Expecting , delimiter", s, end) + + try: + if s[end] in _ws: + end += 1 + if s[end] in _ws: + end = _w(s, end + 1).end() + except IndexError: + pass + + return values, end + +class JSONDecoder(object): + """Simple JSON decoder + + Performs the following translations in decoding by default: + + +---------------+-------------------+ + | JSON | Python | + +===============+===================+ + | object | dict | + +---------------+-------------------+ + | array | list | + +---------------+-------------------+ + | string | unicode | + +---------------+-------------------+ + | number (int) | int, long | + +---------------+-------------------+ + | number (real) | float | + +---------------+-------------------+ + | true | True | + +---------------+-------------------+ + | false | False | + +---------------+-------------------+ + | null | None | + +---------------+-------------------+ + + It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as + their corresponding ``float`` values, which is outside the JSON spec. + + """ + + def __init__(self, encoding=None, object_hook=None, parse_float=None, + parse_int=None, parse_constant=None, strict=True, + object_pairs_hook=None): + """ + *encoding* determines the encoding used to interpret any + :class:`str` objects decoded by this instance (``'utf-8'`` by + default). It has no effect when decoding :class:`unicode` objects. + + Note that currently only encodings that are a superset of ASCII work, + strings of other encodings should be passed in as :class:`unicode`. + + *object_hook*, if specified, will be called with the result of every + JSON object decoded and its return value will be used in place of the + given :class:`dict`. This can be used to provide custom + deserializations (e.g. to support JSON-RPC class hinting). + + *object_pairs_hook* is an optional function that will be called with + the result of any object literal decode with an ordered list of pairs. + The return value of *object_pairs_hook* will be used instead of the + :class:`dict`. This feature can be used to implement custom decoders + that rely on the order that the key and value pairs are decoded (for + example, :func:`collections.OrderedDict` will remember the order of + insertion). If *object_hook* is also defined, the *object_pairs_hook* + takes priority. + + *parse_float*, if specified, will be called with the string of every + JSON float to be decoded. By default, this is equivalent to + ``float(num_str)``. This can be used to use another datatype or parser + for JSON floats (e.g. :class:`decimal.Decimal`). + + *parse_int*, if specified, will be called with the string of every + JSON int to be decoded. By default, this is equivalent to + ``int(num_str)``. This can be used to use another datatype or parser + for JSON integers (e.g. :class:`float`). + + *parse_constant*, if specified, will be called with one of the + following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. This + can be used to raise an exception if invalid JSON numbers are + encountered. + + *strict* controls the parser's behavior when it encounters an + invalid control character in a string. The default setting of + ``True`` means that unescaped control characters are parse errors, if + ``False`` then control characters will be allowed in strings. + + """ + self.encoding = encoding + self.object_hook = object_hook + self.object_pairs_hook = object_pairs_hook + self.parse_float = parse_float or float + self.parse_int = parse_int or int + self.parse_constant = parse_constant or _CONSTANTS.__getitem__ + self.strict = strict + self.parse_object = JSONObject + self.parse_array = JSONArray + self.parse_string = scanstring + self.memo = {} + self.scan_once = make_scanner(self) + + def decode(self, s, _w=WHITESPACE.match): + """Return the Python representation of ``s`` (a ``str`` or ``unicode`` + instance containing a JSON document) + + """ + obj, end = self.raw_decode(s, idx=_w(s, 0).end()) + end = _w(s, end).end() + if end != len(s): + raise JSONDecodeError("Extra data", s, end, len(s)) + return obj + + def raw_decode(self, s, idx=0): + """Decode a JSON document from ``s`` (a ``str`` or ``unicode`` + beginning with a JSON document) and return a 2-tuple of the Python + representation and the index in ``s`` where the document ended. + + This can be used to decode a JSON document from a string that may + have extraneous data at the end. + + """ + try: + obj, end = self.scan_once(s, idx) + except StopIteration: + raise JSONDecodeError("No JSON object could be decoded", s, idx) + return obj, end diff --git a/shotgun_api3/lib/simplejson/encoder.py b/shotgun_api3/lib/simplejson/encoder.py new file mode 100644 index 000000000..f43f6f430 --- /dev/null +++ b/shotgun_api3/lib/simplejson/encoder.py @@ -0,0 +1,503 @@ +"""Implementation of JSONEncoder +""" +import re +from decimal import Decimal + +def _import_speedups(): + try: + from simplejson import _speedups + return _speedups.encode_basestring_ascii, _speedups.make_encoder + except ImportError: + return None, None +c_encode_basestring_ascii, c_make_encoder = _import_speedups() + +from simplejson.decoder import PosInf + +ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]') +ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])') +HAS_UTF8 = re.compile(r'[\x80-\xff]') +ESCAPE_DCT = { + '\\': '\\\\', + '"': '\\"', + '\b': '\\b', + '\f': '\\f', + '\n': '\\n', + '\r': '\\r', + '\t': '\\t', +} +for i in range(0x20): + #ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i)) + ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,)) + +FLOAT_REPR = repr + +def encode_basestring(s): + """Return a JSON representation of a Python string + + """ + if isinstance(s, str) and HAS_UTF8.search(s) is not None: + s = s.decode('utf-8') + def replace(match): + return ESCAPE_DCT[match.group(0)] + return u'"' + ESCAPE.sub(replace, s) + u'"' + + +def py_encode_basestring_ascii(s): + """Return an ASCII-only JSON representation of a Python string + + """ + if isinstance(s, str) and HAS_UTF8.search(s) is not None: + s = s.decode('utf-8') + def replace(match): + s = match.group(0) + try: + return ESCAPE_DCT[s] + except KeyError: + n = ord(s) + if n < 0x10000: + #return '\\u{0:04x}'.format(n) + return '\\u%04x' % (n,) + else: + # surrogate pair + n -= 0x10000 + s1 = 0xd800 | ((n >> 10) & 0x3ff) + s2 = 0xdc00 | (n & 0x3ff) + #return '\\u{0:04x}\\u{1:04x}'.format(s1, s2) + return '\\u%04x\\u%04x' % (s1, s2) + return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"' + + +encode_basestring_ascii = ( + c_encode_basestring_ascii or py_encode_basestring_ascii) + +class JSONEncoder(object): + """Extensible JSON encoder for Python data structures. + + Supports the following objects and types by default: + + +-------------------+---------------+ + | Python | JSON | + +===================+===============+ + | dict | object | + +-------------------+---------------+ + | list, tuple | array | + +-------------------+---------------+ + | str, unicode | string | + +-------------------+---------------+ + | int, long, float | number | + +-------------------+---------------+ + | True | true | + +-------------------+---------------+ + | False | false | + +-------------------+---------------+ + | None | null | + +-------------------+---------------+ + + To extend this to recognize other objects, subclass and implement a + ``.default()`` method with another method that returns a serializable + object for ``o`` if possible, otherwise it should call the superclass + implementation (to raise ``TypeError``). + + """ + item_separator = ', ' + key_separator = ': ' + def __init__(self, skipkeys=False, ensure_ascii=True, + check_circular=True, allow_nan=True, sort_keys=False, + indent=None, separators=None, encoding='utf-8', default=None, + use_decimal=False): + """Constructor for JSONEncoder, with sensible defaults. + + If skipkeys is false, then it is a TypeError to attempt + encoding of keys that are not str, int, long, float or None. If + skipkeys is True, such items are simply skipped. + + If ensure_ascii is true, the output is guaranteed to be str + objects with all incoming unicode characters escaped. If + ensure_ascii is false, the output will be unicode object. + + If check_circular is true, then lists, dicts, and custom encoded + objects will be checked for circular references during encoding to + prevent an infinite recursion (which would cause an OverflowError). + Otherwise, no such check takes place. + + If allow_nan is true, then NaN, Infinity, and -Infinity will be + encoded as such. This behavior is not JSON specification compliant, + but is consistent with most JavaScript based encoders and decoders. + Otherwise, it will be a ValueError to encode such floats. + + If sort_keys is true, then the output of dictionaries will be + sorted by key; this is useful for regression tests to ensure + that JSON serializations can be compared on a day-to-day basis. + + If indent is a string, then JSON array elements and object members + will be pretty-printed with a newline followed by that string repeated + for each level of nesting. ``None`` (the default) selects the most compact + representation without any newlines. For backwards compatibility with + versions of simplejson earlier than 2.1.0, an integer is also accepted + and is converted to a string with that many spaces. + + If specified, separators should be a (item_separator, key_separator) + tuple. The default is (', ', ': '). To get the most compact JSON + representation you should specify (',', ':') to eliminate whitespace. + + If specified, default is a function that gets called for objects + that can't otherwise be serialized. It should return a JSON encodable + version of the object or raise a ``TypeError``. + + If encoding is not None, then all input strings will be + transformed into unicode using that encoding prior to JSON-encoding. + The default is UTF-8. + + If use_decimal is true (not the default), ``decimal.Decimal`` will + be supported directly by the encoder. For the inverse, decode JSON + with ``parse_float=decimal.Decimal``. + + """ + + self.skipkeys = skipkeys + self.ensure_ascii = ensure_ascii + self.check_circular = check_circular + self.allow_nan = allow_nan + self.sort_keys = sort_keys + self.use_decimal = use_decimal + if isinstance(indent, (int, long)): + indent = ' ' * indent + self.indent = indent + if separators is not None: + self.item_separator, self.key_separator = separators + elif indent is not None: + self.item_separator = ',' + if default is not None: + self.default = default + self.encoding = encoding + + def default(self, o): + """Implement this method in a subclass such that it returns + a serializable object for ``o``, or calls the base implementation + (to raise a ``TypeError``). + + For example, to support arbitrary iterators, you could + implement default like this:: + + def default(self, o): + try: + iterable = iter(o) + except TypeError: + pass + else: + return list(iterable) + return JSONEncoder.default(self, o) + + """ + raise TypeError(repr(o) + " is not JSON serializable") + + def encode(self, o): + """Return a JSON string representation of a Python data structure. + + >>> from simplejson import JSONEncoder + >>> JSONEncoder().encode({"foo": ["bar", "baz"]}) + '{"foo": ["bar", "baz"]}' + + """ + # This is for extremely simple cases and benchmarks. + if isinstance(o, basestring): + if isinstance(o, str): + _encoding = self.encoding + if (_encoding is not None + and not (_encoding == 'utf-8')): + o = o.decode(_encoding) + if self.ensure_ascii: + return encode_basestring_ascii(o) + else: + return encode_basestring(o) + # This doesn't pass the iterator directly to ''.join() because the + # exceptions aren't as detailed. The list call should be roughly + # equivalent to the PySequence_Fast that ''.join() would do. + chunks = self.iterencode(o, _one_shot=True) + if not isinstance(chunks, (list, tuple)): + chunks = list(chunks) + if self.ensure_ascii: + return ''.join(chunks) + else: + return u''.join(chunks) + + def iterencode(self, o, _one_shot=False): + """Encode the given object and yield each string + representation as available. + + For example:: + + for chunk in JSONEncoder().iterencode(bigobject): + mysocket.write(chunk) + + """ + if self.check_circular: + markers = {} + else: + markers = None + if self.ensure_ascii: + _encoder = encode_basestring_ascii + else: + _encoder = encode_basestring + if self.encoding != 'utf-8': + def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding): + if isinstance(o, str): + o = o.decode(_encoding) + return _orig_encoder(o) + + def floatstr(o, allow_nan=self.allow_nan, + _repr=FLOAT_REPR, _inf=PosInf, _neginf=-PosInf): + # Check for specials. Note that this type of test is processor + # and/or platform-specific, so do tests which don't depend on + # the internals. + + if o != o: + text = 'NaN' + elif o == _inf: + text = 'Infinity' + elif o == _neginf: + text = '-Infinity' + else: + return _repr(o) + + if not allow_nan: + raise ValueError( + "Out of range float values are not JSON compliant: " + + repr(o)) + + return text + + + key_memo = {} + if (_one_shot and c_make_encoder is not None + and self.indent is None): + _iterencode = c_make_encoder( + markers, self.default, _encoder, self.indent, + self.key_separator, self.item_separator, self.sort_keys, + self.skipkeys, self.allow_nan, key_memo, self.use_decimal) + else: + _iterencode = _make_iterencode( + markers, self.default, _encoder, self.indent, floatstr, + self.key_separator, self.item_separator, self.sort_keys, + self.skipkeys, _one_shot, self.use_decimal) + try: + return _iterencode(o, 0) + finally: + key_memo.clear() + + +class JSONEncoderForHTML(JSONEncoder): + """An encoder that produces JSON safe to embed in HTML. + + To embed JSON content in, say, a script tag on a web page, the + characters &, < and > should be escaped. They cannot be escaped + with the usual entities (e.g. &) because they are not expanded + within