From 5a13ad03c264e5b0534dff2e65e78dff7afd5f20 Mon Sep 17 00:00:00 2001 From: Eric Martin Date: Wed, 15 Oct 2014 14:25:29 -0500 Subject: [PATCH 01/44] Initial commit --- README.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..b8ece0d --- /dev/null +++ b/README.md @@ -0,0 +1,4 @@ +stackify-api-python +=================== + +Stackify API for Python From 814b8d13bf5eb2bcf98cfb1a90e717423911bd57 Mon Sep 17 00:00:00 2001 From: Elpedio Adoptante Jr Date: Wed, 27 Feb 2019 00:00:08 +0800 Subject: [PATCH 02/44] Python Logging Integration Changes: - clean up codes - created constants.py for CONSTANTS - added requirements.txt for dependencies - added time interval for sending messages --- .gitignore | 113 ++++++++++++++++++++++++++++++++------ README.md | 22 ++++++-- requirements.txt | 5 ++ setup.cfg | 23 ++++++++ setup.py | 33 ++++++----- stackify/__init__.py | 36 +++--------- stackify/application.py | 5 +- stackify/constants.py | 22 ++++++++ stackify/error.py | 21 ------- stackify/handler.py | 54 ++++++++++++------ stackify/http.py | 30 +++++----- stackify/log.py | 5 +- stackify/timer.py | 36 ++++++++++++ tests/bases.py | 3 +- tests/test_application.py | 22 ++++---- tests/test_formats.py | 5 +- tests/test_handler.py | 12 ++-- tests/test_http.py | 39 +++++++------ tests/test_init.py | 24 ++++---- tests/test_log.py | 20 ++----- 20 files changed, 334 insertions(+), 196 deletions(-) create mode 100644 requirements.txt create mode 100644 setup.cfg create mode 100644 stackify/constants.py create mode 100644 stackify/timer.py diff --git a/.gitignore b/.gitignore index 1ae3595..25344cd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,32 +1,109 @@ -*.py[co] +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class -# Packages -*.egg -*.egg-info -dist -build -eggs -parts -var -sdist -develop-eggs +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ .installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec # Installer logs pip-log.txt +pip-delete-this-directory.txt # Unit test / coverage reports +htmlcov/ +.tox/ .coverage -.tox +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ -#Translations +# Translations *.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ -#Mr Developer -.mr.developer.cfg +# PyBuilder +target/ -# virutalenvs +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env .venv +env/ +venv/ +env*/ +venv*/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ -# debug stuff -test.py +# Intellij +.idea diff --git a/README.md b/README.md index f29f16a..d30414e 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ Stackify API for Python -======= +======================= [Stackify](https://stackify.com) support for Python programs. @@ -35,12 +35,28 @@ export STACKIFY_API_KEY=****** These options can also be provided in your code: ```python +# Standard API import stackify logger = stackify.getLogger(application="MyApp", environment="Dev", api_key=******) logger.warning('Something happened') ``` +```python +# Python Logging Integration +import logging +import stackify + +# your existing logging +logger = logging.getLogger() +... + +stackify_handler = stackify.StackifyHandler(application="MyApp", environment="Dev", api_key=******) +logger.addHandler(stackify_handler) + +logger.warning('Something happened') +``` + ## Usage stackify-python handles uploads in batches of 100 messages at a time on another thread. @@ -48,10 +64,6 @@ When your program exits, it will shut the thread down and upload the remaining m Stackify can store extra data along with your log message: ```python -import stackify - -logger = stackify.getLogger() - try: user_string = raw_input("Enter a number: ") print("You entered", int(user_string)) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..3c83c04 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +mock==2.0.0 +pytest==4.3.0 +pytest-cov==2.6.1 +requests==2.21.0 +retrying==1.3.3 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..4878226 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,23 @@ +[flake8] +ignore = E501, W605 +exclude = + .git, + __pycache__, + build, + dist, + env*, + venv*, + setup.cfg, + README.md, + LICENSE.txt, + requirements.txt, + + +[coverage:run] +include = + stackify/* +omit = + *tests* + +[tool:pytest] +python_files=tests.py test.py test_*.py *_test.py tests_*.py *_tests.py diff --git a/setup.py b/setup.py index d19d42b..b0c3c9f 100755 --- a/setup.py +++ b/setup.py @@ -5,10 +5,10 @@ try: from pypandoc import convert - read_md = lambda f: convert(f, 'rst') + read_md = lambda f: convert(f, 'rst') # noqa except ImportError: print('warning: pypandoc module not found, could not convert Markdown to RST') - read_md = lambda f: open(f).read() + read_md = lambda f: open(f).read() # noqa version_re = re.compile(r'__version__\s+=\s+(.*)') @@ -17,26 +17,25 @@ version = ast.literal_eval(version_re.search(f).group(1)) setup( - name = 'stackify', - version = version, - author = 'Matthew Thompson', - author_email = 'chameleonator@gmail.com', - packages = ['stackify'], - url = 'https://github.com/stackify/stackify-api-python', - license = open('LICENSE.txt').readline(), - description = 'Stackify API for Python', - long_description = read_md('README.md'), - download_url = 'https://github.com/stackify/stackify-api-python/tarball/0.0.1', - keywords = ['logging', 'stackify', 'exception'], + name='stackify', + version=version, + author='Matthew Thompson', + author_email='chameleonator@gmail.com', + packages=['stackify'], + url='https://github.com/stackify/stackify-api-python', + license=open('LICENSE.txt').readline(), + description='Stackify API for Python', + long_description=read_md('README.md'), + download_url='https://github.com/stackify/stackify-api-python/tarball/0.0.1', + keywords=['logging', 'stackify', 'exception'], classifiers=["Programming Language :: Python"], - install_requires = [ + install_requires=[ 'retrying>=1.2.3', 'requests>=2.4.1' ], - test_suite = 'tests', - tests_requires = [ + test_suite='tests', + tests_requires=[ 'mock>=1.0.1', 'nose==1.3.4' ] ) - diff --git a/stackify/__init__.py b/stackify/__init__.py index f276a43..0e1a298 100644 --- a/stackify/__init__.py +++ b/stackify/__init__.py @@ -1,45 +1,24 @@ """ Stackify Python API """ - +__all__ = ("VERSION") __version__ = '0.0.1' - -API_URL = 'https://api.stackify.com' - -READ_TIMEOUT = 5000 - -MAX_BATCH = 100 - -QUEUE_SIZE = 1000 - import logging import inspect import atexit -DEFAULT_LEVEL = logging.ERROR - -LOGGING_LEVELS = { - logging.CRITICAL: 'CRITICAL', - logging.ERROR: 'ERROR', - logging.WARNING: 'WARNING', - logging.INFO: 'INFO', - logging.DEBUG: 'DEBUG', - logging.NOTSET: 'NOTSET' -} +from stackify.application import ApiConfiguration # noqa +from stackify.constants import DEFAULT_LEVEL +from stackify.handler import StackifyHandler class NullHandler(logging.Handler): def emit(self, record): pass -logging.getLogger(__name__).addHandler(NullHandler()) - -from stackify.application import ApiConfiguration -from stackify.http import HTTPClient - -from stackify.handler import StackifyHandler +logging.getLogger(__name__).addHandler(NullHandler()) def getLogger(name=None, auto_shutdown=True, basic_config=True, **kwargs): @@ -77,8 +56,6 @@ def getLogger(name=None, auto_shutdown=True, basic_config=True, **kwargs): if not [isinstance(x, StackifyHandler) for x in logger.handlers]: internal_logger = logging.getLogger(__name__) internal_logger.debug('Creating handler for logger %s', name) - handler = StackifyHandler(**kwargs) - logger.addHandler(handler) if auto_shutdown: internal_logger.debug('Registering atexit callback') @@ -87,6 +64,9 @@ def getLogger(name=None, auto_shutdown=True, basic_config=True, **kwargs): if logger.getEffectiveLevel() == logging.NOTSET: logger.setLevel(DEFAULT_LEVEL) + handler = StackifyHandler(ensure_at_exit=not auto_shutdown, **kwargs) + logger.addHandler(handler) + handler.listener.start() return logger diff --git a/stackify/application.py b/stackify/application.py index 69e8e71..2201df9 100644 --- a/stackify/application.py +++ b/stackify/application.py @@ -1,7 +1,7 @@ import socket import os -from stackify import API_URL +from stackify.constants import API_URL from stackify.formats import JSONObject @@ -32,8 +32,7 @@ def arg_or_env(name, args, default=None): if default: return default else: - raise NameError('You must specify the keyword argument {0} or ' - 'environment variable {1}'.format(name, env_name)) + raise NameError('You must specify the keyword argument {0} or environment variable {1}'.format(name, env_name)) def get_configuration(**kwargs): diff --git a/stackify/constants.py b/stackify/constants.py new file mode 100644 index 0000000..ddb2d36 --- /dev/null +++ b/stackify/constants.py @@ -0,0 +1,22 @@ +import logging + + +API_URL = 'https://api.stackify.com' +IDENTIFY_URL = '/Metrics/IdentifyApp' +LOG_SAVE_URL = '/Log/Save' +API_REQUEST_INTERVAL_IN_SEC = 5 + +MIN_BATCH = 10 +MAX_BATCH = 100 +QUEUE_SIZE = 1000 +READ_TIMEOUT = 5000 + +LOGGING_LEVELS = { + logging.CRITICAL: 'CRITICAL', + logging.ERROR: 'ERROR', + logging.WARNING: 'WARNING', + logging.INFO: 'INFO', + logging.DEBUG: 'DEBUG', + logging.NOTSET: 'NOTSET' +} +DEFAULT_LEVEL = logging.ERROR diff --git a/stackify/error.py b/stackify/error.py index 7ce5c42..2d59607 100644 --- a/stackify/error.py +++ b/stackify/error.py @@ -1,5 +1,4 @@ import traceback -import time import sys from stackify.formats import JSONObject @@ -37,31 +36,11 @@ def __init__(self, filename, lineno, method): self.Method = method -class WebRequestDetail(JSONObject): - def __init__(self): - self.UserIPAddress = None - self.HttpMethod = None - self.RequestProtocol = None - self.RequestUrl = None - self.RequestUrlRoot = None - self.ReferralUrl = None - self.Headers = {} - self.Cookies = {} - self.QueryString = {} - self.PostData = {} - self.SessionData = {} - self.PostDataRaw = None - self.MVCAction = None - self.MVCController = None - self.MVCArea = None - - class StackifyError(JSONObject): def __init__(self): self.EnvironmentDetail = None # environment detail object self.OccurredEpochMillis = None self.Error = None # ErrorItem object - self.WebRequestDetail = None # WebRequestDetail object self.CustomerName = None self.UserName = None diff --git a/stackify/handler.py b/stackify/handler.py index 20b44a4..9a24b9b 100644 --- a/stackify/handler.py +++ b/stackify/handler.py @@ -1,10 +1,9 @@ import logging -import threading -import os +import atexit try: from logging.handlers import QueueHandler, QueueListener -except: # pragma: no cover +except Exception: # pragma: no cover from stackify.handler_backport import QueueHandler, QueueListener try: @@ -12,11 +11,13 @@ except ImportError: # pragma: no cover import queue -from stackify import QUEUE_SIZE, API_URL, MAX_BATCH -from stackify.log import LogMsg, LogMsgGroup -from stackify.error import ErrorItem -from stackify.http import HTTPClient from stackify.application import get_configuration +from stackify.constants import API_REQUEST_INTERVAL_IN_SEC +from stackify.constants import MAX_BATCH +from stackify.constants import QUEUE_SIZE +from stackify.http import HTTPClient +from stackify.log import LogMsg, LogMsgGroup +from stackify.timer import RepeatedTimer class StackifyHandler(QueueHandler): @@ -25,10 +26,9 @@ class StackifyHandler(QueueHandler): transmission to Stackify servers. ''' - def __init__(self, queue_=None, listener=None, **kwargs): + def __init__(self, queue_=None, listener=None, ensure_at_exit=True, **kwargs): if queue_ is None: queue_ = queue.Queue(QUEUE_SIZE) - logger = logging.getLogger(__name__) super(StackifyHandler, self).__init__(queue_) @@ -36,6 +36,10 @@ def __init__(self, queue_=None, listener=None, **kwargs): listener = StackifyListener(queue_, **kwargs) self.listener = listener + self.listener.start() + + if ensure_at_exit: + atexit.register(self.listener.stop) def enqueue(self, record): ''' @@ -45,8 +49,7 @@ def enqueue(self, record): self.queue.put_nowait(record) except queue.Full: logger = logging.getLogger(__name__) - logger.warn('StackifyHandler queue is full, ' - 'evicting oldest record') + logger.warning('StackifyHandler queue is full, evicting oldest record') self.queue.get_nowait() self.queue.put_nowait(record) @@ -65,6 +68,9 @@ def __init__(self, queue_, max_batch=MAX_BATCH, config=None, **kwargs): self.max_batch = max_batch self.messages = [] self.http = HTTPClient(config) + self.timer = RepeatedTimer(API_REQUEST_INTERVAL_IN_SEC, self.send_group) + + self._started = False def handle(self, record): if not self.http.identified: @@ -80,22 +86,36 @@ def handle(self, record): self.send_group() def send_group(self): + if not self.messages: + return + group = LogMsgGroup(self.messages) try: self.http.send_log_group(group) - except: + except Exception: logger = logging.getLogger(__name__) - logger.exception('Could not send %s log messages, discarding', - len(self.messages)) + logger.exception('Could not send {} log messages, discarding'.format(len(self.messages))) del self.messages[:] + def start(self): + logger = logging.getLogger(__name__) + logger.debug('Starting up listener') + + if not self._started: + super(StackifyListener, self).start() + self.timer.start() + self._started = True + def stop(self): logger = logging.getLogger(__name__) logger.debug('Shutting down listener') - super(StackifyListener, self).stop() + + if self._started: + super(StackifyListener, self).stop() + self.timer.stop() + self._started = False # send any remaining messages if self.messages: - logger.debug('%s messages left on shutdown, uploading', - len(self.messages)) + logger.debug('{} messages left on shutdown, uploading'.format(len(self.messages))) self.send_group() diff --git a/stackify/http.py b/stackify/http.py index 82b8ca7..ee465b3 100644 --- a/stackify/http.py +++ b/stackify/http.py @@ -5,12 +5,17 @@ try: from cStringIO import StringIO -except: +except Exception: try: from StringIO import StringIO - except: + except Exception: pass # python 3, we use a new function in gzip +from stackify.application import EnvironmentDetail +from stackify.constants import IDENTIFY_URL +from stackify.constants import LOG_SAVE_URL +from stackify.constants import READ_TIMEOUT + def gzip_compress(data): if hasattr(gzip, 'compress'): @@ -23,10 +28,6 @@ def gzip_compress(data): return s.getvalue() -from stackify.application import EnvironmentDetail -from stackify import READ_TIMEOUT - - class HTTPClient: def __init__(self, api_config): self.api_config = api_config @@ -41,7 +42,7 @@ def __init__(self, api_config): def POST(self, url, json_object, use_gzip=False): request_url = self.api_config.api_url + url logger = logging.getLogger(__name__) - logger.debug('Request URL: %s', request_url) + logger.debug('Request URL: {}'.format(request_url)) headers = { 'Content-Type': 'application/json', @@ -51,7 +52,7 @@ def POST(self, url, json_object, use_gzip=False): try: payload_data = json_object.toJSON() - logger.debug('POST data: %s', payload_data) + logger.debug('POST data: {}'.format(payload_data)) if use_gzip: headers['Content-Encoding'] = 'gzip' @@ -61,19 +62,17 @@ def POST(self, url, json_object, use_gzip=False): data=payload_data, headers=headers, timeout=READ_TIMEOUT) - logger.debug('Response: %s', response.text) + logger.debug('Response: {}'.format(response.text)) return response.json() except requests.exceptions.RequestException: logger.exception('HTTP exception') - raise - except ValueError as e: + except ValueError: # could not read json response logger.exception('Cannot decode JSON response') - raise @retrying.retry(wait_exponential_multiplier=1000, stop_max_delay=10000) def identify_application(self): - result = self.POST('/Metrics/IdentifyApp', self.environment_detail) + result = self.POST(IDENTIFY_URL, self.environment_detail) self.app_name_id = result.get('AppNameID') self.app_env_id = result.get('AppEnvID') self.device_id = result.get('DeviceID') @@ -87,6 +86,5 @@ def send_log_group(self, group): group.CDAppID = self.device_app_id group.AppNameID = self.app_name_id group.ServerName = self.device_alias - if not group.ServerName: - group.ServerName = self.environment_detail.deviceName - self.POST('/Log/Save', group, True) + group.ServerName = group.ServerName or self.environment_detail.deviceName + self.POST(LOG_SAVE_URL, group, True) diff --git a/stackify/log.py b/stackify/log.py index c51f5f3..21e2e0b 100644 --- a/stackify/log.py +++ b/stackify/log.py @@ -2,14 +2,11 @@ import logging from stackify.formats import JSONObject - -from stackify import MAX_BATCH, LOGGING_LEVELS from stackify.error import StackifyError # this is used to separate builtin keys from user-specified keys -RECORD_VARS = set(logging.LogRecord('', '', '', '', - '', '', '', '').__dict__.keys()) +RECORD_VARS = set(logging.LogRecord('', '', '', '', '', '', '', '').__dict__.keys()) # the "message" attribute is saved on the record object by a Formatter RECORD_VARS.add('message') diff --git a/stackify/timer.py b/stackify/timer.py new file mode 100644 index 0000000..0bb0c0a --- /dev/null +++ b/stackify/timer.py @@ -0,0 +1,36 @@ +import time +from threading import Event, Thread + + +class RepeatedTimer(object): + ''' + Repeater class that call the function every interval seconds. + ''' + + def __init__(self, interval, function, *args, **kwargs): + self.interval = interval + self.function = function + self.args = args + self.kwargs = kwargs + self.started = time.time() + self.event = Event() + self.thread = Thread(target=self._target) + self._started = False + + def _target(self): + while not self.event.wait(self._time): + self.function(*self.args, **self.kwargs) + + @property + def _time(self): + return self.interval - ((time.time() - self.started) % self.interval) + + def start(self): + if not self._started: + self.thread.setDaemon(True) + self.thread.start() + + def stop(self): + if self._started: + self.event.set() + self.thread.join() diff --git a/tests/bases.py b/tests/bases.py index 7c4e4b0..d5575f1 100644 --- a/tests/bases.py +++ b/tests/bases.py @@ -1,6 +1,6 @@ import os import unittest -import retrying + class ClearEnvTest(unittest.TestCase): ''' @@ -27,4 +27,3 @@ def tearDown(self): for key, item in self.saved.items(): os.environ[key] = item del self.saved - diff --git a/tests/test_application.py b/tests/test_application.py index 7e43248..55b0296 100644 --- a/tests/test_application.py +++ b/tests/test_application.py @@ -4,10 +4,9 @@ import unittest from mock import patch -import os from .bases import ClearEnvTest -from stackify import API_URL +from stackify.constants import API_URL from stackify.application import get_configuration @@ -67,10 +66,11 @@ def test_kwarg_mix(self): def test_kwargs(self): '''API configuration can load from kwargs''' config = get_configuration( - application = 'test3_appname', - environment = 'test3_environment', - api_key = 'test3_apikey', - api_url = 'test3_apiurl') + application='test3_appname', + environment='test3_environment', + api_key='test3_apikey', + api_url='test3_apiurl', + ) self.assertEqual(config.application, 'test3_appname') self.assertEqual(config.environment, 'test3_environment') @@ -80,9 +80,10 @@ def test_kwargs(self): def test_api_url_default(self): '''API URL is set automatically''' config = get_configuration( - application = 'test4_appname', - environment = 'test4_environment', - api_key = 'test4_apikey') + application='test4_appname', + environment='test4_environment', + api_key='test4_apikey', + ) self.assertEqual(config.application, 'test4_appname') self.assertEqual(config.environment, 'test4_environment') @@ -90,6 +91,5 @@ def test_api_url_default(self): self.assertEqual(config.api_url, API_URL) -if __name__=='__main__': +if __name__ == '__main__': unittest.main() - diff --git a/tests/test_formats.py b/tests/test_formats.py index 6c10b64..bfa6c11 100644 --- a/tests/test_formats.py +++ b/tests/test_formats.py @@ -3,11 +3,11 @@ """ import unittest -from mock import patch, Mock import json from stackify.formats import JSONObject + class TestJSONObject(unittest.TestCase): ''' Test the JSON serializer object @@ -51,6 +51,5 @@ def __init__(self): self.assertEqual(json.loads(result), {'a': '1', 'b': False, 'd': []}) -if __name__=='__main__': +if __name__ == '__main__': unittest.main() - diff --git a/tests/test_handler.py b/tests/test_handler.py index 0ac201b..d0dfd34 100644 --- a/tests/test_handler.py +++ b/tests/test_handler.py @@ -41,10 +41,11 @@ class TestListener(unittest.TestCase): def setUp(self): self.config = ApiConfiguration( - application = 'test_appname', - environment = 'test_environment', - api_key = 'test_apikey', - api_url = 'test_apiurl') + application='test_appname', + environment='test_environment', + api_key='test_apikey', + api_url='test_apiurl', + ) # don't print warnings on http crashes, so mute stackify logger logging.getLogger('stackify').propagate = False @@ -110,6 +111,5 @@ def test_send_group_crash(self, send_log_group, logmsggroup, logmsg): self.assertEqual(send_log_group.call_count, 1) -if __name__=='__main__': +if __name__ == '__main__': unittest.main() - diff --git a/tests/test_http.py b/tests/test_http.py index 225e8c9..b1d8217 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -11,14 +11,16 @@ from stackify.log import LogMsgGroup from stackify.application import ApiConfiguration -from stackify import READ_TIMEOUT +from stackify.constants import READ_TIMEOUT old_retry = retrying.retry + def fake_retry_decorator(retries): def fake_retry(*args, **kwargs): - kwargs['wait_exponential_max'] = 0 # no delay between retries + kwargs['wait_exponential_max'] = 0 # no delay between retries kwargs['stop_max_attempt_number'] = retries + def inner(func): return old_retry(*args, **kwargs)(func) return inner @@ -43,10 +45,11 @@ def tearDownClass(cls): def setUp(self): self.config = ApiConfiguration( - application = 'test_appname', - environment = 'test_environment', - api_key = 'test_apikey', - api_url = 'test_apiurl') + application='test_appname', + environment='test_environment', + api_key='test_apikey', + api_url='test_apiurl', + ) self.client = stackify.http.HTTPClient(self.config) @@ -54,14 +57,16 @@ def test_logger_no_config(self): '''GZIP encoder works''' correct = list(b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\xf3H\xcd\xc9\xc9\xd7Q(\xcf/\xcaIQ\x04\x00\xe6\xc6\xe6\xeb\r\x00\x00\x00') gzipped = list(stackify.http.gzip_compress('Hello, world!')) - gzipped[4:8] = b'\x00\x00\x00\x00' # blank the mtime + gzipped[4:8] = b'\x00\x00\x00\x00' # blank the mtime self.assertEqual(gzipped, correct) def test_identify_retrying(self): '''HTTP identify should retry''' client = self.client - class CustomException(Exception): pass + class CustomException(Exception): + pass + crash = Mock(side_effect=CustomException) with patch.object(client, 'POST', crash): @@ -97,7 +102,9 @@ def test_send_log_group_retrying(self): '''HTTP sending log groups should retry''' client = self.client - class CustomException(Exception): pass + class CustomException(Exception): + pass + crash = Mock(side_effect=CustomException) group = LogMsgGroup([]) @@ -128,7 +135,6 @@ def test_send_log_group(self): self.assertEqual(group.AppNameID, client.app_name_id) self.assertEqual(group.ServerName, client.device_alias) - @patch('requests.post') def test_post_arguments(self, post): '''HTTP post has correct headers''' @@ -145,9 +151,9 @@ def test_post_arguments(self, post): self.assertTrue(post.called) args, kwargs = post.call_args - self.assertEquals(kwargs['headers'], headers) - self.assertEquals(kwargs['timeout'], READ_TIMEOUT) - self.assertEquals(kwargs['data'], payload.toJSON()) + self.assertEqual(kwargs['headers'], headers) + self.assertEqual(kwargs['timeout'], READ_TIMEOUT) + self.assertEqual(kwargs['data'], payload.toJSON()) @patch('requests.post') def test_post_gzip(self, post): @@ -162,10 +168,9 @@ def test_post_gzip(self, post): self.assertTrue(post.called) args, kwargs = post.call_args - self.assertEquals(kwargs['headers']['Content-Encoding'], 'gzip') - self.assertEquals(kwargs['data'], '1_gzipped') + self.assertEqual(kwargs['headers']['Content-Encoding'], 'gzip') + self.assertEqual(kwargs['data'], '1_gzipped') -if __name__=='__main__': +if __name__ == '__main__': unittest.main() - diff --git a/tests/test_init.py b/tests/test_init.py index 3a24a8a..bccee01 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -3,12 +3,9 @@ """ import unittest -from mock import patch, Mock +from mock import patch from .bases import ClearEnvTest -import os -import atexit - import stackify import logging @@ -21,10 +18,10 @@ class TestInit(ClearEnvTest): def setUp(self): super(TestInit, self).setUp() self.config = stackify.ApiConfiguration( - application = 'test_appname', - environment = 'test_environment', - api_key = 'test_apikey', - api_url = 'test_apiurl') + application='test_appname', + environment='test_environment', + api_key='test_apikey', + api_url='test_apiurl') self.loggers = [] def tearDown(self): @@ -85,11 +82,11 @@ def test_get_logger_defaults(self): config = handler.listener.http.api_config self.assertEqual(logger.name, 'tests.test_init') - self.assertEqual(config.api_url, stackify.API_URL) - self.assertEqual(handler.listener.max_batch, stackify.MAX_BATCH) - self.assertEqual(handler.queue.maxsize, stackify.QUEUE_SIZE) + self.assertEqual(config.api_url, stackify.constants.API_URL) + self.assertEqual(handler.listener.max_batch, stackify.constants.MAX_BATCH) + self.assertEqual(handler.queue.maxsize, stackify.constants.QUEUE_SIZE) # nose will goof with the following assert - #self.assertEqual(logger.getEffectiveLevel(), logging.WARNING) + # self.assertEqual(logger.getEffectiveLevel(), logging.WARNING) def test_get_logger_reuse(self): '''Grabbing a logger twice results in the same logger''' @@ -112,6 +109,5 @@ def test_get_handlers(self): self.assertEqual(logger.handlers, stackify.getHandlers(logger)) -if __name__=='__main__': +if __name__ == '__main__': unittest.main() - diff --git a/tests/test_log.py b/tests/test_log.py index 32f20a6..2d28b37 100644 --- a/tests/test_log.py +++ b/tests/test_log.py @@ -3,17 +3,12 @@ """ import unittest -from mock import patch, Mock import json import sys -import stackify.log - from stackify.log import LogMsg import logging -import json import time -#logging.LogRecord('name','level','pathname','lineno','msg','args','exc_info','func') class TestLogPopulate(unittest.TestCase): @@ -23,9 +18,8 @@ class TestLogPopulate(unittest.TestCase): def test_record_to_error(self): '''LogMsgs can load logger records''' - record = logging.LogRecord('name',logging.WARNING,'pathname',32, - 'message %s',('here'),(),'func') - record.my_extra = [1,2,3] + record = logging.LogRecord('name', logging.WARNING, 'pathname', 32, 'message %s', ('here'), (), 'func') + record.my_extra = [1, 2, 3] msg = LogMsg() msg.from_record(record) @@ -36,7 +30,7 @@ def test_record_to_error(self): self.assertEqual(msg.Th, 'MainThread') self.assertEqual(msg.Msg, 'message here') self.assertTrue(msg.EpochMs <= curr_ms) - self.assertEqual(json.loads(msg.data), {'my_extra':[1,2,3]}) + self.assertEqual(json.loads(msg.data), {'my_extra': [1, 2, 3]}) def test_record_exception(self): '''LogMsgs can parse exception information''' @@ -46,9 +40,8 @@ def __str__(self): try: raise CustomException() - except: - record = logging.LogRecord('my exception',logging.WARNING,'somepath',12, - 'a thing happened',(),sys.exc_info()) + except Exception: + record = logging.LogRecord('my exception', logging.WARNING, 'somepath', 12, 'a thing happened', (), sys.exc_info()) msg = LogMsg() msg.from_record(record) @@ -61,6 +54,5 @@ def __str__(self): self.assertEqual(msg.Ex.Error.SourceMethod, 'test_record_exception') -if __name__=='__main__': +if __name__ == '__main__': unittest.main() - From 0e3b38f559cf9d7f40ac895769c4150061703f80 Mon Sep 17 00:00:00 2001 From: Elpedio Adoptante Jr Date: Mon, 4 Mar 2019 20:27:55 +0800 Subject: [PATCH 03/44] set post interval to 30 secs --- stackify/__init__.py | 1 - stackify/constants.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/stackify/__init__.py b/stackify/__init__.py index 0e1a298..72fb7a7 100644 --- a/stackify/__init__.py +++ b/stackify/__init__.py @@ -1,7 +1,6 @@ """ Stackify Python API """ -__all__ = ("VERSION") __version__ = '0.0.1' import logging diff --git a/stackify/constants.py b/stackify/constants.py index ddb2d36..da30e56 100644 --- a/stackify/constants.py +++ b/stackify/constants.py @@ -4,9 +4,8 @@ API_URL = 'https://api.stackify.com' IDENTIFY_URL = '/Metrics/IdentifyApp' LOG_SAVE_URL = '/Log/Save' -API_REQUEST_INTERVAL_IN_SEC = 5 +API_REQUEST_INTERVAL_IN_SEC = 30 -MIN_BATCH = 10 MAX_BATCH = 100 QUEUE_SIZE = 1000 READ_TIMEOUT = 5000 From e6c65e79e3e709c960efabdded114a05f09fb13a Mon Sep 17 00:00:00 2001 From: Elpedio Adoptante Jr Date: Wed, 13 Mar 2019 17:03:45 +0800 Subject: [PATCH 04/44] update setup values --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index b0c3c9f..dad6000 100755 --- a/setup.py +++ b/setup.py @@ -17,10 +17,10 @@ version = ast.literal_eval(version_re.search(f).group(1)) setup( - name='stackify', + name='stackify-api-python', version=version, - author='Matthew Thompson', - author_email='chameleonator@gmail.com', + author='Stackify', + author_email='support@stackify.com', packages=['stackify'], url='https://github.com/stackify/stackify-api-python', license=open('LICENSE.txt').readline(), From a7764a29ba22e38793ccfc859cd82f9012ca2922 Mon Sep 17 00:00:00 2001 From: Darin Howard Date: Fri, 22 Mar 2019 11:17:29 -0500 Subject: [PATCH 05/44] Updated default logging level to INFO. Updated license. Adding environment to logging post data. Updating readme instructions. --- LICENSE.md | 62 ++++++++++++++++++++++++++++++++++++++ LICENSE.txt | 1 - README.md | 70 +++++++++++-------------------------------- setup.cfg | 2 +- setup.py | 2 -- stackify/constants.py | 2 +- stackify/http.py | 2 +- 7 files changed, 83 insertions(+), 58 deletions(-) create mode 100644 LICENSE.md delete mode 100644 LICENSE.txt diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..b10266c --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,62 @@ +Stackify Python API LICENSE +--- + +Certain inventions disclosed in this file may be claimed within patents owned or patent applications filed by +Stackify, LLC (“Stackify”) or third parties. + +Stackify grants you a revocable, non-exclusive, non-transferable, limited license to download, install and use this +Stackify Python API package (“Application”) strictly in accordance with the terms of this Agreement and the terms found +at https://stackify.com/terms-conditions. + +You agree not to, and you will not permit others to: license, sell, rent, lease, assign, distribute, transmit, host, +outsource, disclose or otherwise commercially exploit the Application or make the Application available to any third +party; modify, make derivative works of, disassemble, decrypt, reverse compile or reverse engineer any part of the +Application; or remove, alter or obscure any proprietary notice (including any notice of copyright or trademark) of +Stackify or its affiliates, partners, suppliers or the licensors of the Application. + +You may install, execute, and distribute these files and their contents only in conjunction with your direct use of +Stackify’s services. These files and their contents shall not be used in conjunction with any other product or +software, including but not limited to those that may compete with any Stackify product, feature, or software. As a +condition to the foregoing grant, you must provide this notice along with each copy you distribute and you must not +remove, alter, or obscure this notice. + +The Application, including without limitation all copyrights, patents, trademarks, trade secrets and other intellectual +property rights are, and shall remain, the sole and exclusive property of Stackify. Any feedback, comments, ideas, +code, pull requests, improvements or suggestions (collectively, "Suggestions") provided by you to Stackify with respect +to the Application shall remain the sole and exclusive property of Stackify. Stackify shall be free to use, copy, +modify, publish, or redistribute the Suggestions for any purpose and in any way without any credit or any compensation +to you. + +Stackify reserves the right to modify, suspend or discontinue, temporarily or permanently, the Application or any +service to which it connects, with or without notice and without liability to you. + +Stackify may from time to time provide enhancements or improvements to the features/functionality of the Application, +which may include patches, bug fixes, updates, upgrades and other modifications ("Updates"). Updates may modify or +delete certain features and/or functionalities of the Application. You agree that Stackify has no obligation to (i) +provide any Updates, or (ii) continue to provide or enable any particular features and/or functionalities of the +Application to you. You further agree that all Updates will be (i) deemed to constitute an integral part of the +Application, and (ii) subject to the terms and conditions of this Agreement. + +All other use, reproduction, modification, distribution, or other exploitation of these files is strictly prohibited, +except as may be set forth in a separate written license agreement between you and Stackify. The terms of any such +license agreement will control over this notice. The license stated above will be automatically terminated and +revoked if you exceed its scope or violate any of the terms of this notice. + +Upon termination of this Agreement, you shall cease all use of the Application and delete all copies of the +Application from your mobile device or from your computer. + +Termination of this Agreement will not limit any of Stackify LLC's rights or remedies at law or in equity in case of +breach by you (during the term of this Agreement) of any of your obligations under the present Agreement. + +You agree to indemnify and hold Stackify LLC and its parents, subsidiaries, affiliates, officers, employees, agents, +partners and licensors (if any) harmless from any claim or demand, including reasonable attorneys' fees, due to or +arising out of your: (a) use of the Application; (b) violation of this Agreement or any law or regulation; or (c) +violation of any right of a third party. + +Unless otherwise expressly agreed by Stackify in a separate written license agreement, these files are provided +AS IS, WITHOUT WARRANTY OF ANY KIND, including without any implied warranties of MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE, TITLE, or NON-INFRINGEMENT. As a condition to your use of these files, you are solely responsible +for such use. Stackify will have no liability to you for direct, indirect, consequential, incidental, special, or +punitive damages or for lost profits or data. + + diff --git a/LICENSE.txt b/LICENSE.txt deleted file mode 100644 index 1333ed7..0000000 --- a/LICENSE.txt +++ /dev/null @@ -1 +0,0 @@ -TODO diff --git a/README.md b/README.md index d30414e..1fe847b 100644 --- a/README.md +++ b/README.md @@ -1,65 +1,46 @@ Stackify API for Python ======================= -[Stackify](https://stackify.com) support for Python programs. - -```python -import stackify - -logger = stackify.getLogger() - -try: - "Make it so, #" + 1 -except: - logger.exception("Can't add strings and numbers") -``` - ## Installation -stackify-python can be installed through pip: -```bash -$ pip install -U stackify -``` -You can also check out the repository and install with setuptools: +**stackify-python-api** can be installed through pip: ```bash -$ ./setup.py install +$ pip install stackify-api-python ``` ## Configuration -Your Stackify setup information can be provided via environment variables. For example: -```bash -export STACKIFY_APPLICATION=MyApp -export STACKIFY_ENVIRONMENT=Dev -export STACKIFY_API_KEY=****** -``` -These options can also be provided in your code: + +#### Standard API ```python -# Standard API import stackify - -logger = stackify.getLogger(application="MyApp", environment="Dev", api_key=******) +logger = stackify.getLogger(application="Python Application", environment="Production", api_key="***") logger.warning('Something happened') ``` +#### Python Logging Integration + ```python -# Python Logging Integration import logging import stackify - -# your existing logging logger = logging.getLogger() -... - -stackify_handler = stackify.StackifyHandler(application="MyApp", environment="Dev", api_key=******) +stackify_handler = stackify.StackifyHandler(application="Python Application", environment="Production", api_key="***") logger.addHandler(stackify_handler) - logger.warning('Something happened') ``` +#### Environment Settings + +```bash +export STACKIFY_APPLICATION=Python Application +export STACKIFY_ENVIRONMENT=Production +export STACKIFY_API_KEY=****** +``` + + ## Usage -stackify-python handles uploads in batches of 100 messages at a time on another thread. +**stackify-python-api** handles uploads in batches of 100 messages at a time on another thread. When your program exits, it will shut the thread down and upload the remaining messages. Stackify can store extra data along with your log message: @@ -74,7 +55,6 @@ except ValueError: You can also name your logger instead of using the automatically generated one: ```python import stackify - logger = stackify.getLogger('mymodule.myfile') ``` @@ -84,7 +64,6 @@ This library has an internal logger it uses for debugging and messaging. For example, if you want to enable debug messages: ```python import logging - logging.getLogger('stackify').setLevel(logging.DEBUG) ``` @@ -96,16 +75,3 @@ import stackify logger = stackify.getLogger(basic_config=False) ``` - -## Testing -Run the test suite with setuptools: -```bash -$ ./setup.py test -``` - -You can obtain a coverage report with nose: -```bash -$ ./setup nosetests --with-coverage --cover-package=stackify -``` -You might need to install the `nose` and `coverage` packages. - diff --git a/setup.cfg b/setup.cfg index 4878226..51d679c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,7 +9,7 @@ exclude = venv*, setup.cfg, README.md, - LICENSE.txt, + LICENSE.md, requirements.txt, diff --git a/setup.py b/setup.py index dad6000..cd783c8 100755 --- a/setup.py +++ b/setup.py @@ -23,10 +23,8 @@ author_email='support@stackify.com', packages=['stackify'], url='https://github.com/stackify/stackify-api-python', - license=open('LICENSE.txt').readline(), description='Stackify API for Python', long_description=read_md('README.md'), - download_url='https://github.com/stackify/stackify-api-python/tarball/0.0.1', keywords=['logging', 'stackify', 'exception'], classifiers=["Programming Language :: Python"], install_requires=[ diff --git a/stackify/constants.py b/stackify/constants.py index da30e56..a2c0dc5 100644 --- a/stackify/constants.py +++ b/stackify/constants.py @@ -18,4 +18,4 @@ logging.DEBUG: 'DEBUG', logging.NOTSET: 'NOTSET' } -DEFAULT_LEVEL = logging.ERROR +DEFAULT_LEVEL = logging.INFO diff --git a/stackify/http.py b/stackify/http.py index ee465b3..33ee2a5 100644 --- a/stackify/http.py +++ b/stackify/http.py @@ -82,9 +82,9 @@ def identify_application(self): @retrying.retry(wait_exponential_multiplier=1000, stop_max_delay=10000) def send_log_group(self, group): + group.Env = self.environment_detail.configuredEnvironmentName group.CDID = self.device_id group.CDAppID = self.device_app_id group.AppNameID = self.app_name_id - group.ServerName = self.device_alias group.ServerName = group.ServerName or self.environment_detail.deviceName self.POST(LOG_SAVE_URL, group, True) From 0be7d42dc628393fe1c2e3bbad13ff72d06128d3 Mon Sep 17 00:00:00 2001 From: Darin Howard Date: Fri, 22 Mar 2019 11:18:00 -0500 Subject: [PATCH 06/44] Updating to version 1.0.0 --- stackify/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackify/__init__.py b/stackify/__init__.py index 72fb7a7..cccc056 100644 --- a/stackify/__init__.py +++ b/stackify/__init__.py @@ -1,7 +1,7 @@ """ Stackify Python API """ -__version__ = '0.0.1' +__version__ = '1.0.0' import logging import inspect From c6bcfb3fca03f7ba77288abffe6942657e4e1588 Mon Sep 17 00:00:00 2001 From: Darin Howard Date: Fri, 22 Mar 2019 11:23:07 -0500 Subject: [PATCH 07/44] Updating to version 1.0.1 --- setup.py | 6 +++++- stackify/__init__.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index cd783c8..b3e88f1 100755 --- a/setup.py +++ b/setup.py @@ -3,6 +3,9 @@ import re import ast +with open("README.md", "r") as fh: + long_description = fh.read() + try: from pypandoc import convert read_md = lambda f: convert(f, 'rst') # noqa @@ -24,7 +27,8 @@ packages=['stackify'], url='https://github.com/stackify/stackify-api-python', description='Stackify API for Python', - long_description=read_md('README.md'), + long_description=long_description, + long_description_content_type="text/markdown", keywords=['logging', 'stackify', 'exception'], classifiers=["Programming Language :: Python"], install_requires=[ diff --git a/stackify/__init__.py b/stackify/__init__.py index cccc056..4151a91 100644 --- a/stackify/__init__.py +++ b/stackify/__init__.py @@ -1,7 +1,7 @@ """ Stackify Python API """ -__version__ = '1.0.0' +__version__ = '1.0.1' import logging import inspect From 1e8280dadccd98d6ce2568411d383db79b971fdc Mon Sep 17 00:00:00 2001 From: Elpedio Adoptante Jr Date: Wed, 3 Apr 2019 23:16:43 +0800 Subject: [PATCH 08/44] Update for trace linking --- stackify/log.py | 5 +++++ tests/test_http.py | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/stackify/log.py b/stackify/log.py index 21e2e0b..bf472a1 100644 --- a/stackify/log.py +++ b/stackify/log.py @@ -10,10 +10,13 @@ # the "message" attribute is saved on the record object by a Formatter RECORD_VARS.add('message') +RECORD_VARS.add('trans_id') +RECORD_VARS.add('log_id') class LogMsg(JSONObject): def __init__(self): + self.ID = None self.Msg = None self.data = None self.Ex = None # a StackifyError object @@ -25,10 +28,12 @@ def __init__(self): self.SrcLine = None def from_record(self, record): + self.ID = hasattr(record, 'log_id') and record.log_id or None self.Msg = record.getMessage() self.Th = record.threadName or record.thread self.EpochMs = int(record.created * 1000) self.Level = record.levelname + self.TransID = hasattr(record, 'trans_id') and record.trans_id or None self.SrcMethod = record.funcName self.SrcLine = record.lineno diff --git a/tests/test_http.py b/tests/test_http.py index b1d8217..c109375 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -133,7 +133,6 @@ def test_send_log_group(self): self.assertEqual(group.CDID, client.device_id) self.assertEqual(group.CDAppID, client.device_app_id) self.assertEqual(group.AppNameID, client.app_name_id) - self.assertEqual(group.ServerName, client.device_alias) @patch('requests.post') def test_post_arguments(self, post): From e5d123b557cb9ff90db66a9c42cb03b3e826c6d2 Mon Sep 17 00:00:00 2001 From: Elpedio Adoptante Jr Date: Wed, 20 Mar 2019 17:05:52 +0800 Subject: [PATCH 09/44] Fix installation process and move testing to dev docs --- README.md | 4 ++++ docs/Developer.md | 11 +++++++++++ 2 files changed, 15 insertions(+) create mode 100644 docs/Developer.md diff --git a/README.md b/README.md index 1fe847b..aefb3d9 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,10 @@ Stackify API for Python ======================= ## Installation +stackify-python can be installed through pip: +```bash +$ pip install -U stackify-api-python +``` **stackify-python-api** can be installed through pip: ```bash diff --git a/docs/Developer.md b/docs/Developer.md new file mode 100644 index 0000000..cbc6905 --- /dev/null +++ b/docs/Developer.md @@ -0,0 +1,11 @@ +## Testing +Run the test suite with setuptools: +```bash +$ ./setup.py test +``` + +You can obtain a coverage report with nose: +```bash +$ ./setup nosetests --with-coverage --cover-package=stackify +``` +You might need to install the `nose` and `coverage` packages. From 755db354f31eeab8e9e6e1db1f6fd6342ef2465f Mon Sep 17 00:00:00 2001 From: Elpedio Adoptante Jr Date: Fri, 22 Mar 2019 17:33:45 +0800 Subject: [PATCH 10/44] safe to dict --- stackify/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackify/log.py b/stackify/log.py index 21e2e0b..56c3f0e 100644 --- a/stackify/log.py +++ b/stackify/log.py @@ -37,7 +37,7 @@ def from_record(self, record): if k not in RECORD_VARS} if data: - self.data = json.dumps(data, default=lambda x: x.__dict__) + self.data = json.dumps(data, default=lambda x: hasattr(x, '__dict__') and x.__dict__ or x.__str__()) if record.exc_info: self.Ex = StackifyError() From bd6c03e8f9d59403da4f4959d550e95e97c70e41 Mon Sep 17 00:00:00 2001 From: Elpedio Adoptante Jr Date: Fri, 22 Mar 2019 17:45:00 +0800 Subject: [PATCH 11/44] add django logging integration --- README.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/README.md b/README.md index aefb3d9..231158e 100644 --- a/README.md +++ b/README.md @@ -79,3 +79,45 @@ import stackify logger = stackify.getLogger(basic_config=False) ``` + +## Django Logging Integration + +You can also use your existing django logging and just append stackify logging handler + +```python +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'handlers': { + 'file': { + 'level': 'DEBUG', + 'class': 'logging.FileHandler', + 'filename': 'debug.log', + }, + 'stackify': { + 'level': 'DEBUG', + 'class': 'stackify.StackifyHandler', + 'application': 'MyApp', + 'environment': 'Dev', + 'api_key': '******', + } + }, + 'loggers': { + 'django': { + 'handlers': ['file', 'stackify'], + 'level': 'DEBUG', + 'propagate': True, + }, + }, +} +``` + +Usage +```python +import logging + +logger = logging.getLogger('django') + + +logger.warning('Something happened') +``` From 8f4c531e9b2ae46feded1194fb7419ee64f313cc Mon Sep 17 00:00:00 2001 From: Darin Howard Date: Mon, 8 Apr 2019 08:52:51 -0500 Subject: [PATCH 12/44] updating version to 1.0.2 --- stackify/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackify/__init__.py b/stackify/__init__.py index 4151a91..e469cf8 100644 --- a/stackify/__init__.py +++ b/stackify/__init__.py @@ -1,7 +1,7 @@ """ Stackify Python API """ -__version__ = '1.0.1' +__version__ = '1.0.2' import logging import inspect From b86cbd2151ab163674ea646b98fe37f3e522126e Mon Sep 17 00:00:00 2001 From: jaygel Date: Mon, 12 Aug 2019 23:40:10 +0800 Subject: [PATCH 13/44] Clean up - remove redundancy and clean up code (#5) * Clean up - remove redundancy and explicitly use stackify for internal logging * Add note when using stackify api python logging * disable stackify logging by default * update internal logging instructions * asign internal logging to a variable --- README.md | 6 ++++-- stackify/__init__.py | 12 ++++++------ stackify/application.py | 10 +++++----- stackify/handler.py | 21 ++++++++++----------- stackify/http.py | 14 ++++++++------ 5 files changed, 33 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 231158e..ba7de4c 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ logger.warning('Something happened') ```python import logging import stackify -logger = logging.getLogger() +logger = logging.getLogger(__name__) stackify_handler = stackify.StackifyHandler(application="Python Application", environment="Production", api_key="***") logger.addHandler(stackify_handler) logger.warning('Something happened') @@ -68,7 +68,9 @@ This library has an internal logger it uses for debugging and messaging. For example, if you want to enable debug messages: ```python import logging -logging.getLogger('stackify').setLevel(logging.DEBUG) +logger = logging.getLogger('stackify') +logger.setLevel(logging.DEBUG) +logger.addHandler(logging.FileHandler('stackify.log')) # or any handler you want ``` By default, it will enable the default logging settings via `logging.basicConfig()` diff --git a/stackify/__init__.py b/stackify/__init__.py index e469cf8..9ad5d55 100644 --- a/stackify/__init__.py +++ b/stackify/__init__.py @@ -17,7 +17,10 @@ def emit(self, record): pass -logging.getLogger(__name__).addHandler(NullHandler()) +internal_logger = logging.getLogger(__name__) +internal_logger.addHandler(NullHandler()) +internal_logger.propagate = False +internal_logger.setLevel(DEFAULT_LEVEL) def getLogger(name=None, auto_shutdown=True, basic_config=True, **kwargs): @@ -52,8 +55,7 @@ def getLogger(name=None, auto_shutdown=True, basic_config=True, **kwargs): logger = logging.getLogger(name) - if not [isinstance(x, StackifyHandler) for x in logger.handlers]: - internal_logger = logging.getLogger(__name__) + if not any([isinstance(x, StackifyHandler) for x in logger.handlers]): internal_logger.debug('Creating handler for logger %s', name) if auto_shutdown: @@ -66,8 +68,6 @@ def getLogger(name=None, auto_shutdown=True, basic_config=True, **kwargs): handler = StackifyHandler(ensure_at_exit=not auto_shutdown, **kwargs) logger.addHandler(handler) - handler.listener.start() - return logger @@ -77,7 +77,7 @@ def stopLogging(logger): Shut down the StackifyHandler on a given logger. This will block and wait for the queue to finish uploading. ''' - internal_logger = logging.getLogger(__name__) + internal_logger.debug('Shutting down all handlers') for handler in getHandlers(logger): handler.listener.stop() diff --git a/stackify/application.py b/stackify/application.py index 2201df9..b3047fe 100644 --- a/stackify/application.py +++ b/stackify/application.py @@ -36,8 +36,8 @@ def arg_or_env(name, args, default=None): def get_configuration(**kwargs): - return ApiConfiguration( - application=arg_or_env('application', kwargs), - environment=arg_or_env('environment', kwargs), - api_key=arg_or_env('api_key', kwargs), - api_url=arg_or_env('api_url', kwargs, API_URL)) + return ApiConfiguration( + application=arg_or_env('application', kwargs), + environment=arg_or_env('environment', kwargs), + api_key=arg_or_env('api_key', kwargs), + api_url=arg_or_env('api_url', kwargs, API_URL)) diff --git a/stackify/handler.py b/stackify/handler.py index 9a24b9b..d911d54 100644 --- a/stackify/handler.py +++ b/stackify/handler.py @@ -20,6 +20,9 @@ from stackify.timer import RepeatedTimer +internal_logger = logging.getLogger(__name__) + + class StackifyHandler(QueueHandler): ''' A handler class to format and queue log messages for later @@ -39,6 +42,7 @@ def __init__(self, queue_=None, listener=None, ensure_at_exit=True, **kwargs): self.listener.start() if ensure_at_exit: + internal_logger.debug('Registering atexit callback') atexit.register(self.listener.stop) def enqueue(self, record): @@ -48,8 +52,7 @@ def enqueue(self, record): try: self.queue.put_nowait(record) except queue.Full: - logger = logging.getLogger(__name__) - logger.warning('StackifyHandler queue is full, evicting oldest record') + internal_logger.warning('StackifyHandler queue is full, evicting oldest record') self.queue.get_nowait() self.queue.put_nowait(record) @@ -74,8 +77,7 @@ def __init__(self, queue_, max_batch=MAX_BATCH, config=None, **kwargs): def handle(self, record): if not self.http.identified: - logger = logging.getLogger(__name__) - logger.debug('Identifying application') + internal_logger.debug('Identifying application') self.http.identify_application() msg = LogMsg() @@ -93,13 +95,11 @@ def send_group(self): try: self.http.send_log_group(group) except Exception: - logger = logging.getLogger(__name__) - logger.exception('Could not send {} log messages, discarding'.format(len(self.messages))) + internal_logger.exception('Could not send {} log messages, discarding'.format(len(self.messages))) del self.messages[:] def start(self): - logger = logging.getLogger(__name__) - logger.debug('Starting up listener') + internal_logger.debug('Starting up listener') if not self._started: super(StackifyListener, self).start() @@ -107,8 +107,7 @@ def start(self): self._started = True def stop(self): - logger = logging.getLogger(__name__) - logger.debug('Shutting down listener') + internal_logger.debug('Shutting down listener') if self._started: super(StackifyListener, self).stop() @@ -117,5 +116,5 @@ def stop(self): # send any remaining messages if self.messages: - logger.debug('{} messages left on shutdown, uploading'.format(len(self.messages))) + internal_logger.debug('{} messages left on shutdown, uploading'.format(len(self.messages))) self.send_group() diff --git a/stackify/http.py b/stackify/http.py index 33ee2a5..dfeb098 100644 --- a/stackify/http.py +++ b/stackify/http.py @@ -17,6 +17,9 @@ from stackify.constants import READ_TIMEOUT +internal_logger = logging.getLogger(__name__) + + def gzip_compress(data): if hasattr(gzip, 'compress'): return gzip.compress(bytes(data, 'utf-8')) # python 3 @@ -41,8 +44,7 @@ def __init__(self, api_config): def POST(self, url, json_object, use_gzip=False): request_url = self.api_config.api_url + url - logger = logging.getLogger(__name__) - logger.debug('Request URL: {}'.format(request_url)) + internal_logger.debug('Request URL: {}'.format(request_url)) headers = { 'Content-Type': 'application/json', @@ -52,7 +54,7 @@ def POST(self, url, json_object, use_gzip=False): try: payload_data = json_object.toJSON() - logger.debug('POST data: {}'.format(payload_data)) + internal_logger.debug('POST data: {}'.format(payload_data)) if use_gzip: headers['Content-Encoding'] = 'gzip' @@ -62,13 +64,13 @@ def POST(self, url, json_object, use_gzip=False): data=payload_data, headers=headers, timeout=READ_TIMEOUT) - logger.debug('Response: {}'.format(response.text)) + internal_logger.debug('Response: {}'.format(response.text)) return response.json() except requests.exceptions.RequestException: - logger.exception('HTTP exception') + internal_logger.exception('HTTP exception') except ValueError: # could not read json response - logger.exception('Cannot decode JSON response') + internal_logger.exception('Cannot decode JSON response') @retrying.retry(wait_exponential_multiplier=1000, stop_max_delay=10000) def identify_application(self): From 6f368b347f1d8c906b73f4d28aaf7d8d4c4a7cfd Mon Sep 17 00:00:00 2001 From: Darin Howard Date: Mon, 12 Aug 2019 10:42:08 -0500 Subject: [PATCH 14/44] Updating to version 1.0.3 --- stackify/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackify/__init__.py b/stackify/__init__.py index 9ad5d55..2a70741 100644 --- a/stackify/__init__.py +++ b/stackify/__init__.py @@ -1,7 +1,7 @@ """ Stackify Python API """ -__version__ = '1.0.2' +__version__ = '1.0.3' import logging import inspect From 2c1127ed7f5aad0afc234c6d9fdf2d82bea07d50 Mon Sep 17 00:00:00 2001 From: jaygel Date: Tue, 10 Sep 2019 23:19:08 +0800 Subject: [PATCH 15/44] Added Unix Socket Domain Transport (#6) Added Unix Socket Domain Transport --- .gitignore | 3 + requirements.txt | 2 + setup.py | 4 +- stackify/__init__.py | 1 - stackify/application.py | 43 - stackify/constants.py | 17 + stackify/handler.py | 28 +- stackify/protos/__init__.py | 0 stackify/protos/stackify_agent_pb2.py | 1127 +++++++++++++++++ stackify/transport/__init__.py | 116 ++ stackify/transport/agent/__init__.py | 1 + stackify/transport/agent/agent_socket.py | 45 + stackify/transport/agent/message.py | 125 ++ stackify/transport/application.py | 58 + stackify/transport/default/__init__.py | 1 + stackify/{ => transport/default}/error.py | 2 +- stackify/{ => transport/default}/formats.py | 0 stackify/{ => transport/default}/http.py | 22 +- stackify/{ => transport/default}/log.py | 15 +- stackify/utils.py | 15 + tests/bases.py | 16 + tests/test_handler.py | 32 +- tests/test_init.py | 8 +- tests/transport/__init__.py | 0 tests/transport/agent/__init__.py | 0 tests/transport/agent/test_agent_socket.py | 64 + tests/transport/agent/test_message.py | 135 ++ tests/transport/default/__init__.py | 0 tests/{ => transport/default}/test_formats.py | 2 +- tests/{ => transport/default}/test_http.py | 36 +- tests/{ => transport/default}/test_log.py | 2 +- tests/{ => transport}/test_application.py | 58 +- tests/transport/test_init.py | 163 +++ 33 files changed, 2011 insertions(+), 130 deletions(-) delete mode 100644 stackify/application.py create mode 100644 stackify/protos/__init__.py create mode 100644 stackify/protos/stackify_agent_pb2.py create mode 100644 stackify/transport/__init__.py create mode 100644 stackify/transport/agent/__init__.py create mode 100644 stackify/transport/agent/agent_socket.py create mode 100644 stackify/transport/agent/message.py create mode 100644 stackify/transport/application.py create mode 100644 stackify/transport/default/__init__.py rename stackify/{ => transport/default}/error.py (96%) rename stackify/{ => transport/default}/formats.py (100%) rename stackify/{ => transport/default}/http.py (86%) rename stackify/{ => transport/default}/log.py (78%) create mode 100644 stackify/utils.py create mode 100644 tests/transport/__init__.py create mode 100644 tests/transport/agent/__init__.py create mode 100644 tests/transport/agent/test_agent_socket.py create mode 100644 tests/transport/agent/test_message.py create mode 100644 tests/transport/default/__init__.py rename tests/{ => transport/default}/test_formats.py (96%) rename tests/{ => transport/default}/test_http.py (84%) rename tests/{ => transport/default}/test_log.py (97%) rename tests/{ => transport}/test_application.py (60%) create mode 100644 tests/transport/test_init.py diff --git a/.gitignore b/.gitignore index 25344cd..2689372 100644 --- a/.gitignore +++ b/.gitignore @@ -107,3 +107,6 @@ venv.bak/ # Intellij .idea + +# Protobuf +*.proto diff --git a/requirements.txt b/requirements.txt index 3c83c04..911174c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,7 @@ mock==2.0.0 +protobuf==3.9.1 pytest==4.3.0 pytest-cov==2.6.1 requests==2.21.0 +requests-unixsocket==0.2.0 retrying==1.3.3 diff --git a/setup.py b/setup.py index b3e88f1..568f68f 100755 --- a/setup.py +++ b/setup.py @@ -32,8 +32,10 @@ keywords=['logging', 'stackify', 'exception'], classifiers=["Programming Language :: Python"], install_requires=[ + 'protobuf>=3.9.1', 'retrying>=1.2.3', - 'requests>=2.4.1' + 'requests>=2.4.1', + 'requests-unixsocket>=0.2.0' ], test_suite='tests', tests_requires=[ diff --git a/stackify/__init__.py b/stackify/__init__.py index 2a70741..513b7d8 100644 --- a/stackify/__init__.py +++ b/stackify/__init__.py @@ -7,7 +7,6 @@ import inspect import atexit -from stackify.application import ApiConfiguration # noqa from stackify.constants import DEFAULT_LEVEL from stackify.handler import StackifyHandler diff --git a/stackify/application.py b/stackify/application.py deleted file mode 100644 index b3047fe..0000000 --- a/stackify/application.py +++ /dev/null @@ -1,43 +0,0 @@ -import socket -import os - -from stackify.constants import API_URL -from stackify.formats import JSONObject - - -class EnvironmentDetail(JSONObject): - def __init__(self, api_config): - self.deviceName = socket.gethostname() - self.appLocation = os.getcwd() - self.configuredAppName = api_config.application - self.configuredEnvironmentName = api_config.environment - - -class ApiConfiguration: - def __init__(self, api_key, application, environment, api_url=API_URL): - self.api_key = api_key - self.api_url = api_url - self.application = application - self.environment = environment - - -def arg_or_env(name, args, default=None): - env_name = 'STACKIFY_{0}'.format(name.upper()) - try: - value = args.get(name) - if not value: - value = os.environ[env_name] - return value - except KeyError: - if default: - return default - else: - raise NameError('You must specify the keyword argument {0} or environment variable {1}'.format(name, env_name)) - - -def get_configuration(**kwargs): - return ApiConfiguration( - application=arg_or_env('application', kwargs), - environment=arg_or_env('environment', kwargs), - api_key=arg_or_env('api_key', kwargs), - api_url=arg_or_env('api_url', kwargs, API_URL)) diff --git a/stackify/constants.py b/stackify/constants.py index a2c0dc5..1d94e4a 100644 --- a/stackify/constants.py +++ b/stackify/constants.py @@ -4,6 +4,12 @@ API_URL = 'https://api.stackify.com' IDENTIFY_URL = '/Metrics/IdentifyApp' LOG_SAVE_URL = '/Log/Save' + +# using `%2F` instead of `/` as per package documentation +DEFAULT_SOCKET_FILE = '%2Fusr%2Flocal%2Fstackify%2Fstackify.sock' +SOCKET_URL = 'http+unix://' + DEFAULT_SOCKET_FILE +SOCKET_LOG_URL = '/log' + API_REQUEST_INTERVAL_IN_SEC = 30 MAX_BATCH = 100 @@ -19,3 +25,14 @@ logging.NOTSET: 'NOTSET' } DEFAULT_LEVEL = logging.INFO + +# this is used to separate builtin keys from user-specified keys +RECORD_VARS = set(logging.LogRecord('', '', '', '', '', '', '', '').__dict__.keys()) + +# the "message" attribute is saved on the record object by a Formatter +RECORD_VARS.add('message') +RECORD_VARS.add('trans_id') +RECORD_VARS.add('log_id') + +TRANSPORT_TYPE_DEFAULT = 'default' +TRANSPORT_TYPE_AGENT_SOCKET = 'agent_socket' diff --git a/stackify/handler.py b/stackify/handler.py index d911d54..9478f36 100644 --- a/stackify/handler.py +++ b/stackify/handler.py @@ -1,3 +1,4 @@ +import copy import logging import atexit @@ -11,13 +12,11 @@ except ImportError: # pragma: no cover import queue -from stackify.application import get_configuration from stackify.constants import API_REQUEST_INTERVAL_IN_SEC from stackify.constants import MAX_BATCH from stackify.constants import QUEUE_SIZE -from stackify.http import HTTPClient -from stackify.log import LogMsg, LogMsgGroup from stackify.timer import RepeatedTimer +from stackify.transport import Transport internal_logger = logging.getLogger(__name__) @@ -56,6 +55,10 @@ def enqueue(self, record): self.queue.get_nowait() self.queue.put_nowait(record) + def prepare(self, record): + record = copy.copy(record) + return record + class StackifyListener(QueueListener): ''' @@ -65,24 +68,15 @@ class StackifyListener(QueueListener): def __init__(self, queue_, max_batch=MAX_BATCH, config=None, **kwargs): super(StackifyListener, self).__init__(queue_) - if config is None: - config = get_configuration(**kwargs) - self.max_batch = max_batch self.messages = [] - self.http = HTTPClient(config) + self.transport = Transport(config, **kwargs) self.timer = RepeatedTimer(API_REQUEST_INTERVAL_IN_SEC, self.send_group) self._started = False def handle(self, record): - if not self.http.identified: - internal_logger.debug('Identifying application') - self.http.identify_application() - - msg = LogMsg() - msg.from_record(record) - self.messages.append(msg) + self.messages.append(self.transport.create_message(record)) if len(self.messages) >= self.max_batch: self.send_group() @@ -91,9 +85,9 @@ def send_group(self): if not self.messages: return - group = LogMsgGroup(self.messages) + group_message = self.transport.create_group_message(self.messages) try: - self.http.send_log_group(group) + self.transport.send(group_message) except Exception: internal_logger.exception('Could not send {} log messages, discarding'.format(len(self.messages))) del self.messages[:] @@ -103,8 +97,8 @@ def start(self): if not self._started: super(StackifyListener, self).start() - self.timer.start() self._started = True + self.timer.start() def stop(self): internal_logger.debug('Shutting down listener') diff --git a/stackify/protos/__init__.py b/stackify/protos/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/stackify/protos/stackify_agent_pb2.py b/stackify/protos/stackify_agent_pb2.py new file mode 100644 index 0000000..a8bbbc0 --- /dev/null +++ b/stackify/protos/stackify_agent_pb2.py @@ -0,0 +1,1127 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: stackify-agent.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='stackify-agent.proto', + package='stackify', + syntax='proto3', + serialized_pb=_b('\n\x14stackify-agent.proto\x12\x08stackify\"\xba\x14\n\x08LogGroup\x12\x13\n\x0b\x65nvironment\x18\x01 \x01(\t\x12\x13\n\x0bserver_name\x18\x02 \x01(\t\x12\x18\n\x10\x61pplication_name\x18\x03 \x01(\t\x12\x1c\n\x14\x61pplication_location\x18\x04 \x01(\t\x12\x0e\n\x06logger\x18\x05 \x01(\t\x12\x10\n\x08platform\x18\x06 \x01(\t\x12$\n\x04logs\x18\x07 \x03(\x0b\x32\x16.stackify.LogGroup.Log\x12/\n\tcontainer\x18\x08 \x01(\x0b\x32\x1c.stackify.LogGroup.Container\x12\x31\n\nkubernetes\x18\t \x01(\x0b\x32\x1d.stackify.LogGroup.Kubernetes\x1ax\n\tContainer\x12\x10\n\x08image_id\x18\x01 \x01(\t\x12\x18\n\x10image_repository\x18\x02 \x01(\t\x12\x11\n\timage_tag\x18\x03 \x01(\t\x12\x14\n\x0c\x63ontainer_id\x18\x04 \x01(\t\x12\x16\n\x0e\x63ontainer_name\x18\x05 \x01(\t\x1aK\n\nKubernetes\x12\x10\n\x08pod_name\x18\x01 \x01(\t\x12\x15\n\rpod_namespace\x18\x02 \x01(\t\x12\x14\n\x0c\x63luster_name\x18\x03 \x01(\t\x1a\xd8\x10\n\x03Log\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x12\x13\n\x0bthread_name\x18\x03 \x01(\t\x12\x13\n\x0b\x64\x61te_millis\x18\x04 \x01(\x03\x12\r\n\x05level\x18\x05 \x01(\t\x12\x16\n\x0etransaction_id\x18\x06 \x01(\t\x12\x15\n\rsource_method\x18\x07 \x01(\t\x12\x13\n\x0bsource_line\x18\x08 \x01(\x05\x12\n\n\x02id\x18\t \x01(\t\x12\x0c\n\x04tags\x18\n \x03(\t\x12+\n\x05\x65rror\x18\x0b \x01(\x0b\x32\x1c.stackify.LogGroup.Log.Error\x1a\xed\x0e\n\x05\x45rror\x12J\n\x12\x65nvironment_detail\x18\x01 \x01(\x0b\x32..stackify.LogGroup.Log.Error.EnvironmentDetail\x12\x13\n\x0b\x64\x61te_millis\x18\x02 \x01(\x03\x12:\n\nerror_item\x18\x03 \x01(\x0b\x32&.stackify.LogGroup.Log.Error.ErrorItem\x12I\n\x12web_request_detail\x18\x04 \x01(\x0b\x32-.stackify.LogGroup.Log.Error.WebRequestDetail\x12K\n\x10server_variables\x18\x05 \x03(\x0b\x32\x31.stackify.LogGroup.Log.Error.ServerVariablesEntry\x12\x15\n\rcustomer_name\x18\x06 \x01(\t\x12\x10\n\x08username\x18\x07 \x01(\t\x1a\x36\n\x14ServerVariablesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\xaa\x01\n\x11\x45nvironmentDetail\x12\x13\n\x0b\x64\x65vice_name\x18\x01 \x01(\t\x12\x18\n\x10\x61pplication_name\x18\x02 \x01(\t\x12\x1c\n\x14\x61pplication_location\x18\x03 \x01(\t\x12#\n\x1b\x63onfigured_application_name\x18\x04 \x01(\t\x12#\n\x1b\x63onfigured_environment_name\x18\x05 \x01(\t\x1a\x9b\x03\n\tErrorItem\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\x12\n\nerror_type\x18\x02 \x01(\t\x12\x17\n\x0f\x65rror_type_code\x18\x03 \x01(\t\x12>\n\x04\x64\x61ta\x18\x04 \x03(\x0b\x32\x30.stackify.LogGroup.Log.Error.ErrorItem.DataEntry\x12\x15\n\rsource_method\x18\x05 \x01(\t\x12\x45\n\nstacktrace\x18\x06 \x03(\x0b\x32\x31.stackify.LogGroup.Log.Error.ErrorItem.TraceFrame\x12;\n\x0binner_error\x18\x07 \x01(\x0b\x32&.stackify.LogGroup.Log.Error.ErrorItem\x1a+\n\tDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1aH\n\nTraceFrame\x12\x15\n\rcode_filename\x18\x01 \x01(\t\x12\x13\n\x0bline_number\x18\x02 \x01(\x05\x12\x0e\n\x06method\x18\x03 \x01(\t\x1a\x82\x07\n\x10WebRequestDetail\x12\x17\n\x0fuser_ip_address\x18\x01 \x01(\t\x12\x13\n\x0bhttp_method\x18\x02 \x01(\t\x12\x18\n\x10request_protocol\x18\x03 \x01(\t\x12\x13\n\x0brequest_url\x18\x04 \x01(\t\x12\x18\n\x10request_url_root\x18\x05 \x01(\t\x12\x14\n\x0creferral_url\x18\x06 \x01(\t\x12K\n\x07headers\x18\x07 \x03(\x0b\x32:.stackify.LogGroup.Log.Error.WebRequestDetail.HeadersEntry\x12K\n\x07\x63ookies\x18\x08 \x03(\x0b\x32:.stackify.LogGroup.Log.Error.WebRequestDetail.CookiesEntry\x12S\n\x0bquerystring\x18\t \x03(\x0b\x32>.stackify.LogGroup.Log.Error.WebRequestDetail.QuerystringEntry\x12N\n\tpost_data\x18\n \x03(\x0b\x32;.stackify.LogGroup.Log.Error.WebRequestDetail.PostDataEntry\x12T\n\x0csession_data\x18\x0b \x03(\x0b\x32>.stackify.LogGroup.Log.Error.WebRequestDetail.SessionDataEntry\x12\x15\n\rpost_data_raw\x18\x0c \x01(\t\x12\x12\n\nmvc_action\x18\r \x01(\t\x12\x16\n\x0emvc_controller\x18\x0e \x01(\t\x12\x10\n\x08mvc_area\x18\x0f \x01(\t\x1a.\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a.\n\x0c\x43ookiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x32\n\x10QuerystringEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a/\n\rPostDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x32\n\x10SessionDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42.\n\x1d\x63om.stackify.api.common.protoB\rStackifyProtob\x06proto3') +) +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + + + + +_LOGGROUP_CONTAINER = _descriptor.Descriptor( + name='Container', + full_name='stackify.LogGroup.Container', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='image_id', full_name='stackify.LogGroup.Container.image_id', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='image_repository', full_name='stackify.LogGroup.Container.image_repository', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='image_tag', full_name='stackify.LogGroup.Container.image_tag', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='container_id', full_name='stackify.LogGroup.Container.container_id', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='container_name', full_name='stackify.LogGroup.Container.container_name', index=4, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=317, + serialized_end=437, +) + +_LOGGROUP_KUBERNETES = _descriptor.Descriptor( + name='Kubernetes', + full_name='stackify.LogGroup.Kubernetes', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='pod_name', full_name='stackify.LogGroup.Kubernetes.pod_name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='pod_namespace', full_name='stackify.LogGroup.Kubernetes.pod_namespace', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='cluster_name', full_name='stackify.LogGroup.Kubernetes.cluster_name', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=439, + serialized_end=514, +) + +_LOGGROUP_LOG_ERROR_SERVERVARIABLESENTRY = _descriptor.Descriptor( + name='ServerVariablesEntry', + full_name='stackify.LogGroup.Log.Error.ServerVariablesEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='stackify.LogGroup.Log.Error.ServerVariablesEntry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='value', full_name='stackify.LogGroup.Log.Error.ServerVariablesEntry.value', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1111, + serialized_end=1165, +) + +_LOGGROUP_LOG_ERROR_ENVIRONMENTDETAIL = _descriptor.Descriptor( + name='EnvironmentDetail', + full_name='stackify.LogGroup.Log.Error.EnvironmentDetail', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='device_name', full_name='stackify.LogGroup.Log.Error.EnvironmentDetail.device_name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='application_name', full_name='stackify.LogGroup.Log.Error.EnvironmentDetail.application_name', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='application_location', full_name='stackify.LogGroup.Log.Error.EnvironmentDetail.application_location', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='configured_application_name', full_name='stackify.LogGroup.Log.Error.EnvironmentDetail.configured_application_name', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='configured_environment_name', full_name='stackify.LogGroup.Log.Error.EnvironmentDetail.configured_environment_name', index=4, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1168, + serialized_end=1338, +) + +_LOGGROUP_LOG_ERROR_ERRORITEM_DATAENTRY = _descriptor.Descriptor( + name='DataEntry', + full_name='stackify.LogGroup.Log.Error.ErrorItem.DataEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='stackify.LogGroup.Log.Error.ErrorItem.DataEntry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='value', full_name='stackify.LogGroup.Log.Error.ErrorItem.DataEntry.value', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1635, + serialized_end=1678, +) + +_LOGGROUP_LOG_ERROR_ERRORITEM_TRACEFRAME = _descriptor.Descriptor( + name='TraceFrame', + full_name='stackify.LogGroup.Log.Error.ErrorItem.TraceFrame', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='code_filename', full_name='stackify.LogGroup.Log.Error.ErrorItem.TraceFrame.code_filename', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='line_number', full_name='stackify.LogGroup.Log.Error.ErrorItem.TraceFrame.line_number', index=1, + number=2, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='method', full_name='stackify.LogGroup.Log.Error.ErrorItem.TraceFrame.method', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1680, + serialized_end=1752, +) + +_LOGGROUP_LOG_ERROR_ERRORITEM = _descriptor.Descriptor( + name='ErrorItem', + full_name='stackify.LogGroup.Log.Error.ErrorItem', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='message', full_name='stackify.LogGroup.Log.Error.ErrorItem.message', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='error_type', full_name='stackify.LogGroup.Log.Error.ErrorItem.error_type', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='error_type_code', full_name='stackify.LogGroup.Log.Error.ErrorItem.error_type_code', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='data', full_name='stackify.LogGroup.Log.Error.ErrorItem.data', index=3, + number=4, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='source_method', full_name='stackify.LogGroup.Log.Error.ErrorItem.source_method', index=4, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='stacktrace', full_name='stackify.LogGroup.Log.Error.ErrorItem.stacktrace', index=5, + number=6, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='inner_error', full_name='stackify.LogGroup.Log.Error.ErrorItem.inner_error', index=6, + number=7, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[_LOGGROUP_LOG_ERROR_ERRORITEM_DATAENTRY, _LOGGROUP_LOG_ERROR_ERRORITEM_TRACEFRAME, ], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1341, + serialized_end=1752, +) + +_LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL_HEADERSENTRY = _descriptor.Descriptor( + name='HeadersEntry', + full_name='stackify.LogGroup.Log.Error.WebRequestDetail.HeadersEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='stackify.LogGroup.Log.Error.WebRequestDetail.HeadersEntry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='value', full_name='stackify.LogGroup.Log.Error.WebRequestDetail.HeadersEntry.value', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=2406, + serialized_end=2452, +) + +_LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL_COOKIESENTRY = _descriptor.Descriptor( + name='CookiesEntry', + full_name='stackify.LogGroup.Log.Error.WebRequestDetail.CookiesEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='stackify.LogGroup.Log.Error.WebRequestDetail.CookiesEntry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='value', full_name='stackify.LogGroup.Log.Error.WebRequestDetail.CookiesEntry.value', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=2454, + serialized_end=2500, +) + +_LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL_QUERYSTRINGENTRY = _descriptor.Descriptor( + name='QuerystringEntry', + full_name='stackify.LogGroup.Log.Error.WebRequestDetail.QuerystringEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='stackify.LogGroup.Log.Error.WebRequestDetail.QuerystringEntry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='value', full_name='stackify.LogGroup.Log.Error.WebRequestDetail.QuerystringEntry.value', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=2502, + serialized_end=2552, +) + +_LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL_POSTDATAENTRY = _descriptor.Descriptor( + name='PostDataEntry', + full_name='stackify.LogGroup.Log.Error.WebRequestDetail.PostDataEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='stackify.LogGroup.Log.Error.WebRequestDetail.PostDataEntry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='value', full_name='stackify.LogGroup.Log.Error.WebRequestDetail.PostDataEntry.value', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=2554, + serialized_end=2601, +) + +_LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL_SESSIONDATAENTRY = _descriptor.Descriptor( + name='SessionDataEntry', + full_name='stackify.LogGroup.Log.Error.WebRequestDetail.SessionDataEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='stackify.LogGroup.Log.Error.WebRequestDetail.SessionDataEntry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='value', full_name='stackify.LogGroup.Log.Error.WebRequestDetail.SessionDataEntry.value', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=2603, + serialized_end=2653, +) + +_LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL = _descriptor.Descriptor( + name='WebRequestDetail', + full_name='stackify.LogGroup.Log.Error.WebRequestDetail', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='user_ip_address', full_name='stackify.LogGroup.Log.Error.WebRequestDetail.user_ip_address', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='http_method', full_name='stackify.LogGroup.Log.Error.WebRequestDetail.http_method', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='request_protocol', full_name='stackify.LogGroup.Log.Error.WebRequestDetail.request_protocol', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='request_url', full_name='stackify.LogGroup.Log.Error.WebRequestDetail.request_url', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='request_url_root', full_name='stackify.LogGroup.Log.Error.WebRequestDetail.request_url_root', index=4, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='referral_url', full_name='stackify.LogGroup.Log.Error.WebRequestDetail.referral_url', index=5, + number=6, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='headers', full_name='stackify.LogGroup.Log.Error.WebRequestDetail.headers', index=6, + number=7, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='cookies', full_name='stackify.LogGroup.Log.Error.WebRequestDetail.cookies', index=7, + number=8, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='querystring', full_name='stackify.LogGroup.Log.Error.WebRequestDetail.querystring', index=8, + number=9, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='post_data', full_name='stackify.LogGroup.Log.Error.WebRequestDetail.post_data', index=9, + number=10, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='session_data', full_name='stackify.LogGroup.Log.Error.WebRequestDetail.session_data', index=10, + number=11, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='post_data_raw', full_name='stackify.LogGroup.Log.Error.WebRequestDetail.post_data_raw', index=11, + number=12, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='mvc_action', full_name='stackify.LogGroup.Log.Error.WebRequestDetail.mvc_action', index=12, + number=13, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='mvc_controller', full_name='stackify.LogGroup.Log.Error.WebRequestDetail.mvc_controller', index=13, + number=14, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='mvc_area', full_name='stackify.LogGroup.Log.Error.WebRequestDetail.mvc_area', index=14, + number=15, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[_LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL_HEADERSENTRY, _LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL_COOKIESENTRY, _LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL_QUERYSTRINGENTRY, _LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL_POSTDATAENTRY, _LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL_SESSIONDATAENTRY, ], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1755, + serialized_end=2653, +) + +_LOGGROUP_LOG_ERROR = _descriptor.Descriptor( + name='Error', + full_name='stackify.LogGroup.Log.Error', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='environment_detail', full_name='stackify.LogGroup.Log.Error.environment_detail', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='date_millis', full_name='stackify.LogGroup.Log.Error.date_millis', index=1, + number=2, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='error_item', full_name='stackify.LogGroup.Log.Error.error_item', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='web_request_detail', full_name='stackify.LogGroup.Log.Error.web_request_detail', index=3, + number=4, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='server_variables', full_name='stackify.LogGroup.Log.Error.server_variables', index=4, + number=5, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='customer_name', full_name='stackify.LogGroup.Log.Error.customer_name', index=5, + number=6, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='username', full_name='stackify.LogGroup.Log.Error.username', index=6, + number=7, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[_LOGGROUP_LOG_ERROR_SERVERVARIABLESENTRY, _LOGGROUP_LOG_ERROR_ENVIRONMENTDETAIL, _LOGGROUP_LOG_ERROR_ERRORITEM, _LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL, ], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=752, + serialized_end=2653, +) + +_LOGGROUP_LOG = _descriptor.Descriptor( + name='Log', + full_name='stackify.LogGroup.Log', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='message', full_name='stackify.LogGroup.Log.message', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='data', full_name='stackify.LogGroup.Log.data', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='thread_name', full_name='stackify.LogGroup.Log.thread_name', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='date_millis', full_name='stackify.LogGroup.Log.date_millis', index=3, + number=4, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='level', full_name='stackify.LogGroup.Log.level', index=4, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='transaction_id', full_name='stackify.LogGroup.Log.transaction_id', index=5, + number=6, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='source_method', full_name='stackify.LogGroup.Log.source_method', index=6, + number=7, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='source_line', full_name='stackify.LogGroup.Log.source_line', index=7, + number=8, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='id', full_name='stackify.LogGroup.Log.id', index=8, + number=9, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='tags', full_name='stackify.LogGroup.Log.tags', index=9, + number=10, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='error', full_name='stackify.LogGroup.Log.error', index=10, + number=11, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[_LOGGROUP_LOG_ERROR, ], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=517, + serialized_end=2653, +) + +_LOGGROUP = _descriptor.Descriptor( + name='LogGroup', + full_name='stackify.LogGroup', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='environment', full_name='stackify.LogGroup.environment', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='server_name', full_name='stackify.LogGroup.server_name', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='application_name', full_name='stackify.LogGroup.application_name', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='application_location', full_name='stackify.LogGroup.application_location', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='logger', full_name='stackify.LogGroup.logger', index=4, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='platform', full_name='stackify.LogGroup.platform', index=5, + number=6, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='logs', full_name='stackify.LogGroup.logs', index=6, + number=7, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='container', full_name='stackify.LogGroup.container', index=7, + number=8, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='kubernetes', full_name='stackify.LogGroup.kubernetes', index=8, + number=9, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[_LOGGROUP_CONTAINER, _LOGGROUP_KUBERNETES, _LOGGROUP_LOG, ], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=35, + serialized_end=2653, +) + +_LOGGROUP_CONTAINER.containing_type = _LOGGROUP +_LOGGROUP_KUBERNETES.containing_type = _LOGGROUP +_LOGGROUP_LOG_ERROR_SERVERVARIABLESENTRY.containing_type = _LOGGROUP_LOG_ERROR +_LOGGROUP_LOG_ERROR_ENVIRONMENTDETAIL.containing_type = _LOGGROUP_LOG_ERROR +_LOGGROUP_LOG_ERROR_ERRORITEM_DATAENTRY.containing_type = _LOGGROUP_LOG_ERROR_ERRORITEM +_LOGGROUP_LOG_ERROR_ERRORITEM_TRACEFRAME.containing_type = _LOGGROUP_LOG_ERROR_ERRORITEM +_LOGGROUP_LOG_ERROR_ERRORITEM.fields_by_name['data'].message_type = _LOGGROUP_LOG_ERROR_ERRORITEM_DATAENTRY +_LOGGROUP_LOG_ERROR_ERRORITEM.fields_by_name['stacktrace'].message_type = _LOGGROUP_LOG_ERROR_ERRORITEM_TRACEFRAME +_LOGGROUP_LOG_ERROR_ERRORITEM.fields_by_name['inner_error'].message_type = _LOGGROUP_LOG_ERROR_ERRORITEM +_LOGGROUP_LOG_ERROR_ERRORITEM.containing_type = _LOGGROUP_LOG_ERROR +_LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL_HEADERSENTRY.containing_type = _LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL +_LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL_COOKIESENTRY.containing_type = _LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL +_LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL_QUERYSTRINGENTRY.containing_type = _LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL +_LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL_POSTDATAENTRY.containing_type = _LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL +_LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL_SESSIONDATAENTRY.containing_type = _LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL +_LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL.fields_by_name['headers'].message_type = _LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL_HEADERSENTRY +_LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL.fields_by_name['cookies'].message_type = _LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL_COOKIESENTRY +_LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL.fields_by_name['querystring'].message_type = _LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL_QUERYSTRINGENTRY +_LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL.fields_by_name['post_data'].message_type = _LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL_POSTDATAENTRY +_LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL.fields_by_name['session_data'].message_type = _LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL_SESSIONDATAENTRY +_LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL.containing_type = _LOGGROUP_LOG_ERROR +_LOGGROUP_LOG_ERROR.fields_by_name['environment_detail'].message_type = _LOGGROUP_LOG_ERROR_ENVIRONMENTDETAIL +_LOGGROUP_LOG_ERROR.fields_by_name['error_item'].message_type = _LOGGROUP_LOG_ERROR_ERRORITEM +_LOGGROUP_LOG_ERROR.fields_by_name['web_request_detail'].message_type = _LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL +_LOGGROUP_LOG_ERROR.fields_by_name['server_variables'].message_type = _LOGGROUP_LOG_ERROR_SERVERVARIABLESENTRY +_LOGGROUP_LOG_ERROR.containing_type = _LOGGROUP_LOG +_LOGGROUP_LOG.fields_by_name['error'].message_type = _LOGGROUP_LOG_ERROR +_LOGGROUP_LOG.containing_type = _LOGGROUP +_LOGGROUP.fields_by_name['logs'].message_type = _LOGGROUP_LOG +_LOGGROUP.fields_by_name['container'].message_type = _LOGGROUP_CONTAINER +_LOGGROUP.fields_by_name['kubernetes'].message_type = _LOGGROUP_KUBERNETES +DESCRIPTOR.message_types_by_name['LogGroup'] = _LOGGROUP + +LogGroup = _reflection.GeneratedProtocolMessageType('LogGroup', (_message.Message,), dict( + + Container = _reflection.GeneratedProtocolMessageType('Container', (_message.Message,), dict( + DESCRIPTOR = _LOGGROUP_CONTAINER, + __module__ = 'stackify_agent_pb2' + # @@protoc_insertion_point(class_scope:stackify.LogGroup.Container) + )) + , + + Kubernetes = _reflection.GeneratedProtocolMessageType('Kubernetes', (_message.Message,), dict( + DESCRIPTOR = _LOGGROUP_KUBERNETES, + __module__ = 'stackify_agent_pb2' + # @@protoc_insertion_point(class_scope:stackify.LogGroup.Kubernetes) + )) + , + + Log = _reflection.GeneratedProtocolMessageType('Log', (_message.Message,), dict( + + Error = _reflection.GeneratedProtocolMessageType('Error', (_message.Message,), dict( + + ServerVariablesEntry = _reflection.GeneratedProtocolMessageType('ServerVariablesEntry', (_message.Message,), dict( + DESCRIPTOR = _LOGGROUP_LOG_ERROR_SERVERVARIABLESENTRY, + __module__ = 'stackify_agent_pb2' + # @@protoc_insertion_point(class_scope:stackify.LogGroup.Log.Error.ServerVariablesEntry) + )) + , + + EnvironmentDetail = _reflection.GeneratedProtocolMessageType('EnvironmentDetail', (_message.Message,), dict( + DESCRIPTOR = _LOGGROUP_LOG_ERROR_ENVIRONMENTDETAIL, + __module__ = 'stackify_agent_pb2' + # @@protoc_insertion_point(class_scope:stackify.LogGroup.Log.Error.EnvironmentDetail) + )) + , + + ErrorItem = _reflection.GeneratedProtocolMessageType('ErrorItem', (_message.Message,), dict( + + DataEntry = _reflection.GeneratedProtocolMessageType('DataEntry', (_message.Message,), dict( + DESCRIPTOR = _LOGGROUP_LOG_ERROR_ERRORITEM_DATAENTRY, + __module__ = 'stackify_agent_pb2' + # @@protoc_insertion_point(class_scope:stackify.LogGroup.Log.Error.ErrorItem.DataEntry) + )) + , + + TraceFrame = _reflection.GeneratedProtocolMessageType('TraceFrame', (_message.Message,), dict( + DESCRIPTOR = _LOGGROUP_LOG_ERROR_ERRORITEM_TRACEFRAME, + __module__ = 'stackify_agent_pb2' + # @@protoc_insertion_point(class_scope:stackify.LogGroup.Log.Error.ErrorItem.TraceFrame) + )) + , + DESCRIPTOR = _LOGGROUP_LOG_ERROR_ERRORITEM, + __module__ = 'stackify_agent_pb2' + # @@protoc_insertion_point(class_scope:stackify.LogGroup.Log.Error.ErrorItem) + )) + , + + WebRequestDetail = _reflection.GeneratedProtocolMessageType('WebRequestDetail', (_message.Message,), dict( + + HeadersEntry = _reflection.GeneratedProtocolMessageType('HeadersEntry', (_message.Message,), dict( + DESCRIPTOR = _LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL_HEADERSENTRY, + __module__ = 'stackify_agent_pb2' + # @@protoc_insertion_point(class_scope:stackify.LogGroup.Log.Error.WebRequestDetail.HeadersEntry) + )) + , + + CookiesEntry = _reflection.GeneratedProtocolMessageType('CookiesEntry', (_message.Message,), dict( + DESCRIPTOR = _LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL_COOKIESENTRY, + __module__ = 'stackify_agent_pb2' + # @@protoc_insertion_point(class_scope:stackify.LogGroup.Log.Error.WebRequestDetail.CookiesEntry) + )) + , + + QuerystringEntry = _reflection.GeneratedProtocolMessageType('QuerystringEntry', (_message.Message,), dict( + DESCRIPTOR = _LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL_QUERYSTRINGENTRY, + __module__ = 'stackify_agent_pb2' + # @@protoc_insertion_point(class_scope:stackify.LogGroup.Log.Error.WebRequestDetail.QuerystringEntry) + )) + , + + PostDataEntry = _reflection.GeneratedProtocolMessageType('PostDataEntry', (_message.Message,), dict( + DESCRIPTOR = _LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL_POSTDATAENTRY, + __module__ = 'stackify_agent_pb2' + # @@protoc_insertion_point(class_scope:stackify.LogGroup.Log.Error.WebRequestDetail.PostDataEntry) + )) + , + + SessionDataEntry = _reflection.GeneratedProtocolMessageType('SessionDataEntry', (_message.Message,), dict( + DESCRIPTOR = _LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL_SESSIONDATAENTRY, + __module__ = 'stackify_agent_pb2' + # @@protoc_insertion_point(class_scope:stackify.LogGroup.Log.Error.WebRequestDetail.SessionDataEntry) + )) + , + DESCRIPTOR = _LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL, + __module__ = 'stackify_agent_pb2' + # @@protoc_insertion_point(class_scope:stackify.LogGroup.Log.Error.WebRequestDetail) + )) + , + DESCRIPTOR = _LOGGROUP_LOG_ERROR, + __module__ = 'stackify_agent_pb2' + # @@protoc_insertion_point(class_scope:stackify.LogGroup.Log.Error) + )) + , + DESCRIPTOR = _LOGGROUP_LOG, + __module__ = 'stackify_agent_pb2' + # @@protoc_insertion_point(class_scope:stackify.LogGroup.Log) + )) + , + DESCRIPTOR = _LOGGROUP, + __module__ = 'stackify_agent_pb2' + # @@protoc_insertion_point(class_scope:stackify.LogGroup) + )) +_sym_db.RegisterMessage(LogGroup) +_sym_db.RegisterMessage(LogGroup.Container) +_sym_db.RegisterMessage(LogGroup.Kubernetes) +_sym_db.RegisterMessage(LogGroup.Log) +_sym_db.RegisterMessage(LogGroup.Log.Error) +_sym_db.RegisterMessage(LogGroup.Log.Error.ServerVariablesEntry) +_sym_db.RegisterMessage(LogGroup.Log.Error.EnvironmentDetail) +_sym_db.RegisterMessage(LogGroup.Log.Error.ErrorItem) +_sym_db.RegisterMessage(LogGroup.Log.Error.ErrorItem.DataEntry) +_sym_db.RegisterMessage(LogGroup.Log.Error.ErrorItem.TraceFrame) +_sym_db.RegisterMessage(LogGroup.Log.Error.WebRequestDetail) +_sym_db.RegisterMessage(LogGroup.Log.Error.WebRequestDetail.HeadersEntry) +_sym_db.RegisterMessage(LogGroup.Log.Error.WebRequestDetail.CookiesEntry) +_sym_db.RegisterMessage(LogGroup.Log.Error.WebRequestDetail.QuerystringEntry) +_sym_db.RegisterMessage(LogGroup.Log.Error.WebRequestDetail.PostDataEntry) +_sym_db.RegisterMessage(LogGroup.Log.Error.WebRequestDetail.SessionDataEntry) + + +DESCRIPTOR.has_options = True +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\035com.stackify.api.common.protoB\rStackifyProto')) +_LOGGROUP_LOG_ERROR_SERVERVARIABLESENTRY.has_options = True +_LOGGROUP_LOG_ERROR_SERVERVARIABLESENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) +_LOGGROUP_LOG_ERROR_ERRORITEM_DATAENTRY.has_options = True +_LOGGROUP_LOG_ERROR_ERRORITEM_DATAENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) +_LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL_HEADERSENTRY.has_options = True +_LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL_HEADERSENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) +_LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL_COOKIESENTRY.has_options = True +_LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL_COOKIESENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) +_LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL_QUERYSTRINGENTRY.has_options = True +_LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL_QUERYSTRINGENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) +_LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL_POSTDATAENTRY.has_options = True +_LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL_POSTDATAENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) +_LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL_SESSIONDATAENTRY.has_options = True +_LOGGROUP_LOG_ERROR_WEBREQUESTDETAIL_SESSIONDATAENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) +# @@protoc_insertion_point(module_scope) diff --git a/stackify/transport/__init__.py b/stackify/transport/__init__.py new file mode 100644 index 0000000..633bb2e --- /dev/null +++ b/stackify/transport/__init__.py @@ -0,0 +1,116 @@ +import logging + +from stackify.constants import LOG_SAVE_URL +from stackify.constants import SOCKET_LOG_URL +from stackify.constants import TRANSPORT_TYPE_AGENT_SOCKET +from stackify.constants import TRANSPORT_TYPE_DEFAULT +from stackify.transport.agent import AgentSocket +from stackify.transport.agent.message import Log +from stackify.transport.agent.message import LogGroup +from stackify.transport.application import get_configuration +from stackify.transport.application import EnvironmentDetail +from stackify.transport.default import HTTPClient +from stackify.transport.default.log import LogMsg +from stackify.transport.default.log import LogMsgGroup + +internal_logger = logging.getLogger(__name__) + + +class TransportTypes(object): + """ + Transport Type class that will determine which transport to use + depending on users config. + + Types: + * DEFAULT - HTTP transport that will directly send logs to the Platform + * AGENT_SOCKET - HTTP warapped Unix Socket Domain that will send logs to the StackifyAgent + """ + + DEFAULT = TRANSPORT_TYPE_DEFAULT + AGENT_SOCKET = TRANSPORT_TYPE_AGENT_SOCKET + + @classmethod + def get_transport(self, api_config=None, env_details=None): + # determine which transport to use depening on users config + if api_config.transport == self.AGENT_SOCKET: + internal_logger.debug('Setting Agent Socket Transport.') + api_config.transport = self.AGENT_SOCKET + return AgentSocket() + + internal_logger.debug('Setting Default Transport.') + api_config.transport = self.DEFAULT + return HTTPClient(api_config, env_details) + + @classmethod + def create_message(self, record, api_config, env_details): + # create message depending on which transport + if api_config.transport == self.AGENT_SOCKET: + return Log(record, api_config, env_details).get_object() + + msg = LogMsg() + msg.from_record(record) + return msg + + @classmethod + def create_group_message(self, messages, api_config, env_details): + # create group message depending on which transport + if api_config.transport == self.AGENT_SOCKET: + return LogGroup(messages, api_config, env_details).get_object() + + return LogMsgGroup(messages) + + @classmethod + def get_log_url(self, api_config): + # return log url depending on which transport + if api_config.transport == self.AGENT_SOCKET: + return api_config.socket_url + SOCKET_LOG_URL + + return LOG_SAVE_URL + + @classmethod + def prepare_message(self, api_config, message): + # convert message depending on which transport + if api_config.transport == self.AGENT_SOCKET: + return message.SerializeToString() + + return message + + +class Transport(object): + """ + Transport base class + """ + + def __init__(self, config=None, **kwargs): + self.api_config = config or get_configuration(**kwargs) + self.env_details = EnvironmentDetail(self.api_config) + self._transport = TransportTypes.get_transport( + self.api_config, + self.env_details, + ) + + def create_message(self, record): + # create message from record + return TransportTypes.create_message( + record, + self.api_config, + self.env_details, + ) + + def create_group_message(self, messages): + # create group message from list of records + return TransportTypes.create_group_message( + messages, + self.api_config, + self.env_details, + ) + + def send(self, group_message): + # send group message + try: + self._transport.send( + TransportTypes.get_log_url(self.api_config), + TransportTypes.prepare_message(self.api_config, group_message), + ) + except Exception as e: + internal_logger.error('Request error: {}'.format(e)) diff --git a/stackify/transport/agent/__init__.py b/stackify/transport/agent/__init__.py new file mode 100644 index 0000000..4ba8479 --- /dev/null +++ b/stackify/transport/agent/__init__.py @@ -0,0 +1 @@ +from .agent_socket import AgentSocket # noqa diff --git a/stackify/transport/agent/agent_socket.py b/stackify/transport/agent/agent_socket.py new file mode 100644 index 0000000..aef8adc --- /dev/null +++ b/stackify/transport/agent/agent_socket.py @@ -0,0 +1,45 @@ +import logging +import os +import retrying +import requests_unixsocket + + +internal_logger = logging.getLogger(__name__) + + +class AgentSocket(object): + """ + AgentSocket class that will post message through unix socket domain + """ + + SOCKET_LOG_FILE = 'http+unix://%2Fusr%2Flocal%2Fstackify%2Fstackify.sock' + SOCKET_SCHEME = 'http+unix://' + + def __init__(self): + self._session = requests_unixsocket.Session() + + def _post(self, url, payload): + # will use stackify default domain socket if url is not given + # or not using http+unix:// + if not url.startswith(self.SOCKET_SCHEME): + url = os.path.join(self.SOCKET_LOG_FILE, url.lstrip('/')) + + internal_logger.debug('Request URL: {}'.format(url)) + internal_logger.debug('POST data: {}'.format(payload)) + + headers = { + 'Content-Type': 'application/x-protobuf', + } + + try: + response = self._session.post(url, payload, headers=headers) + internal_logger.debug('Response status: {}'.format(response.status_code)) + return response + except Exception as e: + internal_logger.debug('HTTP UNIX Socket domain exception: {}.'.format(e)) + raise + + @retrying.retry(wait_exponential_multiplier=1000, stop_max_delay=32000) + def send(self, url, payload): + # send payload through socket domain using _post method + self._post(url, payload) diff --git a/stackify/transport/agent/message.py b/stackify/transport/agent/message.py new file mode 100644 index 0000000..444fb5f --- /dev/null +++ b/stackify/transport/agent/message.py @@ -0,0 +1,125 @@ +import json +import sys +import traceback + +from stackify.constants import RECORD_VARS +from stackify.protos import stackify_agent_pb2 + + +class BaseMessage(object): + """ + Base Class wrapper for protobuf classes + This will help to create protobuf object with ease + """ + obj = None + + def get_object(self): + # return protobuf object + return self.obj + + +class EnvironmentDetail(BaseMessage): + """ + Class wrapper for protobuf LogGroup.Log.Error.EnvironmentDetail class + """ + + def __init__(self, api_config, environment_details): + self.obj = env_details = stackify_agent_pb2.LogGroup.Log.Error.EnvironmentDetail() + env_details.application_name = api_config.application + env_details.configured_application_name = api_config.application + env_details.configured_environment_name = api_config.environment + env_details.device_name = environment_details.deviceName + env_details.application_location = environment_details.appLocation + + +class TraceFrame(BaseMessage): + """ + Class wrapper for protobuf LogGroup.Log.Error.ErrorItem.TraceFrame class + """ + + def __init__(self, filename, lineno, method): + self.obj = trace_frame = stackify_agent_pb2.LogGroup.Log.Error.ErrorItem.TraceFrame() + trace_frame.code_filename = filename + trace_frame.line_number = lineno + trace_frame.method = method + + +class ErrorItem(BaseMessage): + """ + Class wrapper for protobuf LogGroup.Log.Error.ErrorItem.TraceFrame class + """ + + def __init__(self, exc_info): + self.obj = error_item = stackify_agent_pb2.LogGroup.Log.Error.ErrorItem() + + if not exc_info: + type_, value, tb = sys.exc_info() + else: + type_, value, tb = exc_info + + stacks = traceback.extract_tb(tb) + + error_item.message = str(value) + error_item.error_type = type_.__name__ + error_item.source_method = stacks[-1][2] + + for filename, lineno, method, text in reversed(stacks): + error_item.stacktrace.append(TraceFrame(filename, lineno, method).get_object()) + + +class Error(BaseMessage): + """ + Class wrapper for protobuf LogGroup.Log.Error class + """ + + def __init__(self, record, api_config, env_details): + self.obj = error = stackify_agent_pb2.LogGroup.Log.Error() + error.date_millis = int(record.created * 1000) + error.error_item.MergeFrom(ErrorItem(record.exc_info).get_object()) + error.environment_detail.MergeFrom(EnvironmentDetail(api_config, env_details).get_object()) + + +class Log(BaseMessage): + """ + Class wrapper for protobuf LogGroup.Log class + """ + + def __init__(self, record, api_config, env_details): + self.obj = log = stackify_agent_pb2.LogGroup.Log() + log.message = record.getMessage() + log.thread_name = record.threadName or record.thread + log.date_millis = int(record.created * 1000) + log.level = record.levelname + log.source_method = record.funcName + log.source_line = record.lineno + + if hasattr(record, 'log_id'): + log.id = record.log_id + + if hasattr(record, 'trans_id'): + log.transaction_id = record.trans_id + + data = {k: v for k, v in record.__dict__.items() + if k not in RECORD_VARS} + + if data: + log.data = json.dumps(data, default=lambda x: hasattr(x, '__dict__') and x.__dict__ or x.__str__()) + + if record.exc_info: + log.error.MergeFrom(Error(record, api_config, env_details).get_object()) + + +class LogGroup(BaseMessage): + """ + Class wrapper for protobuf LogGroup class + """ + + def __init__(self, messages, api_config, env_details, logger=None): + self.obj = log_group = stackify_agent_pb2.LogGroup() + log_group.environment = api_config.environment + log_group.application_name = api_config.application + log_group.server_name = env_details.deviceName + log_group.application_location = env_details.appLocation + log_group.logger = logger or __name__ + log_group.platform = 'python' + log_group.logs.MergeFrom(messages) diff --git a/stackify/transport/application.py b/stackify/transport/application.py new file mode 100644 index 0000000..5813f88 --- /dev/null +++ b/stackify/transport/application.py @@ -0,0 +1,58 @@ +import socket +import os + +from stackify.utils import arg_or_env +from stackify.constants import API_URL +from stackify.constants import SOCKET_URL +from stackify.constants import TRANSPORT_TYPE_AGENT_SOCKET +from stackify.constants import TRANSPORT_TYPE_DEFAULT +from stackify.transport.default.formats import JSONObject + + +class EnvironmentDetail(JSONObject): + """ + EnvironmentDetail class that stores application environment + and user define details + """ + + def __init__(self, api_config): + self.deviceName = socket.gethostname() + self.appLocation = os.getcwd() + self.configuredAppName = api_config.application + self.configuredEnvironmentName = api_config.environment + + +class ApiConfiguration: + """ + ApiConfiguration class that stores application configurations + """ + + def __init__(self, api_key, application, environment, api_url=API_URL, socket_url=SOCKET_URL, transport=None): + self.api_key = api_key + self.api_url = api_url + self.application = application + self.environment = environment + self.socket_url = socket_url + self.transport = transport + + +def get_configuration(**kwargs): + """ + return application configuration depending on users input, + application environment and application config + """ + + transport = arg_or_env('transport', kwargs, TRANSPORT_TYPE_DEFAULT) + if transport == TRANSPORT_TYPE_AGENT_SOCKET: + api_key = arg_or_env('api_key', kwargs, '') + else: + api_key = arg_or_env('api_key', kwargs) + + return ApiConfiguration( + application=arg_or_env('application', kwargs), + environment=arg_or_env('environment', kwargs), + api_key=api_key, + api_url=arg_or_env('api_url', kwargs, API_URL), + socket_url=arg_or_env('socket_url', kwargs, SOCKET_URL), + transport=transport, + ) diff --git a/stackify/transport/default/__init__.py b/stackify/transport/default/__init__.py new file mode 100644 index 0000000..419fdfd --- /dev/null +++ b/stackify/transport/default/__init__.py @@ -0,0 +1 @@ +from .http import HTTPClient # noqa diff --git a/stackify/error.py b/stackify/transport/default/error.py similarity index 96% rename from stackify/error.py rename to stackify/transport/default/error.py index 2d59607..c2ef4c4 100644 --- a/stackify/error.py +++ b/stackify/transport/default/error.py @@ -1,7 +1,7 @@ import traceback import sys -from stackify.formats import JSONObject +from stackify.transport.default.formats import JSONObject class ErrorItem(JSONObject): diff --git a/stackify/formats.py b/stackify/transport/default/formats.py similarity index 100% rename from stackify/formats.py rename to stackify/transport/default/formats.py diff --git a/stackify/http.py b/stackify/transport/default/http.py similarity index 86% rename from stackify/http.py rename to stackify/transport/default/http.py index dfeb098..c619ad6 100644 --- a/stackify/http.py +++ b/stackify/transport/default/http.py @@ -11,9 +11,7 @@ except Exception: pass # python 3, we use a new function in gzip -from stackify.application import EnvironmentDetail from stackify.constants import IDENTIFY_URL -from stackify.constants import LOG_SAVE_URL from stackify.constants import READ_TIMEOUT @@ -32,9 +30,9 @@ def gzip_compress(data): class HTTPClient: - def __init__(self, api_config): + def __init__(self, api_config, env_detail): self.api_config = api_config - self.environment_detail = EnvironmentDetail(api_config) + self.environment_detail = env_detail self.app_name_id = None self.app_env_id = None self.device_id = None @@ -72,8 +70,9 @@ def POST(self, url, json_object, use_gzip=False): # could not read json response internal_logger.exception('Cannot decode JSON response') - @retrying.retry(wait_exponential_multiplier=1000, stop_max_delay=10000) + @retrying.retry(wait_exponential_multiplier=1000, stop_max_delay=32000) def identify_application(self): + internal_logger.debug('Identifying application') result = self.POST(IDENTIFY_URL, self.environment_detail) self.app_name_id = result.get('AppNameID') self.app_env_id = result.get('AppEnvID') @@ -82,11 +81,18 @@ def identify_application(self): self.device_alias = result.get('DeviceAlias') self.identified = True - @retrying.retry(wait_exponential_multiplier=1000, stop_max_delay=10000) - def send_log_group(self, group): + @retrying.retry(wait_exponential_multiplier=1000, stop_max_delay=32000) + def send_log_group(self, url, group): + internal_logger.debug('Sending logs by group') group.Env = self.environment_detail.configuredEnvironmentName group.CDID = self.device_id group.CDAppID = self.device_app_id group.AppNameID = self.app_name_id group.ServerName = group.ServerName or self.environment_detail.deviceName - self.POST(LOG_SAVE_URL, group, True) + self.POST(url, group, True) + + def send(self, url, group): + if not self.identified: + self.identify_application() + + self.send_log_group(url, group) diff --git a/stackify/log.py b/stackify/transport/default/log.py similarity index 78% rename from stackify/log.py rename to stackify/transport/default/log.py index b711ce4..5b09da0 100644 --- a/stackify/log.py +++ b/stackify/transport/default/log.py @@ -1,17 +1,8 @@ import json -import logging -from stackify.formats import JSONObject -from stackify.error import StackifyError - - -# this is used to separate builtin keys from user-specified keys -RECORD_VARS = set(logging.LogRecord('', '', '', '', '', '', '', '').__dict__.keys()) - -# the "message" attribute is saved on the record object by a Formatter -RECORD_VARS.add('message') -RECORD_VARS.add('trans_id') -RECORD_VARS.add('log_id') +from stackify.constants import RECORD_VARS +from stackify.transport.default.formats import JSONObject +from stackify.transport.default.error import StackifyError class LogMsg(JSONObject): diff --git a/stackify/utils.py b/stackify/utils.py new file mode 100644 index 0000000..5eb8c1e --- /dev/null +++ b/stackify/utils.py @@ -0,0 +1,15 @@ +import os + + +def arg_or_env(name, args, default=None): + env_name = 'STACKIFY_{0}'.format(name.upper()) + try: + value = args.get(name) + if not value: + value = os.environ[env_name] + return value + except KeyError: + if default is not None: + return default + else: + raise NameError('You must specify the keyword argument {0} or environment variable {1}'.format(name, env_name)) diff --git a/tests/bases.py b/tests/bases.py index d5575f1..d495602 100644 --- a/tests/bases.py +++ b/tests/bases.py @@ -1,7 +1,11 @@ import os +import retrying import unittest +old_retry = retrying.retry + + class ClearEnvTest(unittest.TestCase): ''' This class clears the environment variables that the @@ -15,6 +19,7 @@ def setUp(self): 'STACKIFY_ENVIRONMENT', 'STACKIFY_API_KEY', 'STACKIFY_API_URL', + 'STACKIFY_TRANSPORT', ] self.saved = {} for key in to_save: @@ -27,3 +32,14 @@ def tearDown(self): for key, item in self.saved.items(): os.environ[key] = item del self.saved + + +def fake_retry_decorator(retries): + def fake_retry(*args, **kwargs): + kwargs['wait_exponential_max'] = 0 # no delay between retries + kwargs['stop_max_attempt_number'] = retries + + def inner(func): + return old_retry(*args, **kwargs)(func) + return inner + return fake_retry diff --git a/tests/test_handler.py b/tests/test_handler.py index d0dfd34..d915a14 100644 --- a/tests/test_handler.py +++ b/tests/test_handler.py @@ -11,7 +11,7 @@ import queue from stackify.handler import StackifyHandler, StackifyListener -from stackify.application import ApiConfiguration +from stackify.transport.application import ApiConfiguration import logging @@ -49,22 +49,22 @@ def setUp(self): # don't print warnings on http crashes, so mute stackify logger logging.getLogger('stackify').propagate = False - @patch('stackify.handler.LogMsg') - @patch('stackify.handler.StackifyListener.send_group') - @patch('stackify.handler.HTTPClient.POST') - def test_not_identified(self, post, send_group, logmsg): + @patch('stackify.transport.Transport.create_message') + @patch('stackify.transport.default.http.HTTPClient.POST') + def test_not_identified(self, post, logmsg): '''The HTTPClient identifies automatically if needed''' listener = StackifyListener(queue_=Mock(), config=self.config) listener.handle(Mock()) - self.assertTrue(listener.http.identified) + listener.send_group() + self.assertTrue(listener.transport._transport.identified) - @patch('stackify.handler.LogMsg') - @patch('stackify.handler.LogMsgGroup') - @patch('stackify.handler.HTTPClient.POST') + @patch('stackify.transport.Transport.create_message') + @patch('stackify.transport.Transport.create_group_message') + @patch('stackify.transport.default.http.HTTPClient.POST') def test_send_group_if_needed(self, post, logmsggroup, logmsg): '''The listener sends groups of messages''' listener = StackifyListener(queue_=Mock(), max_batch=3, config=self.config) - listener.http.identified = True + listener.transport._transport.identified = True listener.handle(1) self.assertFalse(post.called) @@ -78,12 +78,12 @@ def test_send_group_if_needed(self, post, logmsggroup, logmsg): self.assertEqual(post.call_count, 1) self.assertEqual(len(listener.messages), 1) - @patch('stackify.handler.LogMsg') + @patch('stackify.transport.Transport.create_message') @patch('stackify.handler.StackifyListener.send_group') def test_clear_queue_shutdown(self, send_group, logmsg): '''The listener sends the leftover messages on the queue when shutting down''' listener = StackifyListener(queue_=Mock(), max_batch=3, config=self.config) - listener.http.identified = True + listener.transport._transport.identified = True listener._thread = Mock() listener.handle(1) @@ -92,13 +92,13 @@ def test_clear_queue_shutdown(self, send_group, logmsg): listener.stop() self.assertTrue(send_group.called) - @patch('stackify.handler.LogMsg') - @patch('stackify.handler.LogMsgGroup') - @patch('stackify.handler.HTTPClient.send_log_group') + @patch('stackify.transport.Transport.create_message') + @patch('stackify.transport.Transport.create_group_message') + @patch('stackify.transport.default.http.HTTPClient.send_log_group') def test_send_group_crash(self, send_log_group, logmsggroup, logmsg): '''The listener drops messages after retrying''' listener = StackifyListener(queue_=Mock(), max_batch=3, config=self.config) - listener.http.identified = True + listener.transport._transport.identified = True send_log_group.side_effect = Exception diff --git a/tests/test_init.py b/tests/test_init.py index bccee01..547f109 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -17,7 +17,7 @@ class TestInit(ClearEnvTest): def setUp(self): super(TestInit, self).setUp() - self.config = stackify.ApiConfiguration( + self.config = stackify.transport.application.ApiConfiguration( application='test_appname', environment='test_environment', api_key='test_apikey', @@ -43,7 +43,7 @@ def test_logger_no_config(self): logger = stackify.getLogger(auto_shutdown=False) self.loggers.append(logger) - config = logger.handlers[0].listener.http.api_config + config = logger.handlers[0].listener.transport.api_config self.assertEqual(config.application, 'test2_appname') self.assertEqual(config.environment, 'test2_environment') @@ -55,7 +55,7 @@ def test_logger_api_config(self): logger = stackify.getLogger(config=self.config, auto_shutdown=False) self.loggers.append(logger) - config = logger.handlers[0].listener.http.api_config + config = logger.handlers[0].listener.transport.api_config self.assertEqual(config.application, 'test_appname') self.assertEqual(config.environment, 'test_environment') @@ -79,7 +79,7 @@ def test_get_logger_defaults(self): self.loggers.append(logger) handler = logger.handlers[0] - config = handler.listener.http.api_config + config = handler.listener.transport.api_config self.assertEqual(logger.name, 'tests.test_init') self.assertEqual(config.api_url, stackify.constants.API_URL) diff --git a/tests/transport/__init__.py b/tests/transport/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/transport/agent/__init__.py b/tests/transport/agent/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/transport/agent/test_agent_socket.py b/tests/transport/agent/test_agent_socket.py new file mode 100644 index 0000000..2bd5229 --- /dev/null +++ b/tests/transport/agent/test_agent_socket.py @@ -0,0 +1,64 @@ +import imp +import retrying +from unittest import TestCase +from mock import patch + +import stackify +from tests.bases import fake_retry_decorator + + +class TestAgentSocket(TestCase): + + @classmethod + def setUpClass(self): + retrying.retry = fake_retry_decorator(3) + imp.reload(stackify.transport.agent.agent_socket) + + @classmethod + def tearDownClass(self): + imp.reload(retrying) + imp.reload(stackify.transport.agent.agent_socket) + + def setUp(self): + self.agent_socket = stackify.transport.agent.agent_socket.AgentSocket() + + @patch('requests_unixsocket.Session.post') + def test_send(self, mock_post): + url = 'http+unix://test_url' + message = 'message' + + self.agent_socket.send(url, message) + + assert mock_post.called + assert mock_post.call_args_list[0][0][0] == 'http+unix://test_url' + + @patch('requests_unixsocket.Session.post') + def test_send_should_use_defaut_socket(self, mock_post): + url = '/test_url' + message = 'message' + + self.agent_socket.send(url, message) + + assert mock_post.called + assert mock_post.call_args_list[0][0][0] == 'http+unix://%2Fusr%2Flocal%2Fstackify%2Fstackify.sock/test_url' + + @patch('requests_unixsocket.Session.post') + def test_send_should_include_headers(self, mock_post): + url = '/test_url' + message = 'message' + + self.agent_socket.send(url, message) + + assert mock_post.called + assert mock_post.call_args_list[0][1]['headers']['Content-Type'] == 'application/x-protobuf' + + @patch('requests_unixsocket.Session.post') + def test_retry(self, mock_post): + url = '/test_url' + message = 'message' + mock_post.side_effect = Exception('some error') + + with self.assertRaises(Exception): + self.agent_socket.send(url, message) + + assert mock_post.call_count == 3 diff --git a/tests/transport/agent/test_message.py b/tests/transport/agent/test_message.py new file mode 100644 index 0000000..7f73886 --- /dev/null +++ b/tests/transport/agent/test_message.py @@ -0,0 +1,135 @@ +import logging +from unittest import TestCase + +from stackify.protos import stackify_agent_pb2 +from stackify.transport import application +from stackify.transport.agent.message import Log +from stackify.transport.agent.message import LogGroup + + +class TestLog(TestCase): + def setUp(self): + self.config = application.ApiConfiguration( + application='test_appname', + environment='test_environment', + api_key='test_apikey', + api_url='test_apiurl', + ) + self.env_details = application.EnvironmentDetail(self.config) + + def test_get_object(self): + with self.assertLogs('foo', level='INFO') as logging_watcher: + logging.getLogger('foo').info('some log') + + log = Log(logging_watcher.records[0], self.config, self.env_details).get_object() + + assert isinstance(log, stackify_agent_pb2.LogGroup.Log) + + def test_info_log_details(self): + with self.assertLogs('foo', level='INFO') as logging_watcher: + logging.getLogger('foo').info('some log') + + log = Log(logging_watcher.records[0], self.config, self.env_details).get_object() + + assert log.message == 'some log' + assert log.level == 'INFO' + assert log.thread_name + assert log.date_millis + assert log.source_method + assert log.source_line + assert not log.HasField('error') + + def test_info_error_details(self): + with self.assertLogs('foo', level='ERROR') as logging_watcher: + logging.getLogger('foo').error('some error') + + log = Log(logging_watcher.records[0], self.config, self.env_details).get_object() + + assert log.message == 'some error' + assert log.level == 'ERROR' + assert log.thread_name + assert log.date_millis + assert log.source_method + assert log.source_line + assert not log.HasField('error') + + def test_info_exception_details(self): + with self.assertLogs('foo', level='ERROR') as logging_watcher: + try: + 1 / 0 + except Exception: + logging.getLogger('foo').exception('some error') + + log = Log(logging_watcher.records[0], self.config, self.env_details).get_object() + + assert log.message == 'some error' + assert log.level == 'ERROR' + assert log.thread_name + assert log.date_millis + assert log.source_method + assert log.source_line + assert log.HasField('error') + + error = log.error + assert error.date_millis + assert error.HasField('environment_detail') + assert error.HasField('error_item') + + environment_detail = error.environment_detail + assert environment_detail.application_name == 'test_appname' + assert environment_detail.configured_application_name == 'test_appname' + assert environment_detail.configured_environment_name == 'test_environment' + assert environment_detail.device_name == self.env_details.deviceName + assert environment_detail.application_location == self.env_details.appLocation + + error_item = error.error_item + assert error_item.message == 'division by zero' + assert error_item.error_type == 'ZeroDivisionError' + assert error_item.source_method == 'test_info_exception_details' + assert len(error_item.stacktrace) + + stacktraces = error_item.stacktrace + for stacktrace in stacktraces: + assert stacktrace.code_filename.endswith('test_message.py') + assert stacktrace.method == 'test_info_exception_details' + assert stacktrace.line_number + + +class TestLogGroup(TestCase): + def setUp(self): + self.config = application.ApiConfiguration( + application='test_appname', + environment='test_environment', + api_key='test_apikey', + api_url='test_apiurl', + ) + self.env_details = application.EnvironmentDetail(self.config) + + def test_get_object(self): + log_group = LogGroup([], self.config, self.env_details).get_object() + + assert isinstance(log_group, stackify_agent_pb2.LogGroup) + + def test_details(self): + with self.assertLogs('foo', level='INFO') as logging_watcher: + logging.getLogger('foo').info('some log') + + log = Log(logging_watcher.records[0], self.config, self.env_details).get_object() + log_group = LogGroup([log], self.config, self.env_details).get_object() + + assert log_group.environment == 'test_environment' + assert log_group.application_name == 'test_appname' + assert log_group.server_name == self.env_details.deviceName + assert log_group.application_location == self.env_details.appLocation + assert log_group.logger + assert log_group.platform == 'python' + assert len(log_group.logs) + + log_group_logs = log_group.logs + assert log_group_logs[0].message == 'some log' + assert log_group_logs[0].level == 'INFO' + assert log_group_logs[0].thread_name + assert log_group_logs[0].date_millis + assert log_group_logs[0].source_method + assert log_group_logs[0].source_line + assert not log_group_logs[0].HasField('error') diff --git a/tests/transport/default/__init__.py b/tests/transport/default/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_formats.py b/tests/transport/default/test_formats.py similarity index 96% rename from tests/test_formats.py rename to tests/transport/default/test_formats.py index bfa6c11..0889c52 100644 --- a/tests/test_formats.py +++ b/tests/transport/default/test_formats.py @@ -5,7 +5,7 @@ import unittest import json -from stackify.formats import JSONObject +from stackify.transport.default.formats import JSONObject class TestJSONObject(unittest.TestCase): diff --git a/tests/test_http.py b/tests/transport/default/test_http.py similarity index 84% rename from tests/test_http.py rename to tests/transport/default/test_http.py index c109375..3512057 100644 --- a/tests/test_http.py +++ b/tests/transport/default/test_http.py @@ -7,24 +7,13 @@ import imp import retrying -import stackify.http +import stackify.transport.default.http -from stackify.log import LogMsgGroup -from stackify.application import ApiConfiguration +from stackify.transport.default.log import LogMsgGroup +from stackify.transport.application import ApiConfiguration from stackify.constants import READ_TIMEOUT - -old_retry = retrying.retry - - -def fake_retry_decorator(retries): - def fake_retry(*args, **kwargs): - kwargs['wait_exponential_max'] = 0 # no delay between retries - kwargs['stop_max_attempt_number'] = retries - - def inner(func): - return old_retry(*args, **kwargs)(func) - return inner - return fake_retry +from stackify.transport.application import EnvironmentDetail +from tests.bases import fake_retry_decorator class TestClient(unittest.TestCase): @@ -36,12 +25,12 @@ class TestClient(unittest.TestCase): def setUpClass(cls): cls.FAKE_RETRIES = 3 retrying.retry = fake_retry_decorator(cls.FAKE_RETRIES) - imp.reload(stackify.http) + imp.reload(stackify.transport.default.http) @classmethod def tearDownClass(cls): imp.reload(retrying) - imp.reload(stackify.http) + imp.reload(stackify.transport.default.http) def setUp(self): self.config = ApiConfiguration( @@ -50,13 +39,14 @@ def setUp(self): api_key='test_apikey', api_url='test_apiurl', ) + self.env_details = EnvironmentDetail(self.config) - self.client = stackify.http.HTTPClient(self.config) + self.client = stackify.transport.default.http.HTTPClient(self.config, self.env_details) def test_logger_no_config(self): '''GZIP encoder works''' correct = list(b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\xf3H\xcd\xc9\xc9\xd7Q(\xcf/\xcaIQ\x04\x00\xe6\xc6\xe6\xeb\r\x00\x00\x00') - gzipped = list(stackify.http.gzip_compress('Hello, world!')) + gzipped = list(stackify.transport.default.http.gzip_compress('Hello, world!')) gzipped[4:8] = b'\x00\x00\x00\x00' # blank the mtime self.assertEqual(gzipped, correct) @@ -111,7 +101,7 @@ class CustomException(Exception): with patch.object(client, 'POST', crash): with self.assertRaises(CustomException): - client.send_log_group(group) + client.send_log_group('url', group) self.assertEqual(crash.call_count, self.FAKE_RETRIES) def test_send_log_group(self): @@ -127,7 +117,7 @@ def test_send_log_group(self): group = LogMsgGroup([]) with patch.object(client, 'POST') as post: - client.send_log_group(group) + client.send_log_group('url', group) self.assertTrue(post.called) self.assertEqual(group.CDID, client.device_id) @@ -162,7 +152,7 @@ def test_post_gzip(self, post): payload.toJSON = Mock(return_value='1') gzip = Mock(side_effect=lambda x: x + '_gzipped') - with patch.object(stackify.http, 'gzip_compress', gzip): + with patch.object(stackify.transport.default.http, 'gzip_compress', gzip): client.POST('url', payload, use_gzip=True) self.assertTrue(post.called) diff --git a/tests/test_log.py b/tests/transport/default/test_log.py similarity index 97% rename from tests/test_log.py rename to tests/transport/default/test_log.py index 2d28b37..39b6269 100644 --- a/tests/test_log.py +++ b/tests/transport/default/test_log.py @@ -6,7 +6,7 @@ import json import sys -from stackify.log import LogMsg +from stackify.transport.default.log import LogMsg import logging import time diff --git a/tests/test_application.py b/tests/transport/test_application.py similarity index 60% rename from tests/test_application.py rename to tests/transport/test_application.py index 55b0296..901872f 100644 --- a/tests/test_application.py +++ b/tests/transport/test_application.py @@ -4,10 +4,10 @@ import unittest from mock import patch -from .bases import ClearEnvTest +from tests.bases import ClearEnvTest from stackify.constants import API_URL -from stackify.application import get_configuration +from stackify.transport.application import get_configuration class TestConfig(ClearEnvTest): @@ -90,6 +90,60 @@ def test_api_url_default(self): self.assertEqual(config.api_key, 'test4_apikey') self.assertEqual(config.api_url, API_URL) + def test_transport_default(self): + config = get_configuration( + application='test4_appname', + environment='test4_environment', + api_key='test4_apikey', + api_url='test3_apiurl', + ) + + self.assertEqual(config.application, 'test4_appname') + self.assertEqual(config.environment, 'test4_environment') + self.assertEqual(config.api_key, 'test4_apikey') + self.assertEqual(config.api_url, 'test3_apiurl') + self.assertEqual(config.transport, 'default') + + def test_transport_given(self): + config = get_configuration( + application='test5_appname', + environment='test5_environment', + api_key='test5_apikey', + api_url='test5_apiurl', + transport='test5_transport' + ) + + self.assertEqual(config.application, 'test5_appname') + self.assertEqual(config.environment, 'test5_environment') + self.assertEqual(config.api_key, 'test5_apikey') + self.assertEqual(config.api_url, 'test5_apiurl') + self.assertEqual(config.transport, 'test5_transport') + + def test_api_key_is_required_on_default_transport(self): + with self.assertRaises(NameError): + get_configuration( + application='test_appname', + environment='test_environment', + api_key='', + api_url='test_apiurl', + transport='default' + ) + + def test_api_key_is_not_required_on_agent_socket_transport(self): + config = get_configuration( + application='test_appname', + environment='test_environment', + api_key='', + api_url='test_apiurl', + transport='agent_socket' + ) + + self.assertEqual(config.application, 'test_appname') + self.assertEqual(config.environment, 'test_environment') + self.assertEqual(config.api_key, '') + self.assertEqual(config.api_url, 'test_apiurl') + self.assertEqual(config.transport, 'agent_socket') + if __name__ == '__main__': unittest.main() diff --git a/tests/transport/test_init.py b/tests/transport/test_init.py new file mode 100644 index 0000000..5910a2c --- /dev/null +++ b/tests/transport/test_init.py @@ -0,0 +1,163 @@ +import logging +from mock import patch + +from tests.bases import ClearEnvTest +from stackify.protos import stackify_agent_pb2 +from stackify.transport import Transport +from stackify.transport.agent import AgentSocket +from stackify.transport.default import HTTPClient +from stackify.transport.default.log import LogMsg +from stackify.transport.default.log import LogMsgGroup + + +class TestTransport(ClearEnvTest): + def test_invalid_transport(self): + config = { + 'application': 'test_appname', + 'environment': 'test_environment', + 'api_key': 'test_apikey', + 'api_url': 'test_apiurl', + 'transport': 'invalid', + } + + transport = Transport(**config) + + assert isinstance(transport._transport, HTTPClient) + + def test_default_transport(self): + config = { + 'application': 'test_appname', + 'environment': 'test_environment', + 'api_key': 'test_apikey', + 'api_url': 'test_apiurl', + } + + transport = Transport(**config) + + assert isinstance(transport._transport, HTTPClient) + + def test_default_create_message(self): + config = { + 'application': 'test_appname', + 'environment': 'test_environment', + 'api_key': 'test_apikey', + 'api_url': 'test_apiurl', + } + + transport = Transport(**config) + message = transport.create_message(logging.makeLogRecord({'mgs': 'message'})) + + assert isinstance(message, LogMsg) + + def test_default_create_group_message(self): + config = { + 'application': 'test_appname', + 'environment': 'test_environment', + 'api_key': 'test_apikey', + 'api_url': 'test_apiurl', + } + + transport = Transport(**config) + message = transport.create_message(logging.makeLogRecord({'mgs': 'message'})) + group_message = transport.create_group_message([message]) + + assert isinstance(group_message, LogMsgGroup) + + @patch('stackify.transport.default.http.HTTPClient.send') + def test_default_send_url(self, mock_send): + config = { + 'application': 'test_appname', + 'environment': 'test_environment', + 'api_key': 'test_apikey', + 'api_url': 'test_apiurl', + } + + transport = Transport(**config) + message = transport.create_message(logging.makeLogRecord({'mgs': 'message'})) + group_message = transport.create_group_message([message]) + transport.send(group_message) + + assert mock_send.called + assert mock_send.call_args_list[0][0][0] == '/Log/Save' + + def test_agent_socket_transport(self): + config = { + 'application': 'test_appname', + 'environment': 'test_environment', + 'api_key': 'test_apikey', + 'api_url': 'test_apiurl', + 'socket_url': 'test_socketurl', + 'transport': 'agent_socket', + } + + transport = Transport(**config) + + assert isinstance(transport._transport, AgentSocket) + + def test_agent_socket_create_message(self): + config = { + 'application': 'test_appname', + 'environment': 'test_environment', + 'api_key': 'test_apikey', + 'api_url': 'test_apiurl', + 'socket_url': 'test_socketurl', + 'transport': 'agent_socket', + } + + transport = Transport(**config) + message = transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + + isinstance(message, stackify_agent_pb2.LogGroup.Log) + + def test_agent_socket_create_group_message(self): + config = { + 'application': 'test_appname', + 'environment': 'test_environment', + 'api_key': 'test_apikey', + 'api_url': 'test_apiurl', + 'socket_url': 'test_socketurl', + 'transport': 'agent_socket', + } + + transport = Transport(**config) + message = transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + group_message = transport.create_group_message([message]) + + assert isinstance(group_message, stackify_agent_pb2.LogGroup) + + @patch('stackify.transport.agent.agent_socket.AgentSocket.send') + def test_agent_socket_send_url(self, mock_send): + config = { + 'application': 'test_appname', + 'environment': 'test_environment', + 'api_key': 'test_apikey', + 'api_url': 'test_apiurl', + 'socket_url': 'test_socketurl', + 'transport': 'agent_socket', + } + + transport = Transport(**config) + message = transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + group_message = transport.create_group_message([message]) + transport.send(group_message) + + assert mock_send.called + assert mock_send.call_args_list[0][0][0] == 'test_socketurl/log' + + @patch('stackify.transport.agent.agent_socket.AgentSocket.send') + def test_agent_socket_send_url_default(self, mock_send): + config = { + 'application': 'test_appname', + 'environment': 'test_environment', + 'api_key': 'test_apikey', + 'api_url': 'test_apiurl', + 'transport': 'agent_socket', + } + + transport = Transport(**config) + message = transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + group_message = transport.create_group_message([message]) + transport.send(group_message) + + assert mock_send.called + assert mock_send.call_args_list[0][0][0] == 'http+unix://%2Fusr%2Flocal%2Fstackify%2Fstackify.sock/log' From 85a9a2008844be8e12b0e1f8796c6cd2516b5a0d Mon Sep 17 00:00:00 2001 From: Darin Howard Date: Tue, 10 Sep 2019 10:21:08 -0500 Subject: [PATCH 16/44] Updating to version 1.0.4 --- stackify/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackify/__init__.py b/stackify/__init__.py index 513b7d8..94773c5 100644 --- a/stackify/__init__.py +++ b/stackify/__init__.py @@ -1,7 +1,7 @@ """ Stackify Python API """ -__version__ = '1.0.3' +__version__ = '1.0.4' import logging import inspect From 60ef39288672c521bd9dee2a10c656390c878eac Mon Sep 17 00:00:00 2001 From: jaygel Date: Thu, 12 Sep 2019 22:33:33 +0800 Subject: [PATCH 17/44] update setup to search for packages and updated readme --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 568f68f..42aee10 100755 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -from setuptools import setup +import setuptools import re import ast @@ -19,12 +19,12 @@ f = f.read() version = ast.literal_eval(version_re.search(f).group(1)) -setup( +setuptools.setup( name='stackify-api-python', version=version, author='Stackify', author_email='support@stackify.com', - packages=['stackify'], + packages=setuptools.find_packages(exclude=("tests", "*tests", "tests*",)), url='https://github.com/stackify/stackify-api-python', description='Stackify API for Python', long_description=long_description, From 494a58aad5c97e2ad51cffcdae9b7305e42832c0 Mon Sep 17 00:00:00 2001 From: Darin Howard Date: Thu, 12 Sep 2019 10:41:34 -0500 Subject: [PATCH 18/44] updating version to 1.0.5 --- stackify/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackify/__init__.py b/stackify/__init__.py index 94773c5..5896495 100644 --- a/stackify/__init__.py +++ b/stackify/__init__.py @@ -1,7 +1,7 @@ """ Stackify Python API """ -__version__ = '1.0.4' +__version__ = '1.0.5' import logging import inspect From f2e47ec74e8274a76b0810f1e8d130b7cb15c016 Mon Sep 17 00:00:00 2001 From: jaygel Date: Tue, 3 Dec 2019 00:52:30 +0800 Subject: [PATCH 19/44] API - Add AGENT_HTTP Transport (#8) API - Add AGENT_HTTP Transport --- requirements.txt | 1 + setup.cfg | 5 +- stackify/constants.py | 4 +- stackify/handler.py | 4 +- stackify/timer.py | 2 + stackify/transport/__init__.py | 104 ++++--------------- stackify/transport/agent/__init__.py | 31 +++++- stackify/transport/agent/agent_http.py | 26 +++++ stackify/transport/agent/agent_socket.py | 2 +- stackify/transport/agent/message.py | 2 +- stackify/transport/application.py | 18 +++- stackify/transport/base.py | 40 ++++++++ stackify/transport/default/__init__.py | 31 +++++- stackify/utils.py | 4 +- test.sh | 47 +++++++++ tests/bases.py | 102 ++++++++++++++++++- tests/test_handler.py | 12 +-- tests/test_init.py | 6 +- tests/test_timer.py | 32 ++++++ tests/transport/agent/test_init.py | 111 +++++++++++++++++++++ tests/transport/agent/test_message.py | 21 +++- tests/transport/default/test_init.py | 47 +++++++++ tests/transport/test_application.py | 73 ++++++++++++++ tests/transport/test_base.py | 84 ++++++++++++++++ tests/transport/test_init.py | 121 +++++++++++++++++++---- 25 files changed, 806 insertions(+), 124 deletions(-) create mode 100644 stackify/transport/agent/agent_http.py create mode 100644 stackify/transport/base.py create mode 100755 test.sh create mode 100644 tests/test_timer.py create mode 100644 tests/transport/agent/test_init.py create mode 100644 tests/transport/default/test_init.py create mode 100644 tests/transport/test_base.py diff --git a/requirements.txt b/requirements.txt index 911174c..54d51bf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +flake8 mock==2.0.0 protobuf==3.9.1 pytest==4.3.0 diff --git a/setup.cfg b/setup.cfg index 51d679c..470d4da 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,13 +11,16 @@ exclude = README.md, LICENSE.md, requirements.txt, + *protos*, [coverage:run] include = stackify/* omit = - *tests* + *tests*, + *handler_backport.py, + *protos*, [tool:pytest] python_files=tests.py test.py test_*.py *_test.py tests_*.py *_tests.py diff --git a/stackify/constants.py b/stackify/constants.py index 1d94e4a..3dce49e 100644 --- a/stackify/constants.py +++ b/stackify/constants.py @@ -7,8 +7,9 @@ # using `%2F` instead of `/` as per package documentation DEFAULT_SOCKET_FILE = '%2Fusr%2Flocal%2Fstackify%2Fstackify.sock' +DEFAULT_HTTP_ENDPOINT = 'https://localhost:10601' SOCKET_URL = 'http+unix://' + DEFAULT_SOCKET_FILE -SOCKET_LOG_URL = '/log' +AGENT_LOG_URL = '/log' API_REQUEST_INTERVAL_IN_SEC = 30 @@ -36,3 +37,4 @@ TRANSPORT_TYPE_DEFAULT = 'default' TRANSPORT_TYPE_AGENT_SOCKET = 'agent_socket' +TRANSPORT_TYPE_AGENT_HTTP = 'agent_http' diff --git a/stackify/handler.py b/stackify/handler.py index 9478f36..f957f8e 100644 --- a/stackify/handler.py +++ b/stackify/handler.py @@ -16,7 +16,7 @@ from stackify.constants import MAX_BATCH from stackify.constants import QUEUE_SIZE from stackify.timer import RepeatedTimer -from stackify.transport import Transport +from stackify.transport import configure_transport internal_logger = logging.getLogger(__name__) @@ -70,7 +70,7 @@ def __init__(self, queue_, max_batch=MAX_BATCH, config=None, **kwargs): self.max_batch = max_batch self.messages = [] - self.transport = Transport(config, **kwargs) + self.transport = configure_transport(config, **kwargs) self.timer = RepeatedTimer(API_REQUEST_INTERVAL_IN_SEC, self.send_group) self._started = False diff --git a/stackify/timer.py b/stackify/timer.py index 0bb0c0a..5dd6c13 100644 --- a/stackify/timer.py +++ b/stackify/timer.py @@ -27,10 +27,12 @@ def _time(self): def start(self): if not self._started: + self._started = True self.thread.setDaemon(True) self.thread.start() def stop(self): if self._started: + self._started = False self.event.set() self.thread.join() diff --git a/stackify/transport/__init__.py b/stackify/transport/__init__.py index 633bb2e..1b57e5c 100644 --- a/stackify/transport/__init__.py +++ b/stackify/transport/__init__.py @@ -1,17 +1,14 @@ import logging -from stackify.constants import LOG_SAVE_URL -from stackify.constants import SOCKET_LOG_URL +from stackify.constants import TRANSPORT_TYPE_AGENT_HTTP from stackify.constants import TRANSPORT_TYPE_AGENT_SOCKET from stackify.constants import TRANSPORT_TYPE_DEFAULT -from stackify.transport.agent import AgentSocket -from stackify.transport.agent.message import Log -from stackify.transport.agent.message import LogGroup +from stackify.transport.agent import AgentSocketTransport +from stackify.transport.agent import AgentHTTPTransport from stackify.transport.application import get_configuration from stackify.transport.application import EnvironmentDetail -from stackify.transport.default import HTTPClient -from stackify.transport.default.log import LogMsg -from stackify.transport.default.log import LogMsgGroup +from stackify.transport.default import DefaultTransport + internal_logger = logging.getLogger(__name__) @@ -24,93 +21,34 @@ class TransportTypes(object): Types: * DEFAULT - HTTP transport that will directly send logs to the Platform * AGENT_SOCKET - HTTP warapped Unix Socket Domain that will send logs to the StackifyAgent + * AGENT_HTTP - HTTP transport that will send logs to the Agent using HTTP requests """ DEFAULT = TRANSPORT_TYPE_DEFAULT AGENT_SOCKET = TRANSPORT_TYPE_AGENT_SOCKET + AGENT_HTTP = TRANSPORT_TYPE_AGENT_HTTP @classmethod def get_transport(self, api_config=None, env_details=None): # determine which transport to use depening on users config if api_config.transport == self.AGENT_SOCKET: internal_logger.debug('Setting Agent Socket Transport.') - api_config.transport = self.AGENT_SOCKET - return AgentSocket() + return AgentSocketTransport(api_config, env_details) + + if api_config.transport == self.AGENT_HTTP: + internal_logger.debug('Setting Agent HTTP Transport.') + return AgentHTTPTransport(api_config, env_details) internal_logger.debug('Setting Default Transport.') api_config.transport = self.DEFAULT - return HTTPClient(api_config, env_details) - - @classmethod - def create_message(self, record, api_config, env_details): - # create message depending on which transport - if api_config.transport == self.AGENT_SOCKET: - return Log(record, api_config, env_details).get_object() - - msg = LogMsg() - msg.from_record(record) - return msg - - @classmethod - def create_group_message(self, messages, api_config, env_details): - # create group message depending on which transport - if api_config.transport == self.AGENT_SOCKET: - return LogGroup(messages, api_config, env_details).get_object() - - return LogMsgGroup(messages) - - @classmethod - def get_log_url(self, api_config): - # return log url depending on which transport - if api_config.transport == self.AGENT_SOCKET: - return api_config.socket_url + SOCKET_LOG_URL - - return LOG_SAVE_URL - - @classmethod - def prepare_message(self, api_config, message): - # convert message depending on which transport - if api_config.transport == self.AGENT_SOCKET: - return message.SerializeToString() - - return message - - -class Transport(object): - """ - Transport base class - """ - - def __init__(self, config=None, **kwargs): - self.api_config = config or get_configuration(**kwargs) - self.env_details = EnvironmentDetail(self.api_config) - self._transport = TransportTypes.get_transport( - self.api_config, - self.env_details, - ) - - def create_message(self, record): - # create message from record - return TransportTypes.create_message( - record, - self.api_config, - self.env_details, - ) + return DefaultTransport(api_config, env_details) - def create_group_message(self, messages): - # create group message from list of records - return TransportTypes.create_group_message( - messages, - self.api_config, - self.env_details, - ) - def send(self, group_message): - # send group message - try: - self._transport.send( - TransportTypes.get_log_url(self.api_config), - TransportTypes.prepare_message(self.api_config, group_message), - ) - except Exception as e: - internal_logger.error('Request error: {}'.format(e)) +def configure_transport(config=None, **kwargs): + # return which transport to use depending on users input + api_config = config or get_configuration(**kwargs) + env_details = EnvironmentDetail(api_config) + return TransportTypes.get_transport( + api_config, + env_details, + ) diff --git a/stackify/transport/agent/__init__.py b/stackify/transport/agent/__init__.py index 4ba8479..7ebc987 100644 --- a/stackify/transport/agent/__init__.py +++ b/stackify/transport/agent/__init__.py @@ -1 +1,30 @@ -from .agent_socket import AgentSocket # noqa +import logging + +from stackify.constants import AGENT_LOG_URL +from stackify.transport.agent import agent_http +from stackify.transport.agent import agent_socket +from stackify.transport.base import AgentBaseTransport + +internal_logger = logging.getLogger(__name__) + + +class AgentSocketTransport(AgentBaseTransport): + """ + Agent Socket Transport handles sending of logs using Unix Socket Domain + """ + + def __init__(self, api_config, env_details): + super(AgentSocketTransport, self).__init__(api_config, env_details) + self.url = api_config.socket_url + AGENT_LOG_URL + self._transport = agent_socket.AgentSocket() + + +class AgentHTTPTransport(AgentBaseTransport): + """ + Agent HTTP Transport handles sending of logs using HTTP requests + """ + + def __init__(self, api_config, env_details): + super(AgentHTTPTransport, self).__init__(api_config, env_details) + self.url = api_config.http_endpoint + AGENT_LOG_URL + self._transport = agent_http.AgentHTTP() diff --git a/stackify/transport/agent/agent_http.py b/stackify/transport/agent/agent_http.py new file mode 100644 index 0000000..1950594 --- /dev/null +++ b/stackify/transport/agent/agent_http.py @@ -0,0 +1,26 @@ +import logging +import requests +import retrying + +internal_logger = logging.getLogger(__name__) + + +class AgentHTTP(object): + """ + AgentHTTP class that handles HTTP post requests + """ + + def _post(self, url, payload): + headers = { + 'Content-Type': 'application/x-protobuf', + } + try: + return requests.post(url, payload, headers=headers) + except Exception as e: + internal_logger.debug('HTTP transport exception: {}.'.format(e)) + raise + + @retrying.retry(wait_exponential_multiplier=1000, stop_max_delay=32000) + def send(self, url, payload): + # send payload through socket domain using _post method + return self._post(url, payload) diff --git a/stackify/transport/agent/agent_socket.py b/stackify/transport/agent/agent_socket.py index aef8adc..af0db72 100644 --- a/stackify/transport/agent/agent_socket.py +++ b/stackify/transport/agent/agent_socket.py @@ -42,4 +42,4 @@ def _post(self, url, payload): @retrying.retry(wait_exponential_multiplier=1000, stop_max_delay=32000) def send(self, url, payload): # send payload through socket domain using _post method - self._post(url, payload) + return self._post(url, payload) diff --git a/stackify/transport/agent/message.py b/stackify/transport/agent/message.py index 444fb5f..3d6c340 100644 --- a/stackify/transport/agent/message.py +++ b/stackify/transport/agent/message.py @@ -122,4 +122,4 @@ def __init__(self, messages, api_config, env_details, logger=None): log_group.application_location = env_details.appLocation log_group.logger = logger or __name__ log_group.platform = 'python' - log_group.logs.MergeFrom(messages) + log_group.logs.extend(messages) diff --git a/stackify/transport/application.py b/stackify/transport/application.py index 5813f88..ac11947 100644 --- a/stackify/transport/application.py +++ b/stackify/transport/application.py @@ -3,7 +3,9 @@ from stackify.utils import arg_or_env from stackify.constants import API_URL +from stackify.constants import DEFAULT_HTTP_ENDPOINT from stackify.constants import SOCKET_URL +from stackify.constants import TRANSPORT_TYPE_AGENT_HTTP from stackify.constants import TRANSPORT_TYPE_AGENT_SOCKET from stackify.constants import TRANSPORT_TYPE_DEFAULT from stackify.transport.default.formats import JSONObject @@ -27,12 +29,22 @@ class ApiConfiguration: ApiConfiguration class that stores application configurations """ - def __init__(self, api_key, application, environment, api_url=API_URL, socket_url=SOCKET_URL, transport=None): + def __init__( + self, + api_key, + application, + environment, + api_url=API_URL, + socket_url=SOCKET_URL, + transport=None, + http_endpoint=DEFAULT_HTTP_ENDPOINT, + ): self.api_key = api_key self.api_url = api_url self.application = application self.environment = environment self.socket_url = socket_url + self.http_endpoint = http_endpoint self.transport = transport @@ -43,7 +55,8 @@ def get_configuration(**kwargs): """ transport = arg_or_env('transport', kwargs, TRANSPORT_TYPE_DEFAULT) - if transport == TRANSPORT_TYPE_AGENT_SOCKET: + + if transport in [TRANSPORT_TYPE_AGENT_SOCKET, TRANSPORT_TYPE_AGENT_HTTP]: api_key = arg_or_env('api_key', kwargs, '') else: api_key = arg_or_env('api_key', kwargs) @@ -54,5 +67,6 @@ def get_configuration(**kwargs): api_key=api_key, api_url=arg_or_env('api_url', kwargs, API_URL), socket_url=arg_or_env('socket_url', kwargs, SOCKET_URL), + http_endpoint=arg_or_env('http_endpoint', kwargs, DEFAULT_HTTP_ENDPOINT, env_key='STACKIFY_TRANSPORT_HTTP_ENDPOINT'), transport=transport, ) diff --git a/stackify/transport/base.py b/stackify/transport/base.py new file mode 100644 index 0000000..15c897e --- /dev/null +++ b/stackify/transport/base.py @@ -0,0 +1,40 @@ +from stackify.transport.agent.message import Log +from stackify.transport.agent.message import LogGroup + + +class BaseTransport(object): + """ + Base Transport + """ + def __init__(self, api_config, env_details): + self._api_config = api_config + self._env_details = env_details + + def create_message(self, record): + raise NotImplementedError + + def create_group_message(self, messages): + raise NotImplementedError + + def send(self, group_message): + raise NotImplementedError + + +class AgentBaseTransport(BaseTransport): + """ + Base Transport for protobuf data + """ + url = None + _transport = None + + def __init__(self, api_config, env_details): + super(AgentBaseTransport, self).__init__(api_config, env_details) + + def create_message(self, record): + return Log(record, self._api_config, self._env_details).get_object() + + def create_group_message(self, messages): + return LogGroup(messages, self._api_config, self._env_details).get_object() + + def send(self, group_message): + return self._transport.send(self.url, group_message.SerializeToString()) diff --git a/stackify/transport/default/__init__.py b/stackify/transport/default/__init__.py index 419fdfd..e4711a9 100644 --- a/stackify/transport/default/__init__.py +++ b/stackify/transport/default/__init__.py @@ -1 +1,30 @@ -from .http import HTTPClient # noqa +from stackify.constants import LOG_SAVE_URL +from stackify.transport.base import BaseTransport +from stackify.transport.default.http import HTTPClient +from stackify.transport.default.log import LogMsg +from stackify.transport.default.log import LogMsgGroup + + +class DefaultTransport(BaseTransport): + """ + Default Transport handles sending of logs directly to platform + """ + _transport = None + + def __init__(self, api_config, env_details): + super(DefaultTransport, self).__init__(api_config, env_details) + self._transport = HTTPClient(api_config, env_details) + + def create_message(self, record): + msg = LogMsg() + msg.from_record(record) + return msg + + def create_group_message(self, messages): + return LogMsgGroup(messages) + + def send(self, group_message): + self._transport.send( + LOG_SAVE_URL, + group_message, + ) diff --git a/stackify/utils.py b/stackify/utils.py index 5eb8c1e..3a6ba54 100644 --- a/stackify/utils.py +++ b/stackify/utils.py @@ -1,8 +1,8 @@ import os -def arg_or_env(name, args, default=None): - env_name = 'STACKIFY_{0}'.format(name.upper()) +def arg_or_env(name, args, default=None, env_key=None): + env_name = env_key or 'STACKIFY_{0}'.format(name.upper()) try: value = args.get(name) if not value: diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..8967876 --- /dev/null +++ b/test.sh @@ -0,0 +1,47 @@ +#!/bin/bash +set -e + + +VERSIONS=('2.7' '3.4' '3.5' '3.6' '3.7' '3.8') + + +function runFlake8() { + echo '<--------------------------------------------->' + # run flake8 and exit on error + # it will check the code base against coding style (PEP8) and programming errors + echo "Running flake8..." + flake8 || { echo 'You increased the number of flak8 errors'; exit 1; } +} + +function runPyTest() { + echo '<--------------------------------------------->' + python_version=${1} + test_venv="venv_test_${python_version//.}" + + echo "Creating virtualenv ${test_venv}..." + virtualenv -p python${python_version} ${test_venv} + + echo "Activating virtualenv ${test_venv}..." + source ${test_venv}/bin/activate + + echo 'Installing dependencies...' + pip install -r requirements.txt + + runFlake8 + + echo 'Running pytest...' + py.test + + echo "Deactivating virtualenv..." + deactivate +} + +echo 'Removing all existing virtualenv' +rm -rf venv_test_* | true + +for i in "${VERSIONS[@]}" +do + runPyTest ${i} +done + +echo 'Done' diff --git a/tests/bases.py b/tests/bases.py index d495602..0b11585 100644 --- a/tests/bases.py +++ b/tests/bases.py @@ -1,9 +1,11 @@ +import collections +import logging import os import retrying import unittest - old_retry = retrying.retry +_LoggingWatcher = collections.namedtuple("_LoggingWatcher", ["records", "output"]) class ClearEnvTest(unittest.TestCase): @@ -20,6 +22,7 @@ def setUp(self): 'STACKIFY_API_KEY', 'STACKIFY_API_URL', 'STACKIFY_TRANSPORT', + 'STACKIFY_TRANSPORT_HTTP_ENDPOINT', ] self.saved = {} for key in to_save: @@ -43,3 +46,100 @@ def inner(func): return old_retry(*args, **kwargs)(func) return inner return fake_retry + + +class _BaseTestCaseContext(object): + + def __init__(self, test_case): + self.test_case = test_case + + def _raiseFailure(self, standardMsg): + msg = self.test_case._formatMessage(self.msg, standardMsg) + raise self.test_case.failureException(msg) + + +class _CapturingHandler(logging.Handler): + """ + A logging handler capturing all (raw and formatted) logging output. + """ + + def __init__(self): + logging.Handler.__init__(self) + self.watcher = _LoggingWatcher([], []) + + def flush(self): + pass + + def emit(self, record): + self.watcher.records.append(record) + msg = self.format(record) + self.watcher.output.append(msg) + + +class _AssertLogsContext(_BaseTestCaseContext): + """A context manager used to implement TestCase.assertLogs().""" + + LOGGING_FORMAT = "%(levelname)s:%(name)s:%(message)s" + + def __init__(self, test_case, logger_name, level): + _BaseTestCaseContext.__init__(self, test_case) + self.logger_name = logger_name + if level: + self.level = logging._levelNames.get(level, level) + else: + self.level = logging.INFO + self.msg = None + + def __enter__(self): + if isinstance(self.logger_name, logging.Logger): + logger = self.logger = self.logger_name + else: + logger = self.logger = logging.getLogger(self.logger_name) + formatter = logging.Formatter(self.LOGGING_FORMAT) + handler = _CapturingHandler() + handler.setFormatter(formatter) + self.watcher = handler.watcher + self.old_handlers = logger.handlers[:] + self.old_level = logger.level + self.old_propagate = logger.propagate + logger.handlers = [handler] + logger.setLevel(self.level) + logger.propagate = False + return handler.watcher + + def __exit__(self, exc_type, exc_value, tb): + self.logger.handlers = self.old_handlers + self.logger.propagate = self.old_propagate + self.logger.setLevel(self.old_level) + if exc_type is not None: + # let unexpected exceptions pass through + return False + if len(self.watcher.records) == 0: + self._raiseFailure( + "no logs of level {} or higher triggered on {}" + .format(logging.getLevelName(self.level), self.logger.name)) + + +class LogTestCase(unittest.TestCase): + + def assertLogs(self, logger=None, level=None): + """Fail unless a log message of level *level* or higher is emitted + on *logger_name* or its children. If omitted, *level* defaults to + INFO and *logger* defaults to the root logger. + + This method must be used as a context manager, and will yield + a recording object with two attributes: `output` and `records`. + At the end of the context manager, the `output` attribute will + be a list of the matching formatted log messages and the + `records` attribute will be a list of the corresponding LogRecord + objects. + + Example:: + + with self.assertLogs('foo', level='INFO') as cm: + logging.getLogger('foo').info('first message') + logging.getLogger('foo.bar').error('second message') + self.assertEqual(cm.output, ['INFO:foo:first message', + 'ERROR:foo.bar:second message']) + """ + return _AssertLogsContext(self, logger, level) diff --git a/tests/test_handler.py b/tests/test_handler.py index d915a14..2f35d05 100644 --- a/tests/test_handler.py +++ b/tests/test_handler.py @@ -49,7 +49,7 @@ def setUp(self): # don't print warnings on http crashes, so mute stackify logger logging.getLogger('stackify').propagate = False - @patch('stackify.transport.Transport.create_message') + @patch('stackify.transport.default.DefaultTransport.create_message') @patch('stackify.transport.default.http.HTTPClient.POST') def test_not_identified(self, post, logmsg): '''The HTTPClient identifies automatically if needed''' @@ -58,8 +58,8 @@ def test_not_identified(self, post, logmsg): listener.send_group() self.assertTrue(listener.transport._transport.identified) - @patch('stackify.transport.Transport.create_message') - @patch('stackify.transport.Transport.create_group_message') + @patch('stackify.transport.default.DefaultTransport.create_message') + @patch('stackify.transport.default.DefaultTransport.create_group_message') @patch('stackify.transport.default.http.HTTPClient.POST') def test_send_group_if_needed(self, post, logmsggroup, logmsg): '''The listener sends groups of messages''' @@ -78,7 +78,7 @@ def test_send_group_if_needed(self, post, logmsggroup, logmsg): self.assertEqual(post.call_count, 1) self.assertEqual(len(listener.messages), 1) - @patch('stackify.transport.Transport.create_message') + @patch('stackify.transport.default.DefaultTransport.create_message') @patch('stackify.handler.StackifyListener.send_group') def test_clear_queue_shutdown(self, send_group, logmsg): '''The listener sends the leftover messages on the queue when shutting down''' @@ -92,8 +92,8 @@ def test_clear_queue_shutdown(self, send_group, logmsg): listener.stop() self.assertTrue(send_group.called) - @patch('stackify.transport.Transport.create_message') - @patch('stackify.transport.Transport.create_group_message') + @patch('stackify.transport.default.DefaultTransport.create_message') + @patch('stackify.transport.default.DefaultTransport.create_group_message') @patch('stackify.transport.default.http.HTTPClient.send_log_group') def test_send_group_crash(self, send_log_group, logmsggroup, logmsg): '''The listener drops messages after retrying''' diff --git a/tests/test_init.py b/tests/test_init.py index 547f109..7e69676 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -43,7 +43,7 @@ def test_logger_no_config(self): logger = stackify.getLogger(auto_shutdown=False) self.loggers.append(logger) - config = logger.handlers[0].listener.transport.api_config + config = logger.handlers[0].listener.transport._api_config self.assertEqual(config.application, 'test2_appname') self.assertEqual(config.environment, 'test2_environment') @@ -55,7 +55,7 @@ def test_logger_api_config(self): logger = stackify.getLogger(config=self.config, auto_shutdown=False) self.loggers.append(logger) - config = logger.handlers[0].listener.transport.api_config + config = logger.handlers[0].listener.transport._api_config self.assertEqual(config.application, 'test_appname') self.assertEqual(config.environment, 'test_environment') @@ -79,7 +79,7 @@ def test_get_logger_defaults(self): self.loggers.append(logger) handler = logger.handlers[0] - config = handler.listener.transport.api_config + config = handler.listener.transport._api_config self.assertEqual(logger.name, 'tests.test_init') self.assertEqual(config.api_url, stackify.constants.API_URL) diff --git a/tests/test_timer.py b/tests/test_timer.py new file mode 100644 index 0000000..09d8f9e --- /dev/null +++ b/tests/test_timer.py @@ -0,0 +1,32 @@ +import time +from unittest import TestCase +try: + from unittest import mock +except Exception: + import mock + +from stackify.timer import RepeatedTimer + + +class TimerTest(TestCase): + def setUp(self): + self.function_mock = mock.Mock() + self.timer = RepeatedTimer(0.1, self.function_mock) + self.timer.start() + + def shutDown(self): + self.timer.stop() + + def test_start(self): + assert self.timer._started + + def test_stop(self): + self.timer.stop() + + assert not self.timer._started + + def test_timer(self): + time.sleep(0.3) + + assert self.function_mock.called + assert self.function_mock.call_count >= 2 diff --git a/tests/transport/agent/test_init.py b/tests/transport/agent/test_init.py new file mode 100644 index 0000000..369af49 --- /dev/null +++ b/tests/transport/agent/test_init.py @@ -0,0 +1,111 @@ +import imp +import logging +import retrying +from unittest import TestCase +from mock import patch + +import stackify +from stackify.protos import stackify_agent_pb2 +from stackify.transport import application +from stackify.transport.agent import AgentHTTPTransport +from stackify.transport.agent import AgentSocketTransport +from tests.bases import fake_retry_decorator + + +class AgentSocketTransportTest(TestCase): + def setUp(self): + self.config = application.ApiConfiguration( + application='test_appname', + environment='test_environment', + api_key='test_apikey', + api_url='test_apiurl', + ) + self.env_details = application.EnvironmentDetail(self.config) + self.agent_socket_transport = AgentSocketTransport(self.config, self.env_details) + + def test_init(self): + assert self.agent_socket_transport._api_config == self.config + assert self.agent_socket_transport._env_details == self.env_details + + def test_create_message(self): + message = self.agent_socket_transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + + assert isinstance(message, stackify_agent_pb2.LogGroup.Log) + + def test_create_group_message(self): + message = self.agent_socket_transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + + group_message = self.agent_socket_transport.create_group_message([message]) + + assert isinstance(group_message, stackify_agent_pb2.LogGroup) + + @patch('requests_unixsocket.Session.post') + def test_send(self, mock_post): + message = self.agent_socket_transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + group_message = self.agent_socket_transport.create_group_message([message]) + + self.agent_socket_transport.send(group_message) + + assert mock_post.called + assert mock_post.call_args_list[0][0][0] == 'http+unix://%2Fusr%2Flocal%2Fstackify%2Fstackify.sock/log' + assert mock_post.call_args_list[0][1]['headers']['Content-Type'] == 'application/x-protobuf' + + +class AgentHTTPTransportTest(TestCase): + @classmethod + def setUpClass(self): + retrying.retry = fake_retry_decorator(3) + imp.reload(stackify.transport.agent.agent_http) + + @classmethod + def tearDownClass(self): + imp.reload(retrying) + imp.reload(stackify.transport.agent.agent_http) + + def setUp(self): + self.config = application.ApiConfiguration( + application='test_appname', + environment='test_environment', + api_key='test_apikey', + api_url='test_apiurl', + ) + self.env_details = application.EnvironmentDetail(self.config) + self.agent_http_transport = AgentHTTPTransport(self.config, self.env_details) + + def test_init(self): + assert self.agent_http_transport._api_config == self.config + assert self.agent_http_transport._env_details == self.env_details + + def test_create_message(self): + message = self.agent_http_transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + + assert isinstance(message, stackify_agent_pb2.LogGroup.Log) + + def test_create_group_message(self): + message = self.agent_http_transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + + group_message = self.agent_http_transport.create_group_message([message]) + + assert isinstance(group_message, stackify_agent_pb2.LogGroup) + + @patch('requests.post') + def test_send(self, mock_post): + message = self.agent_http_transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + group_message = self.agent_http_transport.create_group_message([message]) + + self.agent_http_transport.send(group_message) + + assert mock_post.called + assert mock_post.call_args_list[0][0][0] == 'https://localhost:10601/log' + assert mock_post.call_args_list[0][1]['headers']['Content-Type'] == 'application/x-protobuf' + + @patch('requests.post') + def test_retry(self, mock_post): + mock_post.side_effect = Exception('some error') + message = self.agent_http_transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + group_message = self.agent_http_transport.create_group_message([message]) + + with self.assertRaises(Exception): + self.agent_http_transport.send(group_message) + + assert mock_post.call_count == 3 diff --git a/tests/transport/agent/test_message.py b/tests/transport/agent/test_message.py index 7f73886..690f3a0 100644 --- a/tests/transport/agent/test_message.py +++ b/tests/transport/agent/test_message.py @@ -1,11 +1,16 @@ import logging -from unittest import TestCase +import sys from stackify.protos import stackify_agent_pb2 from stackify.transport import application from stackify.transport.agent.message import Log from stackify.transport.agent.message import LogGroup +if sys.version_info[0] == 2: + from tests.bases import LogTestCase as TestCase +else: + from unittest import TestCase + class TestLog(TestCase): def setUp(self): @@ -25,6 +30,18 @@ def test_get_object(self): assert isinstance(log, stackify_agent_pb2.LogGroup.Log) + def test_get_object_with_trans_id_and_log_id(self): + with self.assertLogs('foo', level='INFO') as logging_watcher: + logging.getLogger('foo').info('some log') + record = logging_watcher.records[0] + record.trans_id = 'trans_id' + record.log_id = 'log_id' + + log = Log(record, self.config, self.env_details).get_object() + + assert log.id == 'log_id' + assert log.transaction_id == 'trans_id' + def test_info_log_details(self): with self.assertLogs('foo', level='INFO') as logging_watcher: logging.getLogger('foo').info('some log') @@ -83,7 +100,7 @@ def test_info_exception_details(self): assert environment_detail.application_location == self.env_details.appLocation error_item = error.error_item - assert error_item.message == 'division by zero' + assert error_item.message in ['integer division or modulo by zero', 'division by zero'] assert error_item.error_type == 'ZeroDivisionError' assert error_item.source_method == 'test_info_exception_details' assert len(error_item.stacktrace) diff --git a/tests/transport/default/test_init.py b/tests/transport/default/test_init.py new file mode 100644 index 0000000..f3e410f --- /dev/null +++ b/tests/transport/default/test_init.py @@ -0,0 +1,47 @@ +import logging +from unittest import TestCase +from mock import patch + +from stackify.transport import application +from stackify.transport.default import DefaultTransport +from stackify.transport.default.log import LogMsg +from stackify.transport.default.log import LogMsgGroup + + +class AgentSocketTransportTest(TestCase): + def setUp(self): + self.config = application.ApiConfiguration( + application='test_appname', + environment='test_environment', + api_key='test_apikey', + api_url='test_apiurl', + ) + self.env_details = application.EnvironmentDetail(self.config) + self.default_transport = DefaultTransport(self.config, self.env_details) + + def test_init(self): + assert self.default_transport._api_config == self.config + assert self.default_transport._env_details == self.env_details + + def test_create_message(self): + message = self.default_transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + + assert isinstance(message, LogMsg) + + def test_create_group_message(self): + message = self.default_transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + + group_message = self.default_transport.create_group_message([message]) + + assert isinstance(group_message, LogMsgGroup) + + @patch('requests.post') + def test_send(self, mock_post): + message = self.default_transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + group_message = self.default_transport.create_group_message([message]) + + self.default_transport.send(group_message) + + assert mock_post.called + assert mock_post.call_args_list[0][0][0] == 'test_apiurl/Metrics/IdentifyApp' + assert mock_post.call_args_list[0][1]['headers']['Content-Type'] == 'application/json' diff --git a/tests/transport/test_application.py b/tests/transport/test_application.py index 901872f..7b4dc62 100644 --- a/tests/transport/test_application.py +++ b/tests/transport/test_application.py @@ -3,10 +3,15 @@ """ import unittest +import os from mock import patch from tests.bases import ClearEnvTest from stackify.constants import API_URL +from stackify.constants import DEFAULT_HTTP_ENDPOINT +from stackify.constants import TRANSPORT_TYPE_AGENT_HTTP +from stackify.constants import TRANSPORT_TYPE_AGENT_SOCKET +from stackify.constants import TRANSPORT_TYPE_DEFAULT from stackify.transport.application import get_configuration @@ -145,5 +150,73 @@ def test_api_key_is_not_required_on_agent_socket_transport(self): self.assertEqual(config.transport, 'agent_socket') +class ConfigEnvironmentVariableTest(ClearEnvTest): + def test_transport_environment_variable_default(self): + os.environ["STACKIFY_TRANSPORT"] = "default" + + config = get_configuration( + application='test_appname', + environment='test_environment', + api_key='test_apikey', + api_url='test_apiurl', + ) + + assert config.transport == TRANSPORT_TYPE_DEFAULT + + del os.environ["STACKIFY_TRANSPORT"] + + def test_transport_environment_variable_agent_socket(self): + os.environ["STACKIFY_TRANSPORT"] = "agent_socket" + + config = get_configuration( + application='test_appname', + environment='test_environment', + api_key='test_apikey', + api_url='test_apiurl', + ) + + assert config.transport == TRANSPORT_TYPE_AGENT_SOCKET + + del os.environ["STACKIFY_TRANSPORT"] + + def test_transport_environment_variable_agent_http(self): + os.environ["STACKIFY_TRANSPORT"] = "agent_http" + + config = get_configuration( + application='test_appname', + environment='test_environment', + api_key='test_apikey', + api_url='test_apiurl', + ) + + assert config.transport == TRANSPORT_TYPE_AGENT_HTTP + + del os.environ["STACKIFY_TRANSPORT"] + + def test_http_endpoint_environment_variable_default(self): + config = get_configuration( + application='test_appname', + environment='test_environment', + api_key='test_apikey', + api_url='test_apiurl', + ) + + assert config.http_endpoint == DEFAULT_HTTP_ENDPOINT + + def test_http_endpoint_environment_variable(self): + os.environ["STACKIFY_TRANSPORT_HTTP_ENDPOINT"] = "test" + + config = get_configuration( + application='test_appname', + environment='test_environment', + api_key='test_apikey', + api_url='test_apiurl', + ) + + assert config.http_endpoint == "test" + + del os.environ["STACKIFY_TRANSPORT_HTTP_ENDPOINT"] + + if __name__ == '__main__': unittest.main() diff --git a/tests/transport/test_base.py b/tests/transport/test_base.py new file mode 100644 index 0000000..d94c891 --- /dev/null +++ b/tests/transport/test_base.py @@ -0,0 +1,84 @@ +import logging + +from tests.bases import ClearEnvTest +from stackify.protos import stackify_agent_pb2 +from stackify.transport.base import BaseTransport +from stackify.transport.base import AgentBaseTransport +from stackify.transport.application import EnvironmentDetail +from stackify.transport.application import get_configuration + +CONFIG = { + 'application': 'test_appname', + 'environment': 'test_environment', + 'api_key': 'test_apikey', + 'api_url': 'test_apiurl', +} + + +class BaseTransportTest(ClearEnvTest): + def test_init(self): + api_config = 'test_api_config' + env_details = 'test_env_details' + + base_transport = BaseTransport(api_config, env_details) + + assert base_transport._api_config == api_config + assert base_transport._env_details == env_details + + def test_create_message(self): + api_config = 'test_api_config' + env_details = 'test_env_details' + base_transport = BaseTransport(api_config, env_details) + + self.assertRaises(NotImplementedError, base_transport.create_message, 'test_record') + + def test_create_group_message(self): + api_config = 'test_api_config' + env_details = 'test_env_details' + base_transport = BaseTransport(api_config, env_details) + + self.assertRaises(NotImplementedError, base_transport.create_group_message, 'test_messages') + + def test_send(self): + api_config = 'test_api_config' + env_details = 'test_env_details' + base_transport = BaseTransport(api_config, env_details) + + self.assertRaises(NotImplementedError, base_transport.send, 'test_group_message') + + +class AgentBaseTransportTest(ClearEnvTest): + def test_init(self): + api_config = 'test_api_config' + env_details = 'test_env_details' + + agent_base_transport = AgentBaseTransport(api_config, env_details) + + assert agent_base_transport._api_config == api_config + assert agent_base_transport._env_details == env_details + + def test_create_message(self): + api_config = 'test_api_config' + env_details = 'test_env_details' + agent_base_transport = AgentBaseTransport(api_config, env_details) + + message = agent_base_transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + + assert isinstance(message, stackify_agent_pb2.LogGroup.Log) + + def test_create_group_message(self): + api_config = get_configuration(**CONFIG) + env_details = EnvironmentDetail(api_config) + agent_base_transport = AgentBaseTransport(api_config, env_details) + + message = agent_base_transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + group_message = agent_base_transport.create_group_message([message]) + + assert isinstance(group_message, stackify_agent_pb2.LogGroup) + + def test_send(self): + api_config = 'test_api_config' + env_details = 'test_env_details' + agent_base_transport = AgentBaseTransport(api_config, env_details) + + self.assertRaises(AttributeError, agent_base_transport.send, 'test_group_message') diff --git a/tests/transport/test_init.py b/tests/transport/test_init.py index 5910a2c..079d249 100644 --- a/tests/transport/test_init.py +++ b/tests/transport/test_init.py @@ -2,10 +2,13 @@ from mock import patch from tests.bases import ClearEnvTest +from stackify.constants import AGENT_LOG_URL +from stackify.constants import DEFAULT_HTTP_ENDPOINT from stackify.protos import stackify_agent_pb2 -from stackify.transport import Transport -from stackify.transport.agent import AgentSocket -from stackify.transport.default import HTTPClient +from stackify.transport import configure_transport +from stackify.transport.agent import AgentHTTPTransport +from stackify.transport.agent import AgentSocketTransport +from stackify.transport.default import DefaultTransport from stackify.transport.default.log import LogMsg from stackify.transport.default.log import LogMsgGroup @@ -20,9 +23,9 @@ def test_invalid_transport(self): 'transport': 'invalid', } - transport = Transport(**config) + transport = configure_transport(**config) - assert isinstance(transport._transport, HTTPClient) + assert isinstance(transport, DefaultTransport) def test_default_transport(self): config = { @@ -32,9 +35,9 @@ def test_default_transport(self): 'api_url': 'test_apiurl', } - transport = Transport(**config) + transport = configure_transport(**config) - assert isinstance(transport._transport, HTTPClient) + assert isinstance(transport, DefaultTransport) def test_default_create_message(self): config = { @@ -44,7 +47,7 @@ def test_default_create_message(self): 'api_url': 'test_apiurl', } - transport = Transport(**config) + transport = configure_transport(**config) message = transport.create_message(logging.makeLogRecord({'mgs': 'message'})) assert isinstance(message, LogMsg) @@ -57,7 +60,7 @@ def test_default_create_group_message(self): 'api_url': 'test_apiurl', } - transport = Transport(**config) + transport = configure_transport(**config) message = transport.create_message(logging.makeLogRecord({'mgs': 'message'})) group_message = transport.create_group_message([message]) @@ -72,7 +75,7 @@ def test_default_send_url(self, mock_send): 'api_url': 'test_apiurl', } - transport = Transport(**config) + transport = configure_transport(**config) message = transport.create_message(logging.makeLogRecord({'mgs': 'message'})) group_message = transport.create_group_message([message]) transport.send(group_message) @@ -90,9 +93,9 @@ def test_agent_socket_transport(self): 'transport': 'agent_socket', } - transport = Transport(**config) + transport = configure_transport(**config) - assert isinstance(transport._transport, AgentSocket) + assert isinstance(transport, AgentSocketTransport) def test_agent_socket_create_message(self): config = { @@ -104,10 +107,10 @@ def test_agent_socket_create_message(self): 'transport': 'agent_socket', } - transport = Transport(**config) + transport = configure_transport(**config) message = transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) - isinstance(message, stackify_agent_pb2.LogGroup.Log) + assert isinstance(message, stackify_agent_pb2.LogGroup.Log) def test_agent_socket_create_group_message(self): config = { @@ -119,7 +122,7 @@ def test_agent_socket_create_group_message(self): 'transport': 'agent_socket', } - transport = Transport(**config) + transport = configure_transport(**config) message = transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) group_message = transport.create_group_message([message]) @@ -136,7 +139,7 @@ def test_agent_socket_send_url(self, mock_send): 'transport': 'agent_socket', } - transport = Transport(**config) + transport = configure_transport(**config) message = transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) group_message = transport.create_group_message([message]) transport.send(group_message) @@ -154,10 +157,94 @@ def test_agent_socket_send_url_default(self, mock_send): 'transport': 'agent_socket', } - transport = Transport(**config) + transport = configure_transport(**config) message = transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) group_message = transport.create_group_message([message]) transport.send(group_message) assert mock_send.called assert mock_send.call_args_list[0][0][0] == 'http+unix://%2Fusr%2Flocal%2Fstackify%2Fstackify.sock/log' + + def test_agent_http_transport(self): + config = { + 'application': 'test_appname', + 'environment': 'test_environment', + 'api_key': 'test_apikey', + 'api_url': 'test_apiurl', + 'http_endpoint': 'test.url', + 'transport': 'agent_http', + } + + transport = configure_transport(**config) + + assert isinstance(transport, AgentHTTPTransport) + + def test_agent_http_create_message(self): + config = { + 'application': 'test_appname', + 'environment': 'test_environment', + 'api_key': 'test_apikey', + 'api_url': 'test_apiurl', + 'http_endpoint': 'test.url', + 'transport': 'agent_http', + } + + transport = configure_transport(**config) + message = transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + + assert isinstance(message, stackify_agent_pb2.LogGroup.Log) + + def test_agent_http_create_group_message(self): + config = { + 'application': 'test_appname', + 'environment': 'test_environment', + 'api_key': 'test_apikey', + 'api_url': 'test_apiurl', + 'http_endpoint': 'test.url', + 'transport': 'agent_http', + } + + transport = configure_transport(**config) + message = transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + group_message = transport.create_group_message([message]) + + assert isinstance(group_message, stackify_agent_pb2.LogGroup) + + @patch('stackify.transport.agent.agent_http.AgentHTTP.send') + def test_agent_http_send_url(self, mock_send): + config = { + 'application': 'test_appname', + 'environment': 'test_environment', + 'api_key': 'test_apikey', + 'api_url': 'test_apiurl', + 'http_endpoint': 'test.url', + 'transport': 'agent_http', + } + + transport = configure_transport(**config) + assert isinstance(transport, AgentHTTPTransport) + message = transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + group_message = transport.create_group_message([message]) + transport.send(group_message) + + assert mock_send.called + assert mock_send.call_args_list[0][0][0] == 'test.url/log' + + @patch('stackify.transport.agent.agent_http.AgentHTTP.send') + def test_agent_http_send_url_default(self, mock_send): + config = { + 'application': 'test_appname', + 'environment': 'test_environment', + 'api_key': 'test_apikey', + 'api_url': 'test_apiurl', + 'transport': 'agent_http', + } + + transport = configure_transport(**config) + assert isinstance(transport, AgentHTTPTransport) + message = transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + group_message = transport.create_group_message([message]) + transport.send(group_message) + + assert mock_send.called + assert mock_send.call_args_list[0][0][0] == DEFAULT_HTTP_ENDPOINT + AGENT_LOG_URL From af162e73df85b749d02dc9c39f8cd228fb2e9dcc Mon Sep 17 00:00:00 2001 From: jaygel Date: Tue, 3 Dec 2019 23:40:50 +0800 Subject: [PATCH 20/44] Python 3.8 support (#9) Changes: - update proper versions for pythons - add support for python 3.8 - add docket tests for different versions --- docker/stackify-python-api-test | 18 ++++++++++++++++++ setup.py | 13 ++++++++++++- test-docker-execute.sh | 22 ++++++++++++++++++++++ test-docker.sh | 29 +++++++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 docker/stackify-python-api-test create mode 100755 test-docker-execute.sh create mode 100755 test-docker.sh diff --git a/docker/stackify-python-api-test b/docker/stackify-python-api-test new file mode 100644 index 0000000..b40682a --- /dev/null +++ b/docker/stackify-python-api-test @@ -0,0 +1,18 @@ +ARG from_version + +FROM python:${from_version} + +ARG version + +RUN \ + apt-get update && \ + apt-get install -y flake8 && \ + pip install --upgrade pip && \ + python --version + +RUN mkdir /build +COPY . /build/ + +RUN cat /build/requirements.txt | xargs -n 1 pip install; exit 0 + +CMD /bin/bash -c "cd /build && source test-docker-execute.sh" diff --git a/setup.py b/setup.py index 42aee10..05cfeab 100755 --- a/setup.py +++ b/setup.py @@ -30,7 +30,18 @@ long_description=long_description, long_description_content_type="text/markdown", keywords=['logging', 'stackify', 'exception'], - classifiers=["Programming Language :: Python"], + classifiers=[ + "Programming Language :: Python", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Operating System :: OS Independent", + ], install_requires=[ 'protobuf>=3.9.1', 'retrying>=1.2.3', diff --git a/test-docker-execute.sh b/test-docker-execute.sh new file mode 100755 index 0000000..fc76dce --- /dev/null +++ b/test-docker-execute.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -e + +function runFlake8() { + echo '<--------------------------------------------->' + # run flake8 and exit on error + # it will check the code base against coding style (PEP8) and programming errors + echo "Running flake8..." + flake8 || { echo 'You have increased the number of flake8 errors'; exit 1; } +} + +function runPyTest() { + echo '<--------------------------------------------->' + echo "Python Version $(python --version)" + echo 'Running pytest...' + py.test +} + +runFlake8 + +runPyTest diff --git a/test-docker.sh b/test-docker.sh new file mode 100755 index 0000000..fca8a40 --- /dev/null +++ b/test-docker.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +set -e + +VERSIONS=('2.7' '3.4' '3.5' '3.6' '3.7' '3.8') +# VERSIONS=('2.7') + +for i in "${VERSIONS[@]}" +do + + if [[ "$(docker images -q stackify-python-api-test-${i}:latest 2> /dev/null)" != "" ]]; then + echo "Delete stackify-python-api-test-${i}..." + docker rm stackify-python-api-test-${i} &>/dev/null + docker rmi stackify-python-api-test-${i}:latest &>/dev/null + fi + + echo "Building stackify-python-api-test-${i}..." + docker build --no-cache --build-arg from_version=${i} --build-arg version=${i} --file docker/stackify-python-api-test . -t stackify-python-api-test-${i}:latest + + echo "Running stackify-python-api-test-${i}..." + docker run --network="host" --name "stackify-python-api-test-${i}" stackify-python-api-test-${i}:latest + + echo "Delete stackify-python-api-test-${i}..." + docker rm stackify-python-api-test-${i} &>/dev/null + docker rmi stackify-python-api-test-${i}:latest &>/dev/null + +done + +echo "Done" From 13ea9a8fcd7577ce794f80e69ab5d8c352a7df2b Mon Sep 17 00:00:00 2001 From: Darin Howard Date: Thu, 5 Dec 2019 10:13:52 -0600 Subject: [PATCH 21/44] updating version to 1.1.0 --- stackify/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackify/__init__.py b/stackify/__init__.py index 5896495..04c7081 100644 --- a/stackify/__init__.py +++ b/stackify/__init__.py @@ -1,7 +1,7 @@ """ Stackify Python API """ -__version__ = '1.0.5' +__version__ = '1.1.0' import logging import inspect From dd52a51a351fa62e47236932e226effe4b9f77d7 Mon Sep 17 00:00:00 2001 From: jaygel Date: Mon, 9 Dec 2019 23:46:00 +0800 Subject: [PATCH 22/44] Python API - Ignore SLL Error from HTTP transport (#10) --- stackify/transport/agent/agent_http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackify/transport/agent/agent_http.py b/stackify/transport/agent/agent_http.py index 1950594..4b47d73 100644 --- a/stackify/transport/agent/agent_http.py +++ b/stackify/transport/agent/agent_http.py @@ -15,7 +15,7 @@ def _post(self, url, payload): 'Content-Type': 'application/x-protobuf', } try: - return requests.post(url, payload, headers=headers) + return requests.post(url, payload, headers=headers, verify=False) except Exception as e: internal_logger.debug('HTTP transport exception: {}.'.format(e)) raise From 1777568a7cfc1cad462526a94c94530d949a32fe Mon Sep 17 00:00:00 2001 From: Darin Howard Date: Tue, 10 Dec 2019 10:24:40 -0600 Subject: [PATCH 23/44] updating version to 1.1.1 --- stackify/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackify/__init__.py b/stackify/__init__.py index 04c7081..e133b4b 100644 --- a/stackify/__init__.py +++ b/stackify/__init__.py @@ -1,7 +1,7 @@ """ Stackify Python API """ -__version__ = '1.1.0' +__version__ = '1.1.1' import logging import inspect From 1c750963e987dccdd9c974bc2257846a816fa40d Mon Sep 17 00:00:00 2001 From: jaygel Date: Thu, 19 Mar 2020 22:34:20 +0800 Subject: [PATCH 24/44] Fix issue with app name using default transport (#11) * Fix for App Name issue * fix issue with python3 Co-authored-by: Elpedio Adoptante Jr --- stackify/transport/default/http.py | 1 + test-docker.sh | 3 ++ tests/transport/default/test_init.py | 70 ++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+) diff --git a/stackify/transport/default/http.py b/stackify/transport/default/http.py index c619ad6..666deff 100644 --- a/stackify/transport/default/http.py +++ b/stackify/transport/default/http.py @@ -84,6 +84,7 @@ def identify_application(self): @retrying.retry(wait_exponential_multiplier=1000, stop_max_delay=32000) def send_log_group(self, url, group): internal_logger.debug('Sending logs by group') + group.AppName = self.environment_detail.configuredAppName group.Env = self.environment_detail.configuredEnvironmentName group.CDID = self.device_id group.CDAppID = self.device_app_id diff --git a/test-docker.sh b/test-docker.sh index fca8a40..5e0bbce 100755 --- a/test-docker.sh +++ b/test-docker.sh @@ -2,6 +2,9 @@ set -e +# remove caches +find . | grep -E "(__pycache__|\.pyc|\.pyo$)" | xargs rm -rf + VERSIONS=('2.7' '3.4' '3.5' '3.6' '3.7' '3.8') # VERSIONS=('2.7') diff --git a/tests/transport/default/test_init.py b/tests/transport/default/test_init.py index f3e410f..e6cdddd 100644 --- a/tests/transport/default/test_init.py +++ b/tests/transport/default/test_init.py @@ -1,6 +1,17 @@ +import gzip +import json import logging from unittest import TestCase from mock import patch +from requests.models import Response + +try: + from cStringIO import StringIO +except Exception: + try: + from StringIO import StringIO + except Exception: + pass # python 3, we use a new function in gzip from stackify.transport import application from stackify.transport.default import DefaultTransport @@ -8,6 +19,19 @@ from stackify.transport.default.log import LogMsgGroup +def parse_gzip_data(data): + if hasattr(gzip, 'decompress'): + return gzip.decompress(data).decode("utf-8") + else: + sio = StringIO() + sio.write(data) + sio.seek(0) + g = gzip.GzipFile(fileobj=sio, mode='rb') + transaction = g.read() + g.close() + return transaction.decode("utf-8") + + class AgentSocketTransportTest(TestCase): def setUp(self): self.config = application.ApiConfiguration( @@ -37,11 +61,57 @@ def test_create_group_message(self): @patch('requests.post') def test_send(self, mock_post): + res = Response() + res._content = json.dumps({ + 'DeviceID': None, + 'DeviceAppID': None, + 'AppNameID': 'test', + 'EnvID': 0, + 'Env': 'test_environment', + 'AppName': 'test_appname', + 'AppEnvID': 'test', + 'DeviceAlias': 'test' + }).encode('UTF-8') + mock_post.side_effect = [res, Response()] message = self.default_transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) group_message = self.default_transport.create_group_message([message]) self.default_transport.send(group_message) assert mock_post.called + assert mock_post.call_count == 2 assert mock_post.call_args_list[0][0][0] == 'test_apiurl/Metrics/IdentifyApp' assert mock_post.call_args_list[0][1]['headers']['Content-Type'] == 'application/json' + assert mock_post.call_args_list[1][0][0] == 'test_apiurl/Log/Save' + assert mock_post.call_args_list[1][1]['headers']['Content-Type'] == 'application/json' + + @patch('requests.post') + def test_json_data(self, mock_post): + res = Response() + res._content = json.dumps({ + 'DeviceID': None, + 'DeviceAppID': None, + 'AppNameID': 'test', + 'EnvID': 0, + 'Env': 'test_environment', + 'AppName': 'test_appname', + 'AppEnvID': 'test', + 'DeviceAlias': 'test' + }).encode('UTF-8') + mock_post.side_effect = [res, Response()] + message = self.default_transport.create_message(logging.makeLogRecord({'mgs': 'message', 'funcName': 'foo'})) + group_message = self.default_transport.create_group_message([message]) + + self.default_transport.send(group_message) + + assert mock_post.called + assert mock_post.call_count == 2 + assert mock_post.call_args_list[1][0][0] == 'test_apiurl/Log/Save' + + payload = json.loads(parse_gzip_data(mock_post.call_args_list[1][1]['data'])) + assert payload.get('AppName') == 'test_appname' + assert payload.get('Env') == 'test_environment' + assert payload.get('ServerName') + assert payload.get('AppNameID') == 'test' + assert payload.get('Logger') + assert payload.get('Msgs') From c2625f558c41dc6c79474226b1ca7766b192dd66 Mon Sep 17 00:00:00 2001 From: Darin Howard Date: Thu, 19 Mar 2020 09:35:50 -0500 Subject: [PATCH 25/44] updating to version 1.1.2 --- stackify/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackify/__init__.py b/stackify/__init__.py index e133b4b..630e76b 100644 --- a/stackify/__init__.py +++ b/stackify/__init__.py @@ -1,7 +1,7 @@ """ Stackify Python API """ -__version__ = '1.1.1' +__version__ = '1.1.2' import logging import inspect From 2ad68853ec0ebf3a67840a5997e540ffc8cd12ee Mon Sep 17 00:00:00 2001 From: Michael Mantos Date: Thu, 22 Jul 2021 01:27:28 +0800 Subject: [PATCH 26/44] INV-1719 - Initial fix for the circular issue for log messages --- docker/stackify-python-api-test | 1 - stackify/handler.py | 5 +- stackify/transport/agent/message.py | 4 +- stackify/transport/default/log.py | 5 +- stackify/utils.py | 23 ++++++ tests/test_handler.py | 19 +++++ tests/test_utils.py | 116 ++++++++++++++++++++++++++++ 7 files changed, 166 insertions(+), 7 deletions(-) create mode 100644 tests/test_utils.py diff --git a/docker/stackify-python-api-test b/docker/stackify-python-api-test index b40682a..d7d7957 100644 --- a/docker/stackify-python-api-test +++ b/docker/stackify-python-api-test @@ -6,7 +6,6 @@ ARG version RUN \ apt-get update && \ - apt-get install -y flake8 && \ pip install --upgrade pip && \ python --version diff --git a/stackify/handler.py b/stackify/handler.py index f957f8e..26b5fd2 100644 --- a/stackify/handler.py +++ b/stackify/handler.py @@ -76,7 +76,10 @@ def __init__(self, queue_, max_batch=MAX_BATCH, config=None, **kwargs): self._started = False def handle(self, record): - self.messages.append(self.transport.create_message(record)) + try: + self.messages.append(self.transport.create_message(record)) + except Exception: + internal_logger.exception('Could not handle log message: {}'.format(hasattr(record, 'getMessage') and record.getMessage() or str(record))) if len(self.messages) >= self.max_batch: self.send_group() diff --git a/stackify/transport/agent/message.py b/stackify/transport/agent/message.py index 3d6c340..bb418e2 100644 --- a/stackify/transport/agent/message.py +++ b/stackify/transport/agent/message.py @@ -1,9 +1,9 @@ -import json import sys import traceback from stackify.constants import RECORD_VARS from stackify.protos import stackify_agent_pb2 +from stackify.utils import data_to_json class BaseMessage(object): @@ -103,7 +103,7 @@ def __init__(self, record, api_config, env_details): if k not in RECORD_VARS} if data: - log.data = json.dumps(data, default=lambda x: hasattr(x, '__dict__') and x.__dict__ or x.__str__()) + log.data = data_to_json(data) if record.exc_info: log.error.MergeFrom(Error(record, api_config, env_details).get_object()) diff --git a/stackify/transport/default/log.py b/stackify/transport/default/log.py index 5b09da0..a501865 100644 --- a/stackify/transport/default/log.py +++ b/stackify/transport/default/log.py @@ -1,8 +1,7 @@ -import json - from stackify.constants import RECORD_VARS from stackify.transport.default.formats import JSONObject from stackify.transport.default.error import StackifyError +from stackify.utils import data_to_json class LogMsg(JSONObject): @@ -33,7 +32,7 @@ def from_record(self, record): if k not in RECORD_VARS} if data: - self.data = json.dumps(data, default=lambda x: hasattr(x, '__dict__') and x.__dict__ or x.__str__()) + self.data = data_to_json(data) if record.exc_info: self.Ex = StackifyError() diff --git a/stackify/utils.py b/stackify/utils.py index 3a6ba54..2da8944 100644 --- a/stackify/utils.py +++ b/stackify/utils.py @@ -1,4 +1,8 @@ import os +import json +import logging + +internal_logger = logging.getLogger(__name__) def arg_or_env(name, args, default=None, env_key=None): @@ -13,3 +17,22 @@ def arg_or_env(name, args, default=None, env_key=None): return default else: raise NameError('You must specify the keyword argument {0} or environment variable {1}'.format(name, env_name)) + + +def data_to_json(data): + try: + if object_is_iterable(data) and 'request' in data and hasattr(data['request'], '_messages'): + data['request'] = get_default_object(data['request']) + + return json.dumps(data, default=lambda x: get_default_object(x)) + except ValueError as e: + internal_logger.exception('Failed to serialize object to json: {} - Exception: {}'.format(data.__str__(), str(e))) + return json.dumps(data.__str__()) # String representation of the object + + +def get_default_object(obj): + return hasattr(obj, '__dict__') and obj.__dict__ or obj.__str__() + + +def object_is_iterable(obj): + return hasattr(obj, '__iter__') or isinstance(obj, str) diff --git a/tests/test_handler.py b/tests/test_handler.py index 2f35d05..acb3487 100644 --- a/tests/test_handler.py +++ b/tests/test_handler.py @@ -110,6 +110,25 @@ def test_send_group_crash(self, send_log_group, logmsggroup, logmsg): self.assertEqual(len(listener.messages), 1) self.assertEqual(send_log_group.call_count, 1) + @patch('stackify.transport.default.DefaultTransport.create_message') + @patch('stackify.transport.default.DefaultTransport.create_group_message') + @patch('stackify.transport.default.http.HTTPClient.send_log_group') + def test_create_message_crash(self, send_log_group, logmsggroup, logmsg): + '''The listener drops messages after retrying''' + listener = StackifyListener(queue_=Mock(), max_batch=3, config=self.config) + listener.transport._transport.identified = True + + logmsg.side_effect = Exception + + listener.handle(1) + listener.handle(2) + listener.handle(3) + self.assertEqual(len(listener.messages), 0) + listener.handle(4) + self.assertEqual(len(listener.messages), 0) # messages not created + self.assertEqual(logmsg.call_count, 4) # we called the function 4 times + self.assertEqual(send_log_group.call_count, 0) # since we have exceptions + if __name__ == '__main__': unittest.main() diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..9da2020 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,116 @@ +""" +Test the stackify.__init__ setup functions +""" + +import unittest +from mock import patch +from .bases import ClearEnvTest + +import stackify +import logging + + +class TestInit(ClearEnvTest): + ''' + Test the logger init functionality + ''' + + def setUp(self): + super(TestInit, self).setUp() + self.config = stackify.transport.application.ApiConfiguration( + application='test_appname', + environment='test_environment', + api_key='test_apikey', + api_url='test_apiurl') + self.loggers = [] + + def tearDown(self): + super(TestInit, self).tearDown() + global_loggers = logging.Logger.manager.loggerDict + for logger in self.loggers: + del global_loggers[logger.name] + + def test_utils_data_to_json_unserializable(self): + dummy = Dummy() + result = stackify.utils.data_to_json(dummy) + substring = ' Date: Thu, 22 Jul 2021 01:43:10 +0800 Subject: [PATCH 27/44] INV-1719 - Add dummy request test --- tests/test_utils.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 9da2020..70e4b78 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -82,7 +82,12 @@ def test_utils_data_to_json_dummy_object_circular(self, func): func.assert_called() substring = "'dummy': Date: Thu, 22 Jul 2021 01:56:31 +0800 Subject: [PATCH 28/44] Updating version to 1.1.3 beta 1 --- stackify/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackify/__init__.py b/stackify/__init__.py index 630e76b..7f88d76 100644 --- a/stackify/__init__.py +++ b/stackify/__init__.py @@ -1,7 +1,7 @@ """ Stackify Python API """ -__version__ = '1.1.2' +__version__ = '1.1.3b1' import logging import inspect From 1db5c4d50b10940f27b8d5682facddf03e66455b Mon Sep 17 00:00:00 2001 From: Michael Mantos Date: Tue, 27 Jul 2021 04:27:30 +0800 Subject: [PATCH 29/44] LNX-329 - Initial commit for RUMv2 support --- docker/stackify-python-api-test | 3 + stackify/compat.py | 74 ++++++++ stackify/config.py | 6 + stackify/constants.py | 3 + stackify/rum.py | 74 ++++++++ stackify/transport/application.py | 44 +++++ stackify/utils.py | 21 +++ test-docker.sh | 2 +- tests/bases.py | 2 + tests/test_compat.py | 38 ++++ tests/test_rum.py | 287 ++++++++++++++++++++++++++++++ tests/test_utils.py | 25 +++ 12 files changed, 578 insertions(+), 1 deletion(-) create mode 100644 stackify/compat.py create mode 100644 stackify/config.py create mode 100644 stackify/rum.py create mode 100644 tests/test_compat.py create mode 100644 tests/test_rum.py diff --git a/docker/stackify-python-api-test b/docker/stackify-python-api-test index d7d7957..5b8f88f 100644 --- a/docker/stackify-python-api-test +++ b/docker/stackify-python-api-test @@ -3,6 +3,8 @@ ARG from_version FROM python:${from_version} ARG version +ARG test +ARG test_repo RUN \ apt-get update && \ @@ -13,5 +15,6 @@ RUN mkdir /build COPY . /build/ RUN cat /build/requirements.txt | xargs -n 1 pip install; exit 0 +RUN if [ "${test}" = 1 ]; then pip install -i "${test_repo}" stackify-python-apm; fi; exit 0 CMD /bin/bash -c "cd /build && source test-docker-execute.sh" diff --git a/stackify/compat.py b/stackify/compat.py new file mode 100644 index 0000000..6aa68f7 --- /dev/null +++ b/stackify/compat.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +import sys +import types + +PY2 = sys.version_info[0] == 2 +PY3 = sys.version_info[0] == 3 + + +if PY2: + import StringIO + import Queue as queue # noqa F401 + import urlparse # noqa F401 + from urllib2 import HTTPError # noqa F401 + from urllib import unquote as unquote_core # noqa F401 + + StringIO = BytesIO = StringIO.StringIO + + string_types = (basestring,) # noqa F821 + integer_types = (int, long) # noqa F821 + class_types = (type, types.ClassType) + text_type = unicode # noqa F821 + binary_type = str + list_type = list + dict_type = dict + + def b(s): + return s + + def iterkeys(d, **kwargs): + return d.iterkeys(**kwargs) + + def iteritems(d, **kwargs): + return d.iteritems(**kwargs) + + def iterlists(d, **kwargs): + return d.iterlists(**kwargs) + + def unquote(*args, **kwargs): # noqa F811 + return unquote_core(*args, **kwargs) +else: + import io + import queue # noqa F401 + from urllib import parse as urlparse # noqa F401 + from urllib.error import HTTPError # noqa F401 + + StringIO = io.StringIO + BytesIO = io.BytesIO + + string_types = (str,) + integer_types = (int,) + class_types = (type,) + text_type = str + binary_type = bytes + list_type = list + dict_type = dict + + def b(s): + return s.encode("latin-1") + + def iterkeys(d, **kwargs): + return iter(d.keys(**kwargs)) + + def iteritems(d, **kwargs): + return iter(d.items(**kwargs)) + + def iterlists(d, **kwargs): + return iter(d.lists(**kwargs)) + + def unquote(*args, **kwargs): + return urlparse.unquote(*args, **kwargs) + + +def multidict_to_dict(d): + return dict((k, v[0] if len(v) == 1 else v) for k, v in iterlists(d)) diff --git a/stackify/config.py b/stackify/config.py new file mode 100644 index 0000000..58d5fa3 --- /dev/null +++ b/stackify/config.py @@ -0,0 +1,6 @@ +from stackify.constants import DEFAULT_RUM_KEY, DEFAULT_RUM_SCRIPT_URL + +rum_key = DEFAULT_RUM_KEY +rum_script_url = DEFAULT_RUM_SCRIPT_URL +application = None +environment = None diff --git a/stackify/constants.py b/stackify/constants.py index 3dce49e..c560063 100644 --- a/stackify/constants.py +++ b/stackify/constants.py @@ -38,3 +38,6 @@ TRANSPORT_TYPE_DEFAULT = 'default' TRANSPORT_TYPE_AGENT_SOCKET = 'agent_socket' TRANSPORT_TYPE_AGENT_HTTP = 'agent_http' + +DEFAULT_RUM_SCRIPT_URL = "https://stckjs.stackify.com/stckjs.js" +DEFAULT_RUM_KEY = "" diff --git a/stackify/rum.py b/stackify/rum.py new file mode 100644 index 0000000..6122bf4 --- /dev/null +++ b/stackify/rum.py @@ -0,0 +1,74 @@ +import json +import base64 +from stackify import config + +apm_installed = False + +try: + apm_installed = True + from stackifyapm import insert_rum_script as insert_rum_script_apm +except ImportError: + pass + + +def insert_rum_script(): + if apm_installed is True: + return insert_rum_script_apm() + + rum_key = config.rum_key + rum_script_url = config.rum_script_url + + if not rum_script_url or not rum_key: + return None + + transaction_id = get_transaction_id() + if not transaction_id: + return None + + reporting_url = get_reporting_url() + if not reporting_url: + return None + + application_name = config.application + if not application_name: + return None + + environment = config.environment + if not environment: + return None + + settings = { + "ID": transaction_id + } + + if application_name: + application_name_b64 = base64.b64encode(application_name.encode("utf-8")).decode("utf-8") + if (application_name_b64): + settings["Name"] = application_name_b64 + + if environment: + environment_b64 = base64.b64encode(environment.encode("utf-8")).decode("utf-8") + if (environment_b64): + settings["Env"] = environment_b64 + + if reporting_url: + reporting_url_b64 = base64.b64encode(reporting_url.encode("utf-8")).decode("utf-8") + if (reporting_url_b64): + settings["Trans"] = reporting_url_b64 + + if not settings: + return None + + return ''.format( + json.dumps(settings), + rum_script_url, + rum_key + ) + + +def get_transaction_id(): + return '' + + +def get_reporting_url(): + return '' diff --git a/stackify/transport/application.py b/stackify/transport/application.py index ac11947..bcbdd66 100644 --- a/stackify/transport/application.py +++ b/stackify/transport/application.py @@ -1,5 +1,6 @@ import socket import os +import logging from stackify.utils import arg_or_env from stackify.constants import API_URL @@ -9,6 +10,12 @@ from stackify.constants import TRANSPORT_TYPE_AGENT_SOCKET from stackify.constants import TRANSPORT_TYPE_DEFAULT from stackify.transport.default.formats import JSONObject +from stackify.constants import DEFAULT_RUM_SCRIPT_URL +from stackify.constants import DEFAULT_RUM_KEY +from stackify.utils import RegexValidator, ConfigError +from stackify import config + +internal_logger = logging.getLogger(__name__) class EnvironmentDetail(JSONObject): @@ -38,6 +45,8 @@ def __init__( socket_url=SOCKET_URL, transport=None, http_endpoint=DEFAULT_HTTP_ENDPOINT, + rum_script_url=DEFAULT_RUM_SCRIPT_URL, + rum_key=DEFAULT_RUM_KEY ): self.api_key = api_key self.api_url = api_url @@ -47,6 +56,39 @@ def __init__( self.http_endpoint = http_endpoint self.transport = transport + self.rum_script_url = DEFAULT_RUM_SCRIPT_URL + self.rum_key = DEFAULT_RUM_KEY + + # Rum config validation + if rum_script_url != DEFAULT_RUM_SCRIPT_URL: + self.validate( + RegexValidator("^((((https?|ftps?|gopher|telnet|nntp)://)|(mailto:|news:))(%[0-9A-Fa-f]{2}|[-\(\)_.!~*';/?:@&=+$,A-Za-z0-9])+)([).!';/?:,][\[:blank:|:blank:\]])?$"), + rum_script_url, + 'rum_script_url' + ) + config.rum_script_url = self.rum_script_url + + if rum_key != DEFAULT_RUM_KEY: + self.validate( + RegexValidator("^[A-Za-z0-9_-]+$"), + rum_key, + 'rum_key' + ) + config.rum_key = self.rum_key + + config.environment = self.environment + config.application = self.application + + def validate(self, validator, value, key): + if not validator: + return + + try: + value = validator(value, key) + setattr(self, key, str(value)) + except ConfigError as e: + internal_logger.exception(str(e)) + def get_configuration(**kwargs): """ @@ -69,4 +111,6 @@ def get_configuration(**kwargs): socket_url=arg_or_env('socket_url', kwargs, SOCKET_URL), http_endpoint=arg_or_env('http_endpoint', kwargs, DEFAULT_HTTP_ENDPOINT, env_key='STACKIFY_TRANSPORT_HTTP_ENDPOINT'), transport=transport, + rum_script_url=arg_or_env('rum_script_url', kwargs, DEFAULT_RUM_SCRIPT_URL, env_key='RETRACE_RUM_SCRIPT_URL'), + rum_key=arg_or_env('rum_key', kwargs, DEFAULT_RUM_KEY, env_key='RETRACE_RUM_KEY') ) diff --git a/stackify/utils.py b/stackify/utils.py index 2da8944..0a092a3 100644 --- a/stackify/utils.py +++ b/stackify/utils.py @@ -1,6 +1,8 @@ import os import json import logging +import re +from stackify import compat internal_logger = logging.getLogger(__name__) @@ -36,3 +38,22 @@ def get_default_object(obj): def object_is_iterable(obj): return hasattr(obj, '__iter__') or isinstance(obj, str) + + +class RegexValidator(object): + def __init__(self, regex, verbose_pattern=None): + self.regex = regex + self.verbose_pattern = verbose_pattern or regex + + def __call__(self, value, field_name): + value = compat.text_type(value) + match = re.match(self.regex, value) + if match: + return value + raise ConfigError("{} does not match pattern {}".format(value, self.verbose_pattern), field_name) + + +class ConfigError(ValueError): + def __init__(self, msg, field_name): + self.field_name = field_name + super(ValueError, self).__init__(msg) diff --git a/test-docker.sh b/test-docker.sh index 5e0bbce..e4f4dea 100755 --- a/test-docker.sh +++ b/test-docker.sh @@ -18,7 +18,7 @@ do fi echo "Building stackify-python-api-test-${i}..." - docker build --no-cache --build-arg from_version=${i} --build-arg version=${i} --file docker/stackify-python-api-test . -t stackify-python-api-test-${i}:latest + docker build --no-cache --build-arg from_version=${i} --build-arg version=${i} --build-arg test=${TEST} --build-arg test_repo=${TEST_REPO} --file docker/stackify-python-api-test . -t stackify-python-api-test-${i}:latest echo "Running stackify-python-api-test-${i}..." docker run --network="host" --name "stackify-python-api-test-${i}" stackify-python-api-test-${i}:latest diff --git a/tests/bases.py b/tests/bases.py index 0b11585..d7298bd 100644 --- a/tests/bases.py +++ b/tests/bases.py @@ -23,6 +23,8 @@ def setUp(self): 'STACKIFY_API_URL', 'STACKIFY_TRANSPORT', 'STACKIFY_TRANSPORT_HTTP_ENDPOINT', + 'RETRACE_RUM_SCRIPT_URL', + 'RETRACE_RUM_KEY' ] self.saved = {} for key in to_save: diff --git a/tests/test_compat.py b/tests/test_compat.py new file mode 100644 index 0000000..9f97bbb --- /dev/null +++ b/tests/test_compat.py @@ -0,0 +1,38 @@ +from unittest import TestCase + +from stackifyapm.utils.compat import b +from stackifyapm.utils.compat import iterkeys +from stackifyapm.utils.compat import iteritems + + +class CompatTest(TestCase): + def setUp(self): + self.dict_data = { + "key1": "value1", + "key2": "value2" + } + self.list_data = ["foo", "bar"] + + def test_convert_string_to_byte(self): + byte = '1' + + value = b(byte) + + assert isinstance(value, bytes) + + def test_iterkeys_should_return_iterator(self): + iter_keys = iterkeys(self.dict_data) + + self.assert_instance_is_an_iterator(iter_keys) + + def test_iteritems_should_return_iterator(self): + iter_items = iteritems(self.dict_data) + + self.assert_instance_is_an_iterator(iter_items) + + def assert_instance_is_an_iterator(self, item): + try: + iter(item) + assert True + except Exception: + assert False diff --git a/tests/test_rum.py b/tests/test_rum.py new file mode 100644 index 0000000..bb9cc82 --- /dev/null +++ b/tests/test_rum.py @@ -0,0 +1,287 @@ +try: + from unittest import mock +except Exception: + import mock # noqa F401 + +import stackify +import os +import stackify.rum +import base64 +import json + +from .bases import ClearEnvTest +from stackify.transport.application import ApiConfiguration +from stackify.utils import arg_or_env +from stackify.constants import DEFAULT_RUM_SCRIPT_URL +from stackifyapm.base import Client +from unittest import TestCase + +APM_CONFIG = { + "SERVICE_NAME": "service_name", + "ENVIRONMENT": "production", + "HOSTNAME": "sample_host", + "FRAMEWORK_NAME": "framework", + "FRAMEWORK_VERSION": "1.0", + "APPLICATION_NAME": "sample_application", + "BASE_DIR": "path/to/application/", + "RUM_SCRIPT_URL": "https://test.com/test.js", + "RUM_KEY": "LOREM123" +} + + +class RumTest(TestCase): + def setUp(self): + self.config_rum_key = stackify.config.rum_key + self.config_rum_script_url = stackify.config.rum_script_url + self.config_application = stackify.config.application + self.config_environment = stackify.config.environment + self.maxDiff = None + + def shutDown(self): + pass + + def test_default_insert_rum_script_from_apm_with_transaction(self): + self.update_apm_installed(True) + client = Client(APM_CONFIG) + + transaction = client.begin_transaction("transaction_test", client=client) + rum_data = stackify.rum.insert_rum_script() + assert rum_data + + rum_settings = { + "ID": transaction.get_trace_parent().trace_id, + "Name": base64.b64encode(APM_CONFIG["APPLICATION_NAME"].encode("utf-8")).decode("utf-8"), + "Env": base64.b64encode(APM_CONFIG["ENVIRONMENT"].encode("utf-8")).decode("utf-8"), + "Trans": base64.b64encode('/'.encode("utf-8")).decode("utf-8") + } + + result_string = ''.format( + json.dumps(rum_settings), + APM_CONFIG["RUM_SCRIPT_URL"], + APM_CONFIG["RUM_KEY"] + ) + + assert rum_data == result_string + client.end_transaction("transaction_test") + self.restore_apm_installed() + + def test_default_insert_rum_script_from_apm_without_transaction(self): + self.update_apm_installed(True) + rum_data = stackify.rum.insert_rum_script() + assert not rum_data + self.restore_apm_installed() + + @mock.patch('stackify.rum.get_reporting_url') + @mock.patch('stackify.rum.get_transaction_id') + def test_default_insert_rum_script(self, func, func_reporting_url): + func.return_value = '123' + func_reporting_url.return_value = 'test reporting url' + self.update_apm_installed(False) + self.update_common_config( + rum_key='asd', + application='app', + environment='env' + ) + + rum_settings = { + "ID": '123', + "Name": 'YXBw', + "Env": 'ZW52', + "Trans": 'dGVzdCByZXBvcnRpbmcgdXJs' + } + + result = stackify.rum.insert_rum_script() + self.reset_common_config() + self.restore_apm_installed() + + assert result == ''.format(json.dumps(rum_settings)) + + def test_default_insert_rum_script_no_transaction_id(self): + self.update_apm_installed(False) + self.update_common_config( + rum_key='asd', + application='app', + environment='env' + ) + + result = stackify.rum.insert_rum_script() + self.reset_common_config() + self.restore_apm_installed() + + assert result is None + + def test_default_insert_rum_script_no_key(self): + self.update_apm_installed(False) + self.update_common_config( + rum_key='', + application='app', + environment='env' + ) + + result = stackify.rum.insert_rum_script() + assert not result + + self.reset_common_config() + self.restore_apm_installed() + + def test_default_insert_rum_script_no_details(self): + self.update_apm_installed(False) + self.update_common_config() + + result = stackify.rum.insert_rum_script() + assert not result + + self.reset_common_config() + self.restore_apm_installed() + + @mock.patch('stackify.rum.get_reporting_url') + @mock.patch('stackify.rum.get_transaction_id') + def test_default_insert_rum_script_from_api(self, func, func_reporting_url): + func.return_value = '123' + func_reporting_url.return_value = 'test reporting url' + self.update_apm_installed(False) + self.create_config( + rum_key='asd1', + application='app1', + environment='env1' + ) + rum_settings = { + "ID": '123', + "Name": 'YXBwMQ==', + "Env": 'ZW52MQ==', + "Trans": 'dGVzdCByZXBvcnRpbmcgdXJs' + } + result = stackify.rum.insert_rum_script() + self.reset_common_config() + self.restore_apm_installed() + assert result == ''.format(json.dumps(rum_settings)) + + def test_default_insert_rum_script_no_key_from_api(self): + self.update_apm_installed(False) + self.create_config( + rum_key=None, + application='app2', + environment='env2' + ) + + result = stackify.rum.insert_rum_script() + self.reset_common_config() + self.restore_apm_installed() + + assert not result + + def test_default_insert_rum_script_no_details_from_api(self): + self.update_apm_installed(False) + self.create_config( + application=None, + environment=None, + rum_key=None + ) + + result = stackify.rum.insert_rum_script() + self.reset_common_config() + self.restore_apm_installed() + + assert not result + + def update_apm_installed(self, installed): + self.apm_installed = stackify.rum.apm_installed + stackify.rum.apm_installed = installed + + def restore_apm_installed(self): + stackify.rum.apm_installed = self.apm_installed + + def update_common_config(self, rum_key=None, rum_script_url=None, application=None, environment=None): + self.config_rum_key = stackify.config.rum_key + self.config_rum_script_url = stackify.config.rum_script_url + self.config_application = stackify.config.application + self.config_environment = stackify.config.environment + + if rum_key is not None: + stackify.config.rum_key = rum_key + if rum_script_url is not None: + stackify.config.rum_script_url = rum_script_url + if application is not None: + stackify.config.application = application + if environment is not None: + stackify.config.environment = environment + + def reset_common_config(self): + stackify.config.rum_key = self.config_rum_key + stackify.config.rum_script_url = self.config_rum_script_url + stackify.config.application = self.config_application + stackify.config.environment = self.config_environment + + def create_config(self, **kwargs): + return ApiConfiguration( + application=kwargs['application'], + environment=kwargs['environment'], + api_key='test_apikey', + api_url='test_apiurl', + rum_script_url=arg_or_env('rum_script_url', kwargs, DEFAULT_RUM_SCRIPT_URL, env_key='RETRACE_RUM_SCRIPT_URL'), + rum_key=arg_or_env('rum_key', kwargs, DEFAULT_RUM_SCRIPT_URL, env_key='RETRACE_RUM_KEY') + ) + + +class RumConfigurationTest(ClearEnvTest): + ''' + Test the logger init functionality + ''' + + def setUp(self): + super(RumConfigurationTest, self).setUp() + self.config_rum_key = stackify.config.rum_key + self.config_rum_script_url = stackify.config.rum_script_url + self.config_application = stackify.config.application + self.config_environment = stackify.config.environment + + def tearDown(self): + super(RumConfigurationTest, self).tearDown() + + def test_rum_script_url_valid(self): + os.environ["RETRACE_RUM_SCRIPT_URL"] = 'https://test.com/test.js' + config = self.create_config() + assert config.rum_script_url == 'https://test.com/test.js' + del os.environ["RETRACE_RUM_SCRIPT_URL"] + self.reset_common_config() + + @mock.patch('logging.Logger.exception') + def test_rum_script_url_invalid(self, func=None): + os.environ["RETRACE_RUM_SCRIPT_URL"] = 'asd' + config = self.create_config() + assert config.rum_script_url == 'https://stckjs.stackify.com/stckjs.js' # Default + del os.environ["RETRACE_RUM_SCRIPT_URL"] + self.reset_common_config() + func.assert_called_with('https://stckjs.stackify.com/stckjs.js does not match pattern ^[A-Za-z0-9_-]+$') + + def test_rum_key_valid(self): + os.environ["RETRACE_RUM_KEY"] = 'TEST123-_' + config = self.create_config() + assert config.rum_key == 'TEST123-_' + del os.environ["RETRACE_RUM_KEY"] + self.reset_common_config() + + @mock.patch('logging.Logger.exception') + def test_rum_key_invalid(self, func=None): + os.environ["RETRACE_RUM_KEY"] = 'asd`1!' + config = self.create_config() + assert config.rum_key == '' # Default + del os.environ["RETRACE_RUM_KEY"] + self.reset_common_config() + func.assert_called_with('asd`1! does not match pattern ^[A-Za-z0-9_-]+$') + + def create_config(self, **kwargs): + return ApiConfiguration( + application='test_appname', + environment='test_environment', + api_key='test_apikey', + api_url='test_apiurl', + rum_script_url=arg_or_env('rum_script_url', kwargs, DEFAULT_RUM_SCRIPT_URL, env_key='RETRACE_RUM_SCRIPT_URL'), + rum_key=arg_or_env('rum_key', kwargs, DEFAULT_RUM_SCRIPT_URL, env_key='RETRACE_RUM_KEY') + ) + + def reset_common_config(self): + stackify.config.rum_key = self.config_rum_key + stackify.config.rum_script_url = self.config_rum_script_url + stackify.config.application = self.config_application + stackify.config.environment = self.config_environment diff --git a/tests/test_utils.py b/tests/test_utils.py index 70e4b78..2ec637c 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -9,6 +9,9 @@ import stackify import logging +from stackify.utils import ConfigError +from stackify.utils import RegexValidator + class TestInit(ClearEnvTest): ''' @@ -91,6 +94,28 @@ def test_utils_data_to_json_dummy_request(self): self.assertTrue(substring in result) +class RegexValidatorTest(ClearEnvTest): + + def test_should_return_correct_value(self): + regex = "^[a-zA-Z0-9 _-]+$" + value = 'some_value' + _validate = RegexValidator(regex) + + validated_value = _validate(value, 'SOME_KEY') + + assert validated_value == value + + def test_should_raise_exception(self): + regex = "^[a-zA-Z0-9 _-]+$" + value = '#$%^' + _validate = RegexValidator(regex) + + with self.assertRaises(ConfigError) as context: + _validate(value, 'SOME_KEY') + + assert 'does not match pattern' in context.exception.args[0] + + class Dummy(object): pass From 9a0b65baf7e6aa7e50b89c3c16b2c1985e9e0055 Mon Sep 17 00:00:00 2001 From: Michael Mantos Date: Fri, 30 Jul 2021 22:27:44 +0800 Subject: [PATCH 30/44] LNX-329 - Fix MethodNotFound error for profiler rum checking --- docker/stackify-python-api-test | 4 +- stackify/rum.py | 20 ++++++-- test-docker-execute.sh | 11 ++++- tests/rum/__init__.py | 0 tests/rum/test_rum_apm.py | 76 ++++++++++++++++++++++++++++++ tests/test_compat.py | 6 +-- tests/test_rum.py | 82 +++++++++------------------------ 7 files changed, 129 insertions(+), 70 deletions(-) create mode 100644 tests/rum/__init__.py create mode 100644 tests/rum/test_rum_apm.py diff --git a/docker/stackify-python-api-test b/docker/stackify-python-api-test index 5b8f88f..7383754 100644 --- a/docker/stackify-python-api-test +++ b/docker/stackify-python-api-test @@ -15,6 +15,8 @@ RUN mkdir /build COPY . /build/ RUN cat /build/requirements.txt | xargs -n 1 pip install; exit 0 -RUN if [ "${test}" = 1 ]; then pip install -i "${test_repo}" stackify-python-apm; fi; exit 0 + +ENV TEST="${test}" +ENV TEST_REPO="${test_repo}" CMD /bin/bash -c "cd /build && source test-docker-execute.sh" diff --git a/stackify/rum.py b/stackify/rum.py index 6122bf4..50350df 100644 --- a/stackify/rum.py +++ b/stackify/rum.py @@ -5,15 +5,16 @@ apm_installed = False try: + from stackifyapm import insert_rum_script as insert_rum_script_from_apm apm_installed = True - from stackifyapm import insert_rum_script as insert_rum_script_apm -except ImportError: +except (ImportError): pass def insert_rum_script(): - if apm_installed is True: - return insert_rum_script_apm() + apm_rum_script = insert_rum_script_apm() + if apm_rum_script: + return apm_rum_script rum_key = config.rum_key rum_script_url = config.rum_script_url @@ -72,3 +73,14 @@ def get_transaction_id(): def get_reporting_url(): return '' + + +def insert_rum_script_apm(): + if not is_apm_installed(): + return + + return insert_rum_script_from_apm() + + +def is_apm_installed(): + return apm_installed diff --git a/test-docker-execute.sh b/test-docker-execute.sh index fc76dce..f87161f 100755 --- a/test-docker-execute.sh +++ b/test-docker-execute.sh @@ -14,7 +14,16 @@ function runPyTest() { echo '<--------------------------------------------->' echo "Python Version $(python --version)" echo 'Running pytest...' - py.test + py.test --ignore=tests/rum + + if [ "${TEST}" = 1 ]; then + pip install -i "${TEST_REPO}" stackify-python-apm; + else + pip install stackify-python-apm; + py.test tests/rum + fi + + pip uninstall -y stackify-python-apm } runFlake8 diff --git a/tests/rum/__init__.py b/tests/rum/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/rum/test_rum_apm.py b/tests/rum/test_rum_apm.py new file mode 100644 index 0000000..fc8a28d --- /dev/null +++ b/tests/rum/test_rum_apm.py @@ -0,0 +1,76 @@ +try: + from unittest import mock +except Exception: + import mock # noqa F401 + +import stackify +import stackify.rum +import base64 +import json +from unittest import TestCase + +apmExist = False +try: + from stackifyapm.base import Client + apmExist = True +except (ImportError): + pass + +APM_CONFIG = { + "SERVICE_NAME": "service_name", + "ENVIRONMENT": "production", + "HOSTNAME": "sample_host", + "FRAMEWORK_NAME": "framework", + "FRAMEWORK_VERSION": "1.0", + "APPLICATION_NAME": "sample_application", + "BASE_DIR": "path/to/application/", + "RUM_SCRIPT_URL": "https://test.com/test.js", + "RUM_KEY": "LOREM123" +} + + +class RumTestApm(TestCase): + def setUp(self): + self.config_rum_key = stackify.config.rum_key + self.config_rum_script_url = stackify.config.rum_script_url + self.config_application = stackify.config.application + self.config_environment = stackify.config.environment + self.maxDiff = None + + def shutDown(self): + pass + + def test_default_insert_rum_script_from_apm_with_transaction(self): + if not apmExist: + return + + client = Client(APM_CONFIG) + print('config') + print(client.config.rum_key) + + transaction = client.begin_transaction("transaction_test", client=client) + rum_data = stackify.rum.insert_rum_script() + + rum_settings = { + "ID": transaction.get_trace_parent().trace_id, + "Name": base64.b64encode(APM_CONFIG["APPLICATION_NAME"].encode("utf-8")).decode("utf-8"), + "Env": base64.b64encode(APM_CONFIG["ENVIRONMENT"].encode("utf-8")).decode("utf-8"), + "Trans": base64.b64encode('/'.encode("utf-8")).decode("utf-8") + } + + result_string = ''.format( + json.dumps(rum_settings), + APM_CONFIG["RUM_SCRIPT_URL"], + APM_CONFIG["RUM_KEY"] + ) + + assert rum_data + assert rum_data == result_string + client.end_transaction("transaction_test") + + def test_default_insert_rum_script_from_apm_without_transaction(self): + if not apmExist: + return + + rum_data = stackify.rum.insert_rum_script() + assert not rum_data diff --git a/tests/test_compat.py b/tests/test_compat.py index 9f97bbb..d2dcfbd 100644 --- a/tests/test_compat.py +++ b/tests/test_compat.py @@ -1,8 +1,8 @@ from unittest import TestCase -from stackifyapm.utils.compat import b -from stackifyapm.utils.compat import iterkeys -from stackifyapm.utils.compat import iteritems +from stackify.compat import b +from stackify.compat import iterkeys +from stackify.compat import iteritems class CompatTest(TestCase): diff --git a/tests/test_rum.py b/tests/test_rum.py index bb9cc82..17fc683 100644 --- a/tests/test_rum.py +++ b/tests/test_rum.py @@ -6,14 +6,12 @@ import stackify import os import stackify.rum -import base64 import json from .bases import ClearEnvTest from stackify.transport.application import ApiConfiguration from stackify.utils import arg_or_env from stackify.constants import DEFAULT_RUM_SCRIPT_URL -from stackifyapm.base import Client from unittest import TestCase APM_CONFIG = { @@ -40,43 +38,13 @@ def setUp(self): def shutDown(self): pass - def test_default_insert_rum_script_from_apm_with_transaction(self): - self.update_apm_installed(True) - client = Client(APM_CONFIG) - - transaction = client.begin_transaction("transaction_test", client=client) - rum_data = stackify.rum.insert_rum_script() - assert rum_data - - rum_settings = { - "ID": transaction.get_trace_parent().trace_id, - "Name": base64.b64encode(APM_CONFIG["APPLICATION_NAME"].encode("utf-8")).decode("utf-8"), - "Env": base64.b64encode(APM_CONFIG["ENVIRONMENT"].encode("utf-8")).decode("utf-8"), - "Trans": base64.b64encode('/'.encode("utf-8")).decode("utf-8") - } - - result_string = ''.format( - json.dumps(rum_settings), - APM_CONFIG["RUM_SCRIPT_URL"], - APM_CONFIG["RUM_KEY"] - ) - - assert rum_data == result_string - client.end_transaction("transaction_test") - self.restore_apm_installed() - - def test_default_insert_rum_script_from_apm_without_transaction(self): - self.update_apm_installed(True) - rum_data = stackify.rum.insert_rum_script() - assert not rum_data - self.restore_apm_installed() - + @mock.patch('stackify.rum.is_apm_installed') @mock.patch('stackify.rum.get_reporting_url') @mock.patch('stackify.rum.get_transaction_id') - def test_default_insert_rum_script(self, func, func_reporting_url): + def test_default_insert_rum_script(self, func, func_reporting_url, func_apm): func.return_value = '123' func_reporting_url.return_value = 'test reporting url' - self.update_apm_installed(False) + func_apm.return_value = False self.update_common_config( rum_key='asd', application='app', @@ -92,12 +60,12 @@ def test_default_insert_rum_script(self, func, func_reporting_url): result = stackify.rum.insert_rum_script() self.reset_common_config() - self.restore_apm_installed() assert result == ''.format(json.dumps(rum_settings)) - def test_default_insert_rum_script_no_transaction_id(self): - self.update_apm_installed(False) + @mock.patch('stackify.rum.is_apm_installed') + def test_default_insert_rum_script_no_transaction_id(self, func_apm): + func_apm.return_value = False self.update_common_config( rum_key='asd', application='app', @@ -106,12 +74,12 @@ def test_default_insert_rum_script_no_transaction_id(self): result = stackify.rum.insert_rum_script() self.reset_common_config() - self.restore_apm_installed() assert result is None - def test_default_insert_rum_script_no_key(self): - self.update_apm_installed(False) + @mock.patch('stackify.rum.is_apm_installed') + def test_default_insert_rum_script_no_key(self, func_apm): + func_apm.return_value = False self.update_common_config( rum_key='', application='app', @@ -122,24 +90,24 @@ def test_default_insert_rum_script_no_key(self): assert not result self.reset_common_config() - self.restore_apm_installed() - def test_default_insert_rum_script_no_details(self): - self.update_apm_installed(False) + @mock.patch('stackify.rum.is_apm_installed') + def test_default_insert_rum_script_no_details(self, func_apm): + func_apm.return_value = False self.update_common_config() result = stackify.rum.insert_rum_script() assert not result self.reset_common_config() - self.restore_apm_installed() + @mock.patch('stackify.rum.is_apm_installed') @mock.patch('stackify.rum.get_reporting_url') @mock.patch('stackify.rum.get_transaction_id') - def test_default_insert_rum_script_from_api(self, func, func_reporting_url): + def test_default_insert_rum_script_from_api(self, func, func_reporting_url, func_apm): func.return_value = '123' + func_apm.return_value = False func_reporting_url.return_value = 'test reporting url' - self.update_apm_installed(False) self.create_config( rum_key='asd1', application='app1', @@ -153,11 +121,11 @@ def test_default_insert_rum_script_from_api(self, func, func_reporting_url): } result = stackify.rum.insert_rum_script() self.reset_common_config() - self.restore_apm_installed() assert result == ''.format(json.dumps(rum_settings)) - def test_default_insert_rum_script_no_key_from_api(self): - self.update_apm_installed(False) + @mock.patch('stackify.rum.is_apm_installed') + def test_default_insert_rum_script_no_key_from_api(self, func_apm): + func_apm.return_value = False self.create_config( rum_key=None, application='app2', @@ -166,12 +134,12 @@ def test_default_insert_rum_script_no_key_from_api(self): result = stackify.rum.insert_rum_script() self.reset_common_config() - self.restore_apm_installed() assert not result - def test_default_insert_rum_script_no_details_from_api(self): - self.update_apm_installed(False) + @mock.patch('stackify.rum.is_apm_installed') + def test_default_insert_rum_script_no_details_from_api(self, func_apm): + func_apm.return_value = False self.create_config( application=None, environment=None, @@ -180,17 +148,9 @@ def test_default_insert_rum_script_no_details_from_api(self): result = stackify.rum.insert_rum_script() self.reset_common_config() - self.restore_apm_installed() assert not result - def update_apm_installed(self, installed): - self.apm_installed = stackify.rum.apm_installed - stackify.rum.apm_installed = installed - - def restore_apm_installed(self): - stackify.rum.apm_installed = self.apm_installed - def update_common_config(self, rum_key=None, rum_script_url=None, application=None, environment=None): self.config_rum_key = stackify.config.rum_key self.config_rum_script_url = stackify.config.rum_script_url From b988b641c16346ac8fafa2af3b8f58b5826211c6 Mon Sep 17 00:00:00 2001 From: Michael Mantos Date: Tue, 3 Aug 2021 20:46:41 +0800 Subject: [PATCH 31/44] LNX-329 - Change None to empty space inject rum script --- stackify/rum.py | 16 ++++++++-------- tests/rum/test_rum_apm.py | 1 + tests/test_rum.py | 7 ++++++- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/stackify/rum.py b/stackify/rum.py index 50350df..10f2f76 100644 --- a/stackify/rum.py +++ b/stackify/rum.py @@ -13,30 +13,30 @@ def insert_rum_script(): apm_rum_script = insert_rum_script_apm() - if apm_rum_script: + if apm_rum_script is not None: return apm_rum_script rum_key = config.rum_key rum_script_url = config.rum_script_url if not rum_script_url or not rum_key: - return None + return '' transaction_id = get_transaction_id() if not transaction_id: - return None + return '' reporting_url = get_reporting_url() if not reporting_url: - return None + return '' application_name = config.application if not application_name: - return None + return '' environment = config.environment if not environment: - return None + return '' settings = { "ID": transaction_id @@ -58,7 +58,7 @@ def insert_rum_script(): settings["Trans"] = reporting_url_b64 if not settings: - return None + return '' return ''.format( json.dumps(settings), @@ -77,7 +77,7 @@ def get_reporting_url(): def insert_rum_script_apm(): if not is_apm_installed(): - return + return None return insert_rum_script_from_apm() diff --git a/tests/rum/test_rum_apm.py b/tests/rum/test_rum_apm.py index fc8a28d..0a9a699 100644 --- a/tests/rum/test_rum_apm.py +++ b/tests/rum/test_rum_apm.py @@ -74,3 +74,4 @@ def test_default_insert_rum_script_from_apm_without_transaction(self): rum_data = stackify.rum.insert_rum_script() assert not rum_data + assert rum_data is '' diff --git a/tests/test_rum.py b/tests/test_rum.py index 17fc683..b1e77fb 100644 --- a/tests/test_rum.py +++ b/tests/test_rum.py @@ -75,7 +75,8 @@ def test_default_insert_rum_script_no_transaction_id(self, func_apm): result = stackify.rum.insert_rum_script() self.reset_common_config() - assert result is None + assert not result + assert result is '' @mock.patch('stackify.rum.is_apm_installed') def test_default_insert_rum_script_no_key(self, func_apm): @@ -88,6 +89,7 @@ def test_default_insert_rum_script_no_key(self, func_apm): result = stackify.rum.insert_rum_script() assert not result + assert result is '' self.reset_common_config() @@ -98,6 +100,7 @@ def test_default_insert_rum_script_no_details(self, func_apm): result = stackify.rum.insert_rum_script() assert not result + assert result is '' self.reset_common_config() @@ -136,6 +139,7 @@ def test_default_insert_rum_script_no_key_from_api(self, func_apm): self.reset_common_config() assert not result + assert result is '' @mock.patch('stackify.rum.is_apm_installed') def test_default_insert_rum_script_no_details_from_api(self, func_apm): @@ -150,6 +154,7 @@ def test_default_insert_rum_script_no_details_from_api(self, func_apm): self.reset_common_config() assert not result + assert result is '' def update_common_config(self, rum_key=None, rum_script_url=None, application=None, environment=None): self.config_rum_key = stackify.config.rum_key From dbf41456280341022162e15dfbe37fb13df9abc1 Mon Sep 17 00:00:00 2001 From: Michael Mantos Date: Fri, 13 Aug 2021 02:43:48 +0800 Subject: [PATCH 32/44] LNX-329 - Add RUMv2 doc --- README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/README.md b/README.md index ba7de4c..81a457d 100644 --- a/README.md +++ b/README.md @@ -123,3 +123,22 @@ logger = logging.getLogger('django') logger.warning('Something happened') ``` + + +## **Real User Monitoring (RUM)** + +Real user monitoring injects a script tag containing the [RUM JS](https://stackify.com/retrace-real-user-monitoring/) that is responsible for capturing information about the http requests on the browser. This approach is manual and needs to be configured. + +### RUM - Setup + +```python +# Configuration - Standard API +logger = stackify.getLogger(..., rum_key="YourRumKey") +# or Configuration - Python Logging Integration +stackify.StackifyHandler(..., rum_key="YourRumKey") + +# Use this to apply on views +import stackify.rum + +stackify.rum.insert_rum_script() +``` \ No newline at end of file From 1884d76e70e8387294dd69fad08524690ec19c52 Mon Sep 17 00:00:00 2001 From: Michael Mantos Date: Fri, 20 Aug 2021 20:06:19 +0800 Subject: [PATCH 33/44] Updating version to 1.1.3 --- stackify/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackify/__init__.py b/stackify/__init__.py index 7f88d76..28f570c 100644 --- a/stackify/__init__.py +++ b/stackify/__init__.py @@ -1,7 +1,7 @@ """ Stackify Python API """ -__version__ = '1.1.3b1' +__version__ = '1.1.3' import logging import inspect From 7a78a10b117e1daa1d77d28bc0ad4a6a2f8c6802 Mon Sep 17 00:00:00 2001 From: Michael Mantos Date: Fri, 20 Aug 2021 20:32:02 +0800 Subject: [PATCH 34/44] Updating version to 1.2.0 beta 1 --- stackify/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackify/__init__.py b/stackify/__init__.py index 28f570c..4b10b4e 100644 --- a/stackify/__init__.py +++ b/stackify/__init__.py @@ -1,7 +1,7 @@ """ Stackify Python API """ -__version__ = '1.1.3' +__version__ = '1.2.0b1' import logging import inspect From 6f03d100c998526ad415f6750365b81ff35ad8dc Mon Sep 17 00:00:00 2001 From: Todd Lair Date: Thu, 2 Sep 2021 17:26:02 -0500 Subject: [PATCH 35/44] Set up CI with Azure Pipelines [skip ci] --- azure-pipelines.yml | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 azure-pipelines.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 0000000..ff3b287 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,40 @@ +# Python package +# Create and test a Python package on multiple Python versions. +# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: +# https://docs.microsoft.com/azure/devops/pipelines/languages/python + +trigger: +- master + +pool: + vmImage: ubuntu-latest +strategy: + matrix: + Python27: + python.version: '2.7' + Python35: + python.version: '3.5' + Python36: + python.version: '3.6' + Python37: + python.version: '3.7' + +steps: +- task: UsePythonVersion@0 + inputs: + versionSpec: '$(python.version)' + displayName: 'Use Python $(python.version)' + +- script: | + python -m pip install --upgrade pip + pip install -r requirements.txt + displayName: 'Install dependencies' + +- task: CmdLine@2 + inputs: + script: 'curl -sSL https://www.sourceclear.com/install | sh' +- task: CmdLine@2 + inputs: + script: 'srcclr scan .' + env: + SRCCLR_API_TOKEN: $(SRCCLR_API_TOKEN) From aeaa43261d204748c4d0d33d407bc85f87086485 Mon Sep 17 00:00:00 2001 From: Todd Lair Date: Thu, 2 Sep 2021 17:28:34 -0500 Subject: [PATCH 36/44] Update azure-pipelines.yml for Azure Pipelines --- azure-pipelines.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ff3b287..67045a9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -10,12 +10,12 @@ pool: vmImage: ubuntu-latest strategy: matrix: - Python27: - python.version: '2.7' - Python35: - python.version: '3.5' - Python36: - python.version: '3.6' +# Python27: +# python.version: '2.7' +# Python35: +# python.version: '3.5' +# Python36: +# python.version: '3.6' Python37: python.version: '3.7' From 605eef79fa91d677bfcc2505e2d2ff4db7ddd6fd Mon Sep 17 00:00:00 2001 From: Todd Lair Date: Thu, 9 Sep 2021 10:00:27 -0500 Subject: [PATCH 37/44] Added Veracode upload --- azure-pipelines.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 67045a9..44bd3ab 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -38,3 +38,15 @@ steps: script: 'srcclr scan .' env: SRCCLR_API_TOKEN: $(SRCCLR_API_TOKEN) + +- task: CmdLine@2 + inputs: + script: 'zip -r stackify-api-python.zip stackify/ requirements.txt setup.py' +- task: Veracode@3 + inputs: + ConnectionDetailsSelection: 'Endpoint' + AnalysisService: 'Veracode' + veracodeAppProfile: 'Retrace Python Library' + version: 'AZ-Devops-Build-$(build.buildNumber)' + filepath: 'stackify-api-python.zip' + maximumWaitTime: '360' From 5241aebd24a6b5bd6d649f88a90ed1330f75bced Mon Sep 17 00:00:00 2001 From: Todd Lair Date: Wed, 24 Nov 2021 10:52:06 -0600 Subject: [PATCH 38/44] Update azure-pipelines.yml for Azure Pipelines --- azure-pipelines.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 44bd3ab..fb3c708 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -6,6 +6,14 @@ trigger: - master +schedules: +- cron: "0 21-22 31 * *" + displayName: Monthly build + branches: + include: + - develop + always: true + pool: vmImage: ubuntu-latest strategy: From b835350a9b18ff2658fc476404b8078c14e216e7 Mon Sep 17 00:00:00 2001 From: Michael Mantos Date: Thu, 3 Mar 2022 17:22:52 +0800 Subject: [PATCH 39/44] INV-1894 - Fix circular error issue for wsgi requests --- stackify/transport/default/log.py | 3 ++ stackify/utils.py | 37 ++++++++++++++++--- test.sh | 9 ++++- tests/test_utils.py | 43 +++++++++++++++++++--- tests/transport/agent/test_agent_socket.py | 6 ++- 5 files changed, 84 insertions(+), 14 deletions(-) diff --git a/stackify/transport/default/log.py b/stackify/transport/default/log.py index a501865..6601bc0 100644 --- a/stackify/transport/default/log.py +++ b/stackify/transport/default/log.py @@ -2,6 +2,7 @@ from stackify.transport.default.formats import JSONObject from stackify.transport.default.error import StackifyError from stackify.utils import data_to_json +from stackify.utils import extract_request class LogMsg(JSONObject): @@ -31,6 +32,8 @@ def from_record(self, record): data = {k: v for k, v in record.__dict__.items() if k not in RECORD_VARS} + data = extract_request(data) + if data: self.data = data_to_json(data) diff --git a/stackify/utils.py b/stackify/utils.py index 2da8944..ac99f4d 100644 --- a/stackify/utils.py +++ b/stackify/utils.py @@ -21,16 +21,41 @@ def arg_or_env(name, args, default=None, env_key=None): def data_to_json(data): try: - if object_is_iterable(data) and 'request' in data and hasattr(data['request'], '_messages'): - data['request'] = get_default_object(data['request']) - return json.dumps(data, default=lambda x: get_default_object(x)) - except ValueError as e: - internal_logger.exception('Failed to serialize object to json: {} - Exception: {}'.format(data.__str__(), str(e))) - return json.dumps(data.__str__()) # String representation of the object + except Exception as e: + internal_logger.exception('Failed to serialize object to json: {} - Exception: {}'.format(str(data), str(e))) + return str(data) # String representation of the object + + +def extract_request(data): + if 'request' in data and "WSGIRequest" in str(data['request']): + new_request = {} + obj = data['request'] + + if hasattr(obj, 'path'): + new_request['path'] = obj.path + + if hasattr(obj, 'method'): + new_request['method'] = obj.method + + if hasattr(obj, 'POST'): + new_request['form'] = obj.POST + + if hasattr(obj, 'GET'): + new_request['query'] = obj.GET + + if hasattr(obj, 'content_type'): + new_request['content_type'] = obj.content_type + + data['request'] = new_request + + return data def get_default_object(obj): + if object_is_iterable(obj): + return [item for item in obj] + return hasattr(obj, '__dict__') and obj.__dict__ or obj.__str__() diff --git a/test.sh b/test.sh index 8967876..e4d34fc 100755 --- a/test.sh +++ b/test.sh @@ -22,7 +22,14 @@ function runPyTest() { virtualenv -p python${python_version} ${test_venv} echo "Activating virtualenv ${test_venv}..." - source ${test_venv}/bin/activate + + if [ -f ${test_venv}/bin/activate ]; then + source ${test_venv}/bin/activate + fi + + if [ -f ${test_venv}/Scripts/activate ]; then + source ${test_venv}/Scripts/activate + fi echo 'Installing dependencies...' pip install -r requirements.txt diff --git a/tests/test_utils.py b/tests/test_utils.py index 70e4b78..97af855 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -30,6 +30,11 @@ def tearDown(self): for logger in self.loggers: del global_loggers[logger.name] + def test_utils_data_to_json_string(self): + dummy = 'test' + result = stackify.utils.data_to_json(dummy) + self.assertEqual('"test"', result) + def test_utils_data_to_json_unserializable(self): dummy = Dummy() result = stackify.utils.data_to_json(dummy) @@ -57,9 +62,9 @@ def test_utils_data_to_json_tuple(self): self.assertEqual(expected, result) def test_utils_data_to_json_dummy_iterable(self): - dummy = DummyInterable() + dummy = DummyIterable() result = stackify.utils.data_to_json(dummy) - expected = '{"a": 21}' + expected = '[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]' self.assertEqual(expected, result) def test_utils_data_to_json_dummy_object_with_property(self): @@ -79,10 +84,15 @@ def test_utils_data_to_json_dummy_object_circular(self, func): data['payload']['dummy'] = data result = stackify.utils.data_to_json(data) - func.assert_called() - substring = "'dummy': Date: Thu, 3 Mar 2022 17:25:28 +0800 Subject: [PATCH 40/44] INV-1894 - Update protobuf dependency --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 54d51bf..811c4e4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ flake8 mock==2.0.0 -protobuf==3.9.1 +protobuf==3.15.0 pytest==4.3.0 pytest-cov==2.6.1 requests==2.21.0 From 427d2248c27c76c43f093bede2ee6a4db691d5a1 Mon Sep 17 00:00:00 2001 From: Michael Mantos Date: Thu, 17 Mar 2022 20:23:15 +0800 Subject: [PATCH 41/44] Updating version to 1.2.0 --- stackify/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackify/__init__.py b/stackify/__init__.py index 4b10b4e..1564d5e 100644 --- a/stackify/__init__.py +++ b/stackify/__init__.py @@ -1,7 +1,7 @@ """ Stackify Python API """ -__version__ = '1.2.0b1' +__version__ = '1.2.0' import logging import inspect From 53082dab2b10c14ba0e719a710dfaf6f59662a05 Mon Sep 17 00:00:00 2001 From: Todd Lair <26470949+t-lair@users.noreply.github.com> Date: Thu, 30 Jun 2022 08:11:52 -0500 Subject: [PATCH 42/44] Fixed trigger for scheduled build --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index fb3c708..c67c204 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -11,7 +11,7 @@ schedules: displayName: Monthly build branches: include: - - develop + - master always: true pool: From 6195c4aed043a88cd9e6b6b71d6c2f0506b3b00c Mon Sep 17 00:00:00 2001 From: Todd Lair <26470949+t-lair@users.noreply.github.com> Date: Fri, 19 Aug 2022 14:33:02 -0500 Subject: [PATCH 43/44] Updated build schedule to weekly --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c67c204..40e96f4 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -7,8 +7,8 @@ trigger: - master schedules: -- cron: "0 21-22 31 * *" - displayName: Monthly build +- cron: "0 21-22 * * 0" + displayName: Weekly build branches: include: - master From 0570aea0b562d1acc71c858f2a51eeff75e680ca Mon Sep 17 00:00:00 2001 From: Todd Lair <26470949+t-lair@users.noreply.github.com> Date: Thu, 20 Oct 2022 11:57:18 -0500 Subject: [PATCH 44/44] Updated the Veracode scan step to try optional arguments to delete incomplete scan --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 40e96f4..1b16f89 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -58,3 +58,4 @@ steps: version: 'AZ-Devops-Build-$(build.buildNumber)' filepath: 'stackify-api-python.zip' maximumWaitTime: '360' + optargs: -deleteincompletescan 2