From 0002222104bf777210a2a593b462c30893868441 Mon Sep 17 00:00:00 2001 From: Patrik Puchala Date: Sun, 29 Oct 2017 19:33:30 +0100 Subject: [PATCH 01/10] #172_issue - kv_value returns string instead of bytes (in python3) - simple utf-8 decode added - docstring updated --- consul/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/consul/base.py b/consul/base.py index 09203b6d..4a1ddbd6 100755 --- a/consul/base.py +++ b/consul/base.py @@ -199,7 +199,7 @@ def json( *one* returns only the first item of the list of items. empty lists are coerced to None. - *decode* if specified this key will be base64 decoded. + *decode* if specified this key will be base64 decoded (bytes -> string in python3). *is_id* only the 'ID' field of the json object will be returned. """ @@ -213,7 +213,7 @@ def cb(response): if decode: for item in data: if item.get(decode) is not None: - item[decode] = base64.b64decode(item[decode]) + item[decode] = str(base64.b64decode(item[decode]).decode('utf-8')) if is_id: data = data['ID'] if one: From 4031bec90d1ef5a9be2e0bf7b7baefe4bd043022 Mon Sep 17 00:00:00 2001 From: Patrik Puchala Date: Sun, 29 Oct 2017 20:16:47 +0100 Subject: [PATCH 02/10] #172_issue - kv_value returns string instead of bytes (in python3) - converting bytes to string done via utf-8 decoding - docstring updated --- consul/base.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/consul/base.py b/consul/base.py index 4a1ddbd6..9c7b6599 100755 --- a/consul/base.py +++ b/consul/base.py @@ -7,11 +7,24 @@ import six from six.moves import urllib - +import codecs log = logging.getLogger(__name__) +def slashescape(err): + """ codecs error handler. err is UnicodeDecode instance. return + a tuple with a replacement for the unencodable part of the input + and a position where encoding should continue""" + # print err, dir(err), err.start, err.end, err.object[:err.start] + thebyte = err.object[err.start:err.end] + repl = u'\\x' + hex(ord(thebyte))[2:] + return repl, err.end + + +codecs.register_error('slashescape', slashescape) + + class ConsulException(Exception): pass @@ -203,6 +216,7 @@ def json( *is_id* only the 'ID' field of the json object will be returned. """ + def cb(response): CB.__status(response, allow_404=allow_404) if response.code == 404: @@ -213,7 +227,7 @@ def cb(response): if decode: for item in data: if item.get(decode) is not None: - item[decode] = str(base64.b64decode(item[decode]).decode('utf-8')) + item[decode] = base64.b64decode(item[decode]).decode('utf-8', 'slashescape') if is_id: data = data['ID'] if one: From 632cdf6e437eeb606ae51b834dad63a0d55204a9 Mon Sep 17 00:00:00 2001 From: Patrik Puchala Date: Mon, 30 Oct 2017 08:15:57 +0100 Subject: [PATCH 03/10] #172_issue - kv_value returns string instead of bytes (in python3) - converting bytes to string done via latin-1 decoding - docstring updated --- consul/base.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/consul/base.py b/consul/base.py index 9c7b6599..32a9d3b0 100755 --- a/consul/base.py +++ b/consul/base.py @@ -7,24 +7,10 @@ import six from six.moves import urllib -import codecs log = logging.getLogger(__name__) -def slashescape(err): - """ codecs error handler. err is UnicodeDecode instance. return - a tuple with a replacement for the unencodable part of the input - and a position where encoding should continue""" - # print err, dir(err), err.start, err.end, err.object[:err.start] - thebyte = err.object[err.start:err.end] - repl = u'\\x' + hex(ord(thebyte))[2:] - return repl, err.end - - -codecs.register_error('slashescape', slashescape) - - class ConsulException(Exception): pass @@ -227,7 +213,7 @@ def cb(response): if decode: for item in data: if item.get(decode) is not None: - item[decode] = base64.b64decode(item[decode]).decode('utf-8', 'slashescape') + item[decode] = base64.b64decode(item[decode]).decode('latin-1') if is_id: data = data['ID'] if one: From 2c7fa0bb2c3729e767d933ae3665d67defb631cf Mon Sep 17 00:00:00 2001 From: Patrik Puchala Date: Mon, 30 Oct 2017 08:41:37 +0100 Subject: [PATCH 04/10] #172_issue - kv_value returns string instead of bytes (in python3) - converting bytes to string done via latin-1 decoding - docstring updated - tests updated --- tests/test_aio.py | 6 +++--- tests/test_std.py | 18 +++++++++--------- tests/test_tornado.py | 8 ++++---- tests/test_twisted.py | 6 +++--- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/test_aio.py b/tests/test_aio.py index b6e0c943..c94d839d 100644 --- a/tests/test_aio.py +++ b/tests/test_aio.py @@ -39,7 +39,7 @@ def main(): response = yield from c.kv.put('foo', 'bar') assert response is True index, data = yield from c.kv.get('foo') - assert data['Value'] == six.b('bar') + assert data['Value'] == 'bar' c.close() loop.run_until_complete(main()) @@ -79,7 +79,7 @@ def main(): index, data = yield from c.kv.get('foo') assert data is None index, data = yield from c.kv.get('foo', index=index) - assert data['Value'] == six.b('bar') + assert data['Value'] == 'bar' yield from fut c.close() @@ -137,7 +137,7 @@ def get(): index, data = yield from c.kv.get('foo') assert data is None index, data = yield from c.kv.get('foo', index=index) - assert data['Value'] == six.b('bar') + assert data['Value'] == 'bar' yield from fut c.close() diff --git a/tests/test_std.py b/tests/test_std.py index ad6e45be..c0ebc266 100644 --- a/tests/test_std.py +++ b/tests/test_std.py @@ -28,7 +28,7 @@ def test_kv(self, consul_port): assert data is None assert c.kv.put('foo', 'bar') is True index, data = c.kv.get('foo') - assert data['Value'] == six.b('bar') + assert data['Value'] == 'bar' def test_kv_wait(self, consul_port): c = consul.Consul(port=consul_port) @@ -48,7 +48,7 @@ def test_kv_encoding(self, consul_port): # test unicode c.kv.put('foo', u'bar') index, data = c.kv.get('foo') - assert data['Value'] == six.b('bar') + assert data['Value'] == 'bar' # test empty-string comes back as `None` c.kv.put('foo', '') @@ -72,7 +72,7 @@ def test_kv_put_cas(self, consul_port): assert c.kv.put('foo', 'bar2', cas=data['ModifyIndex']-1) is False assert c.kv.put('foo', 'bar2', cas=data['ModifyIndex']) is True index, data = c.kv.get('foo') - assert data['Value'] == six.b('bar2') + assert data['Value'] == 'bar2' def test_kv_put_flags(self, consul_port): c = consul.Consul(port=consul_port) @@ -100,7 +100,7 @@ def test_kv_recurse(self, consul_port): assert [x['Key'] for x in data] == [ 'foo/', 'foo/bar1', 'foo/bar2', 'foo/bar3'] assert [x['Value'] for x in data] == [ - None, six.b('1'), six.b('2'), six.b('3')] + None, '1', '2', '3'] def test_kv_delete(self, consul_port): c = consul.Consul(port=consul_port) @@ -691,7 +691,7 @@ def test_session_delete_ttl_renew(self, consul_port): # trying out the behavior assert c.kv.put('foo', '1', acquire=s) is True index, data = c.kv.get('foo') - assert data['Value'] == six.b('1') + assert data['Value'] == '1' c.session.destroy(s) index, data = c.kv.get('foo') @@ -757,13 +757,13 @@ def test_acl_explict_token_use(self, acl_consul): c.kv.put('foo', 'bar') c.kv.put('private/foo', 'bar') - assert c.kv.get('foo', token=token)[1]['Value'] == six.b('bar') + assert c.kv.get('foo', token=token)[1]['Value'] == 'bar' pytest.raises( consul.ACLPermissionDenied, c.kv.put, 'foo', 'bar2', token=token) pytest.raises( consul.ACLPermissionDenied, c.kv.delete, 'foo', token=token) - assert c.kv.get('private/foo')[1]['Value'] == six.b('bar') + assert c.kv.get('private/foo')[1]['Value'] == 'bar' assert c.kv.get('private/foo', token=token)[1] is None pytest.raises( consul.ACLPermissionDenied, @@ -828,13 +828,13 @@ def test_acl_implicit_token_use(self, acl_consul): c.kv.put('private/foo', 'bar') c_limited = consul.Consul(port=acl_consul.port, token=token) - assert c_limited.kv.get('foo')[1]['Value'] == six.b('bar') + assert c_limited.kv.get('foo')[1]['Value'] == 'bar' pytest.raises( consul.ACLPermissionDenied, c_limited.kv.put, 'foo', 'bar2') pytest.raises( consul.ACLPermissionDenied, c_limited.kv.delete, 'foo') - assert c.kv.get('private/foo')[1]['Value'] == six.b('bar') + assert c.kv.get('private/foo')[1]['Value'] == 'bar' assert c_limited.kv.get('private/foo')[1] is None pytest.raises( consul.ACLPermissionDenied, diff --git a/tests/test_tornado.py b/tests/test_tornado.py index dbe859be..fb90fe31 100644 --- a/tests/test_tornado.py +++ b/tests/test_tornado.py @@ -39,7 +39,7 @@ def main(): response = yield c.kv.put('foo', 'bar') assert response is True index, data = yield c.kv.get('foo') - assert data['Value'] == six.b('bar') + assert data['Value'] == 'bar' loop.stop() loop.run_sync(main) @@ -62,7 +62,7 @@ def main(): index, data = yield c.kv.get('foo') assert data is None index, data = yield c.kv.get('foo', index=index) - assert data['Value'] == six.b('bar') + assert data['Value'] == 'bar' loop.stop() @gen.coroutine @@ -116,7 +116,7 @@ def get(): index, data = yield c.kv.get('foo') assert data is None index, data = yield c.kv.get('foo', index=index) - assert data['Value'] == six.b('bar') + assert data['Value'] == 'bar' loop.stop() @gen.coroutine @@ -142,7 +142,7 @@ def main(): response = yield c.kv.put('foo', u'bar') assert response is True index, data = yield c.kv.get('foo') - assert data['Value'] == six.b('bar') + assert data['Value'] == 'bar' # test empty-string comes back as `None` response = yield c.kv.put('foo', '') diff --git a/tests/test_twisted.py b/tests/test_twisted.py index ce380005..19051c99 100644 --- a/tests/test_twisted.py +++ b/tests/test_twisted.py @@ -32,7 +32,7 @@ def test_kv(self, consul_port): response = yield c.kv.put('foo', 'bar') assert response is True index, data = yield c.kv.get('foo') - assert data['Value'] == six.b('bar') + assert data['Value'] == 'bar' @pytest.inlineCallbacks def test_kv_binary(self, consul_port): @@ -49,7 +49,7 @@ def test_kv_missing(self, consul_port): index, data = yield c.kv.get('foo') assert data is None index, data = yield c.kv.get('foo', index=index) - assert data['Value'] == six.b('bar') + assert data['Value'] == 'bar' @pytest.inlineCallbacks def test_kv_put_flags(self, consul_port): @@ -94,7 +94,7 @@ def put(): index, data = yield c.kv.get('foo') assert data is None index, data = yield c.kv.get('foo', index=index) - assert data['Value'] == six.b('bar') + assert data['Value'] == 'bar' @pytest.inlineCallbacks def test_transaction(self, consul_port): From 181df98e3fa1b2f7386faeef00d1553572082c76 Mon Sep 17 00:00:00 2001 From: Patrik Puchala Date: Thu, 2 Nov 2017 13:16:59 +0100 Subject: [PATCH 05/10] #172_issue - kv_value returns string instead of bytes (in python3) - converting bytes to string done via latin-1 decoding - docstring updated - tests updated --- tests/test_aio.py | 4 ++-- tests/test_std.py | 2 +- tests/test_tornado.py | 4 ++-- tests/test_twisted.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_aio.py b/tests/test_aio.py index c94d839d..559e1e47 100644 --- a/tests/test_aio.py +++ b/tests/test_aio.py @@ -52,7 +52,7 @@ def main(): assert c._loop is loop yield from c.kv.put('foo', struct.pack('i', 1000)) index, data = yield from c.kv.get('foo') - assert struct.unpack('i', data['Value']) == (1000,) + assert struct.unpack('i', data['Value'].encode('latin-1')) == (1000,) c.close() asyncio.set_event_loop(loop) @@ -64,7 +64,7 @@ def main(): c = consul.aio.Consul(port=consul_port, loop=loop) yield from c.kv.put('foo', struct.pack('i', 1000)) index, data = yield from c.kv.get('foo') - assert struct.unpack('i', data['Value']) == (1000,) + assert struct.unpack('i', data['Value'].encode('latin-1')) == (1000,) c.close() loop.run_until_complete(main()) diff --git a/tests/test_std.py b/tests/test_std.py index c0ebc266..f31dfac3 100644 --- a/tests/test_std.py +++ b/tests/test_std.py @@ -43,7 +43,7 @@ def test_kv_encoding(self, consul_port): # test binary c.kv.put('foo', struct.pack('i', 1000)) index, data = c.kv.get('foo') - assert struct.unpack('i', data['Value']) == (1000,) + assert struct.unpack('i', data['Value'].encode('latin-1')) == (1000,) # test unicode c.kv.put('foo', u'bar') diff --git a/tests/test_tornado.py b/tests/test_tornado.py index fb90fe31..e388c0fa 100644 --- a/tests/test_tornado.py +++ b/tests/test_tornado.py @@ -49,7 +49,7 @@ def main(): c = consul.tornado.Consul(port=consul_port) yield c.kv.put('foo', struct.pack('i', 1000)) index, data = yield c.kv.get('foo') - assert struct.unpack('i', data['Value']) == (1000,) + assert struct.unpack('i', data['Value'].encode('latin-1')) == (1000,) loop.stop() loop.run_sync(main) @@ -136,7 +136,7 @@ def main(): response = yield c.kv.put('foo', struct.pack('i', 1000)) assert response is True index, data = yield c.kv.get('foo') - assert struct.unpack('i', data['Value']) == (1000,) + assert struct.unpack('i', data['Value'].encode('latin-1')) == (1000,) # test unicode response = yield c.kv.put('foo', u'bar') diff --git a/tests/test_twisted.py b/tests/test_twisted.py index 19051c99..7f6293cb 100644 --- a/tests/test_twisted.py +++ b/tests/test_twisted.py @@ -39,7 +39,7 @@ def test_kv_binary(self, consul_port): c = consul.twisted.Consul(port=consul_port) yield c.kv.put('foo', struct.pack('i', 1000)) index, data = yield c.kv.get('foo') - assert struct.unpack('i', data['Value']) == (1000,) + assert struct.unpack('i', data['Value'].encode('latin-1')) == (1000,) @pytest.inlineCallbacks def test_kv_missing(self, consul_port): From 7461765639016f336e8562c93ee194a95668e56b Mon Sep 17 00:00:00 2001 From: Patrik Puchala Date: Thu, 2 Nov 2017 13:32:22 +0100 Subject: [PATCH 06/10] #172_issue - kv_value returns string instead of bytes (in python3) - converting bytes to string done via latin-1 decoding - docstring updated - tests updated --- tests/test_aio.py | 1 - tests/test_std.py | 1 - tests/test_tornado.py | 1 - tests/test_twisted.py | 1 - 4 files changed, 4 deletions(-) diff --git a/tests/test_aio.py b/tests/test_aio.py index 559e1e47..1900b77d 100644 --- a/tests/test_aio.py +++ b/tests/test_aio.py @@ -1,6 +1,5 @@ import base64 import pytest -import six import struct import sys diff --git a/tests/test_std.py b/tests/test_std.py index f31dfac3..9284dec4 100644 --- a/tests/test_std.py +++ b/tests/test_std.py @@ -4,7 +4,6 @@ import time import pytest -import six import consul import consul.std diff --git a/tests/test_tornado.py b/tests/test_tornado.py index e388c0fa..e2f27e53 100644 --- a/tests/test_tornado.py +++ b/tests/test_tornado.py @@ -3,7 +3,6 @@ import time import pytest -import six from tornado import ioloop from tornado import gen diff --git a/tests/test_twisted.py b/tests/test_twisted.py index 7f6293cb..cb7dc809 100644 --- a/tests/test_twisted.py +++ b/tests/test_twisted.py @@ -2,7 +2,6 @@ import struct import pytest -import six from twisted.internet import defer, reactor import consul From 78e6bb714e682ea9f24fdb60fc6a2a1bbe15f3db Mon Sep 17 00:00:00 2001 From: Patrik Puchala Date: Thu, 2 Nov 2017 13:58:48 +0100 Subject: [PATCH 07/10] #172_issue - pep8 update --- consul/base.py | 5 +++-- tests/test_aio.py | 6 ++++-- tests/test_tornado.py | 6 ++++-- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/consul/base.py b/consul/base.py index 32a9d3b0..38eeb6ee 100755 --- a/consul/base.py +++ b/consul/base.py @@ -198,7 +198,7 @@ def json( *one* returns only the first item of the list of items. empty lists are coerced to None. - *decode* if specified this key will be base64 decoded (bytes -> string in python3). + *decode* if specified this key will be base64 decoded *is_id* only the 'ID' field of the json object will be returned. """ @@ -213,7 +213,8 @@ def cb(response): if decode: for item in data: if item.get(decode) is not None: - item[decode] = base64.b64decode(item[decode]).decode('latin-1') + raw_value = base64.b64decode(item[decode]) + item[decode] = raw_value.decode('latin-1') if is_id: data = data['ID'] if one: diff --git a/tests/test_aio.py b/tests/test_aio.py index 1900b77d..6dd818e5 100644 --- a/tests/test_aio.py +++ b/tests/test_aio.py @@ -51,7 +51,8 @@ def main(): assert c._loop is loop yield from c.kv.put('foo', struct.pack('i', 1000)) index, data = yield from c.kv.get('foo') - assert struct.unpack('i', data['Value'].encode('latin-1')) == (1000,) + encoded = data['Value'].encode('latin-1') + assert struct.unpack('i', encoded) == (1000,) c.close() asyncio.set_event_loop(loop) @@ -63,7 +64,8 @@ def main(): c = consul.aio.Consul(port=consul_port, loop=loop) yield from c.kv.put('foo', struct.pack('i', 1000)) index, data = yield from c.kv.get('foo') - assert struct.unpack('i', data['Value'].encode('latin-1')) == (1000,) + encoded = data['Value'].encode('latin-1') + assert struct.unpack('i', encoded) == (1000,) c.close() loop.run_until_complete(main()) diff --git a/tests/test_tornado.py b/tests/test_tornado.py index e2f27e53..c0dc7d82 100644 --- a/tests/test_tornado.py +++ b/tests/test_tornado.py @@ -48,7 +48,8 @@ def main(): c = consul.tornado.Consul(port=consul_port) yield c.kv.put('foo', struct.pack('i', 1000)) index, data = yield c.kv.get('foo') - assert struct.unpack('i', data['Value'].encode('latin-1')) == (1000,) + encoded = data['Value'].encode('latin-1') + assert struct.unpack('i', encoded) == (1000,) loop.stop() loop.run_sync(main) @@ -135,7 +136,8 @@ def main(): response = yield c.kv.put('foo', struct.pack('i', 1000)) assert response is True index, data = yield c.kv.get('foo') - assert struct.unpack('i', data['Value'].encode('latin-1')) == (1000,) + encoded = data['Value'].encode('latin-1') + assert struct.unpack('i', encoded) == (1000,) # test unicode response = yield c.kv.put('foo', u'bar') From af43723ef21225b7063ac92b941c35ac604b44bf Mon Sep 17 00:00:00 2001 From: Patrik Puchala Date: Thu, 2 Nov 2017 14:38:12 +0100 Subject: [PATCH 08/10] #172_issue - pep8 update --- consul/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/consul/base.py b/consul/base.py index 38eeb6ee..308bcd32 100755 --- a/consul/base.py +++ b/consul/base.py @@ -215,6 +215,7 @@ def cb(response): if item.get(decode) is not None: raw_value = base64.b64decode(item[decode]) item[decode] = raw_value.decode('latin-1') + if is_id: data = data['ID'] if one: From 67827160541da2674f22a6e727b7aadbec79f8e3 Mon Sep 17 00:00:00 2001 From: Patrik Puchala Date: Thu, 2 Nov 2017 14:43:04 +0100 Subject: [PATCH 09/10] #172_issue - pep8 update --- consul/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/consul/base.py b/consul/base.py index 308bcd32..38eeb6ee 100755 --- a/consul/base.py +++ b/consul/base.py @@ -215,7 +215,6 @@ def cb(response): if item.get(decode) is not None: raw_value = base64.b64decode(item[decode]) item[decode] = raw_value.decode('latin-1') - if is_id: data = data['ID'] if one: From 0e866d350f1064981bec210560f04f68b0191960 Mon Sep 17 00:00:00 2001 From: Patrik Puchala Date: Thu, 2 Nov 2017 15:00:26 +0100 Subject: [PATCH 10/10] #172_issue - pep8 update --- consul/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/consul/base.py b/consul/base.py index 38eeb6ee..308bcd32 100755 --- a/consul/base.py +++ b/consul/base.py @@ -215,6 +215,7 @@ def cb(response): if item.get(decode) is not None: raw_value = base64.b64decode(item[decode]) item[decode] = raw_value.decode('latin-1') + if is_id: data = data['ID'] if one: