From 681fb9724980cc42ef0041f2f628b48576361e49 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Fri, 22 Nov 2019 17:12:01 -0600 Subject: [PATCH 01/13] fix #690 -- blob packing/unpacking of native python bool, int, float, and complex --- datajoint/blob.py | 33 +++++++++++++++++++++++++++++---- tests/test_blob.py | 22 +++++++++++++++++----- 2 files changed, 46 insertions(+), 9 deletions(-) diff --git a/datajoint/blob.py b/datajoint/blob.py index 390ef04bd..3f447d313 100644 --- a/datajoint/blob.py +++ b/datajoint/blob.py @@ -116,13 +116,14 @@ def read_blob(self, n_bytes=None): "S": self.read_struct, # matlab struct array "C": self.read_cell_array, # matlab cell array # Python-native - "\xFF": self.read_none, # None + "\xFF": self.read_none, # None "\1": self.read_tuple, # a Sequence "\2": self.read_list, # a MutableSequence "\3": self.read_set, # a Set "\4": self.read_dict, # a Mapping "\5": self.read_string, # a UTF8-encoded string "\6": self.read_bytes, # a ByteString + "s": self.read_scalar, # python-native scalars: bool, int, float, and complex "F": self.read_recarray, # numpy array with fields, including recarrays "d": self.read_decimal, # a decimal "t": self.read_datetime, # date, time, or datetime @@ -146,13 +147,13 @@ def pack_blob(self, obj): # blob types in the expanded dj0 blob format self.set_dj0() + if isinstance(obj, (bool, float, int, complex)): + return self.pack_scalar(obj) if isinstance(obj, np.ndarray) and obj.dtype.fields: return self.pack_recarray(np.array(obj)) if isinstance(obj, np.number): return self.pack_array(np.array(obj)) - if isinstance(obj, (bool, np.bool, np.bool_)): - return self.pack_array(np.array(obj)) - if isinstance(obj, (float, int, complex)): + if isinstance(obj, (np.bool, np.bool_)): return self.pack_array(np.array(obj)) if isinstance(obj, (datetime.datetime, datetime.date, datetime.time)): return self.pack_datetime(obj) @@ -251,6 +252,30 @@ def pack_recarray(self, array): def read_sparse_array(self): raise DataJointError('datajoint-python does not yet support sparse arrays. Issue (#590)') + def read_scalar(self): + code = self.read_value('uint8') + if code == 1: # boolean + return bool(self.read_value('bool')) + if code == 2: # integer + return int(self.read_value('int64')) + if code == 3: # float + return float(self.read_value('float64')) + if code == 4: # complex + return complex(self.read_value('complex128')) + raise DataJointError('Unsupported code in blob. Upgrade datajoint.') + + @staticmethod + def pack_scalar(v): + if isinstance(v, bool): + return b"s\1\1" if v else b"s\1\0" + if isinstance(v, int): + return b"s\2" + np.array(v, dtype='int64').tobytes() + if isinstance(v, float): + return b"s\3" + np.array(v, dtype='float64').tobytes() + if isinstance(v, complex): + return b"s\4" + np.array(v, dtype='complex128').tobytes() + assert False + def read_decimal(self): return Decimal(self.read_string()) diff --git a/tests/test_blob.py b/tests/test_blob.py index 4520a83c0..6ef725e98 100644 --- a/tests/test_blob.py +++ b/tests/test_blob.py @@ -23,9 +23,6 @@ def test_pack(): x = np.random.randn(10) assert_array_equal(x, unpack(pack(x)), "Arrays do not match!") - x = 7j - assert_equal(x, unpack(pack(x)), "Complex scalar does not match") - x = np.float32(np.random.randn(3, 4, 5)) assert_array_equal(x, unpack(pack(x)), "Arrays do not match!") @@ -33,12 +30,27 @@ def test_pack(): assert_array_equal(x, unpack(pack(x)), "Arrays do not match!") x = None - assert_true(x is None, "None did not match") + assert_true(unpack(pack(x)) is None, "None did not match") + + x = 7 + y = unpack(pack(x)) + assert_true(x == y and isinstance(y, int), "Native int did not match") + + x = 7. + y = unpack(pack(x)) + assert_true(x == y and isinstance(y, float), "Native float did not match") + + x = 7j + y = unpack(pack(x)) + assert_true(x == y and isinstance(y, complex), "Native complex did not match") + + x = True + assert_true(unpack(pack(x)) is True, "Native bool did not match") x = [None] assert_list_equal(x, unpack(pack(x))) - x = {'name': 'Anonymous', 'age': 15, 99: datetime.now(), 'range': [110, 190], (11,12): None} + x = {'name': 'Anonymous', 'age': 15, 99: datetime.now(), 'range': [110, 190], (11, 12): None} assert_dict_equal(x, unpack(pack(x)), "Dict do not match!") x = uuid.uuid4() From a4e5382949ed8ed133c0953ffcaeb8c8eda4773b Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Fri, 22 Nov 2019 17:17:08 -0600 Subject: [PATCH 02/13] minor --- datajoint/blob.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datajoint/blob.py b/datajoint/blob.py index 3f447d313..e84e807b7 100644 --- a/datajoint/blob.py +++ b/datajoint/blob.py @@ -130,7 +130,7 @@ def read_blob(self, n_bytes=None): "u": self.read_uuid, # UUID }[data_structure_code] except KeyError: - raise DataJointError('Unknown data structure code "%s"' % data_structure_code) + raise DataJointError('Unknown data structure code "%s". Upgrade datajoint.' % data_structure_code) v = call() if n_bytes is not None and self._pos - start != n_bytes: raise DataJointError('Blob length check failed! Invalid blob') From e348426ed310a298419a1660e3e591674a65ad87 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Mon, 25 Nov 2019 12:57:26 -0600 Subject: [PATCH 03/13] reduce encoding length for native python types in blobs --- datajoint/blob.py | 83 +++++++++++++++++++++++++++------------------- tests/test_blob.py | 12 ++++--- 2 files changed, 56 insertions(+), 39 deletions(-) diff --git a/datajoint/blob.py b/datajoint/blob.py index e84e807b7..0b1862af1 100644 --- a/datajoint/blob.py +++ b/datajoint/blob.py @@ -117,17 +117,20 @@ def read_blob(self, n_bytes=None): "C": self.read_cell_array, # matlab cell array # Python-native "\xFF": self.read_none, # None - "\1": self.read_tuple, # a Sequence - "\2": self.read_list, # a MutableSequence - "\3": self.read_set, # a Set - "\4": self.read_dict, # a Mapping - "\5": self.read_string, # a UTF8-encoded string - "\6": self.read_bytes, # a ByteString - "s": self.read_scalar, # python-native scalars: bool, int, float, and complex - "F": self.read_recarray, # numpy array with fields, including recarrays - "d": self.read_decimal, # a decimal - "t": self.read_datetime, # date, time, or datetime - "u": self.read_uuid, # UUID + "\x01": self.read_tuple, # a Sequence (e.g. tuple) + "\x02": self.read_list, # a MutableSequence (e.g. list) + "\x03": self.read_set, # a Set + "\x04": self.read_dict, # a Mapping (e.g. dict) + "\x05": self.read_string, # a UTF8-encoded string + "\x06": self.read_bytes, # a ByteString + "\x0a": self.read_int, # python-native int + "\x0b": self.read_bool, # python-native bool + "\x0c": self.read_complex, # python-native complex + "\x0d": self.read_float, # python-native float + "F": self.read_recarray, # numpy array with fields, including recarrays + "d": self.read_decimal, # a decimal + "t": self.read_datetime, # date, time, or datetime + "u": self.read_uuid, # UUID }[data_structure_code] except KeyError: raise DataJointError('Unknown data structure code "%s". Upgrade datajoint.' % data_structure_code) @@ -147,8 +150,16 @@ def pack_blob(self, obj): # blob types in the expanded dj0 blob format self.set_dj0() - if isinstance(obj, (bool, float, int, complex)): - return self.pack_scalar(obj) + if not isinstance(obj, np.ndarray): + # python built-in data types + if isinstance(obj, bool): + return self.pack_bool(obj) + if isinstance(obj, int): + return self.pack_int(obj) + if isinstance(obj, complex): + return self.pack_complex(obj) + if isinstance(obj, float): + return self.pack_float(obj) if isinstance(obj, np.ndarray) and obj.dtype.fields: return self.pack_recarray(np.array(obj)) if isinstance(obj, np.number): @@ -252,29 +263,33 @@ def pack_recarray(self, array): def read_sparse_array(self): raise DataJointError('datajoint-python does not yet support sparse arrays. Issue (#590)') - def read_scalar(self): - code = self.read_value('uint8') - if code == 1: # boolean - return bool(self.read_value('bool')) - if code == 2: # integer - return int(self.read_value('int64')) - if code == 3: # float - return float(self.read_value('float64')) - if code == 4: # complex - return complex(self.read_value('complex128')) - raise DataJointError('Unsupported code in blob. Upgrade datajoint.') + def read_int(self): + return int(self.read_value('int64')) @staticmethod - def pack_scalar(v): - if isinstance(v, bool): - return b"s\1\1" if v else b"s\1\0" - if isinstance(v, int): - return b"s\2" + np.array(v, dtype='int64').tobytes() - if isinstance(v, float): - return b"s\3" + np.array(v, dtype='float64').tobytes() - if isinstance(v, complex): - return b"s\4" + np.array(v, dtype='complex128').tobytes() - assert False + def pack_int(v): + return b"\x0a" + np.array(v, dtype='int64').tobytes() + + def read_bool(self): + return bool(self.read_value('bool')) + + @staticmethod + def pack_bool(v): + return b"\x0b" + np.array(v, dtype='bool').tobytes() + + def read_complex(self): + return complex(self.read_value('complex128')) + + @staticmethod + def pack_complex(v): + return b"\x0c" + np.array(v, dtype='complex128').tobytes() + + def read_float(self): + return float(self.read_value('float64')) + + @staticmethod + def pack_float(v): + return b"\x0d" + np.array(v, dtype='float64').tobytes() def read_decimal(self): return Decimal(self.read_string()) diff --git a/tests/test_blob.py b/tests/test_blob.py index 6ef725e98..4bb4e66dd 100644 --- a/tests/test_blob.py +++ b/tests/test_blob.py @@ -4,7 +4,7 @@ from datetime import datetime from datajoint.blob import pack, unpack from numpy.testing import assert_array_equal -from nose.tools import assert_equal, assert_true, \ +from nose.tools import assert_equal, assert_true, assert_false, \ assert_list_equal, assert_set_equal, assert_tuple_equal, assert_dict_equal @@ -34,15 +34,15 @@ def test_pack(): x = 7 y = unpack(pack(x)) - assert_true(x == y and isinstance(y, int), "Native int did not match") + assert_true(x == y and isinstance(y, int) and not isinstance(y, np.ndarray), "Native int did not match") x = 7. y = unpack(pack(x)) - assert_true(x == y and isinstance(y, float), "Native float did not match") + assert_true(x == y and isinstance(y, float) and not isinstance(y, np.ndarray), "Native float did not match") x = 7j y = unpack(pack(x)) - assert_true(x == y and isinstance(y, complex), "Native complex did not match") + assert_true(x == y and isinstance(y, complex) and not isinstance(y, np.ndarray), "Native complex did not match") x = True assert_true(unpack(pack(x)) is True, "Native bool did not match") @@ -51,7 +51,9 @@ def test_pack(): assert_list_equal(x, unpack(pack(x))) x = {'name': 'Anonymous', 'age': 15, 99: datetime.now(), 'range': [110, 190], (11, 12): None} - assert_dict_equal(x, unpack(pack(x)), "Dict do not match!") + y = unpack(pack(x)) + assert_dict_equal(x, y, "Dict do not match!") + assert_false(isinstance(['range'][0], np.ndarray), "Python-native scalars did not match.") x = uuid.uuid4() assert_equal(x, unpack(pack(x)), 'UUID did not match') From 86a2c2cd5af09eb237642bf5d616fe602539702a Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Mon, 16 Dec 2019 13:00:47 -0600 Subject: [PATCH 04/13] ensure that np.number is encoded as a numpy scalar --- datajoint/blob.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datajoint/blob.py b/datajoint/blob.py index 0b1862af1..46d52f937 100644 --- a/datajoint/blob.py +++ b/datajoint/blob.py @@ -150,7 +150,7 @@ def pack_blob(self, obj): # blob types in the expanded dj0 blob format self.set_dj0() - if not isinstance(obj, np.ndarray): + if not isinstance(obj, (np.ndarray, np.number)): # python built-in data types if isinstance(obj, bool): return self.pack_bool(obj) From 106239c3f9ebd18715695537d82e0350079ae6d0 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Mon, 30 Dec 2019 09:50:50 -0600 Subject: [PATCH 05/13] add support for unbounded integers in blob serialization --- datajoint/blob.py | 4 ++-- tests/test_blob.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/datajoint/blob.py b/datajoint/blob.py index 46d52f937..c794d5aff 100644 --- a/datajoint/blob.py +++ b/datajoint/blob.py @@ -264,11 +264,11 @@ def read_sparse_array(self): raise DataJointError('datajoint-python does not yet support sparse arrays. Issue (#590)') def read_int(self): - return int(self.read_value('int64')) + return int.from_bytes(self.read_value('int64'), byteorder='little', signed=True) @staticmethod def pack_int(v): - return b"\x0a" + np.array(v, dtype='int64').tobytes() + return b"\x0a" + v.to_bytes((v.bit_length() + 7 + (v < 0)) // 8, byteorder='little', signed=True) def read_bool(self): return bool(self.read_value('bool')) diff --git a/tests/test_blob.py b/tests/test_blob.py index 4bb4e66dd..329be87a0 100644 --- a/tests/test_blob.py +++ b/tests/test_blob.py @@ -32,7 +32,7 @@ def test_pack(): x = None assert_true(unpack(pack(x)) is None, "None did not match") - x = 7 + x = -255 y = unpack(pack(x)) assert_true(x == y and isinstance(y, int) and not isinstance(y, np.ndarray), "Native int did not match") From eadde37d7c0a06ed949274f595b20b3003f89b3b Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Mon, 30 Dec 2019 09:52:38 -0600 Subject: [PATCH 06/13] add test for unbounded integer --- tests/test_blob.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_blob.py b/tests/test_blob.py index 329be87a0..a584bf320 100644 --- a/tests/test_blob.py +++ b/tests/test_blob.py @@ -36,6 +36,10 @@ def test_pack(): y = unpack(pack(x)) assert_true(x == y and isinstance(y, int) and not isinstance(y, np.ndarray), "Native int did not match") + x = -25523987234234287910987234987098245697129798713407812347 + y = unpack(pack(x)) + assert_true(x == y and isinstance(y, int) and not isinstance(y, np.ndarray), "Native int did not match") + x = 7. y = unpack(pack(x)) assert_true(x == y and isinstance(y, float) and not isinstance(y, np.ndarray), "Native float did not match") From f1e6da60ab404ec2e908d66f59defb2cf645733f Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Mon, 30 Dec 2019 14:50:41 -0600 Subject: [PATCH 07/13] update CHANGELOG and version for release 0.12.4 --- CHANGELOG.md | 9 ++++++++- datajoint/version.py | 2 +- tests/test_blob.py | 12 ++++++------ 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 27dd1d75d..562af2acd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,13 @@ ## Release notes -## 0.12.3 -- Nov 22, 2019 +### 0.12.4 -- Jan 03, 2020 +* Support for simple scalar datatypes in blobs (#690 and PR #706) +* Add support for the `serial` data type in declarations: alias for `bigint unsigned auto_increment` (#713) +* Improve the log table to avoid primary key collisions (#713) +* Improve documentation in README + + +### 0.12.3 -- Nov 22, 2019 * Bugfix #675 (PR #705) networkx 2.4+ is now supported * Bugfix #698 and #699 (PR #706) display table definition in doc string and help * Bugfix #701 (PR #702) job reservation works with native python datatype support disabled diff --git a/datajoint/version.py b/datajoint/version.py index 18c950f21..1762cf69c 100644 --- a/datajoint/version.py +++ b/datajoint/version.py @@ -1,3 +1,3 @@ -__version__ = "0.12.3" +__version__ = "0.12.4" assert len(__version__) <= 10 # The log table limits version to the 10 characters diff --git a/tests/test_blob.py b/tests/test_blob.py index a584bf320..225fb775c 100644 --- a/tests/test_blob.py +++ b/tests/test_blob.py @@ -34,22 +34,22 @@ def test_pack(): x = -255 y = unpack(pack(x)) - assert_true(x == y and isinstance(y, int) and not isinstance(y, np.ndarray), "Native int did not match") + assert_true(x == y and isinstance(y, int) and not isinstance(y, np.ndarray), "Scalar int did not match") x = -25523987234234287910987234987098245697129798713407812347 y = unpack(pack(x)) - assert_true(x == y and isinstance(y, int) and not isinstance(y, np.ndarray), "Native int did not match") + assert_true(x == y and isinstance(y, int) and not isinstance(y, np.ndarray), "Unbounded int did not match") x = 7. y = unpack(pack(x)) - assert_true(x == y and isinstance(y, float) and not isinstance(y, np.ndarray), "Native float did not match") + assert_true(x == y and isinstance(y, float) and not isinstance(y, np.ndarray), "Scalar float did not match") x = 7j y = unpack(pack(x)) - assert_true(x == y and isinstance(y, complex) and not isinstance(y, np.ndarray), "Native complex did not match") + assert_true(x == y and isinstance(y, complex) and not isinstance(y, np.ndarray), "Complex scalar did not match") x = True - assert_true(unpack(pack(x)) is True, "Native bool did not match") + assert_true(unpack(pack(x)) is True, "Scalar bool did not match") x = [None] assert_list_equal(x, unpack(pack(x))) @@ -57,7 +57,7 @@ def test_pack(): x = {'name': 'Anonymous', 'age': 15, 99: datetime.now(), 'range': [110, 190], (11, 12): None} y = unpack(pack(x)) assert_dict_equal(x, y, "Dict do not match!") - assert_false(isinstance(['range'][0], np.ndarray), "Python-native scalars did not match.") + assert_false(isinstance(['range'][0], np.ndarray), "Scalar int was coerced into arrray.") x = uuid.uuid4() assert_equal(x, unpack(pack(x)), 'UUID did not match') From 392d56a61ae56008e2e42774a046c1236a4ac9fb Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Tue, 31 Dec 2019 13:09:59 -0600 Subject: [PATCH 08/13] correct computation of number of bits for unbounded integers in blobs --- datajoint/blob.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datajoint/blob.py b/datajoint/blob.py index c794d5aff..3322d07ef 100644 --- a/datajoint/blob.py +++ b/datajoint/blob.py @@ -268,7 +268,7 @@ def read_int(self): @staticmethod def pack_int(v): - return b"\x0a" + v.to_bytes((v.bit_length() + 7 + (v < 0)) // 8, byteorder='little', signed=True) + return b"\x0a" + v.to_bytes(v.bit_length() // 8 + 1, byteorder='little', signed=True) def read_bool(self): return bool(self.read_value('bool')) From 61362e78749e6860750bdcb8d6dc97a36c425b6f Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Fri, 3 Jan 2020 10:15:38 -0600 Subject: [PATCH 09/13] fix unbounded integer encoding in blobs --- datajoint/blob.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/datajoint/blob.py b/datajoint/blob.py index 3322d07ef..512e55e2a 100644 --- a/datajoint/blob.py +++ b/datajoint/blob.py @@ -221,7 +221,7 @@ def pack_array(self, array): if is_complex: array, imaginary = np.real(array), np.imag(array) type_id = (rev_class_id[array.dtype] if array.dtype.char != 'U' - else rev_class_id[np.dtype('O')]) + else rev_class_id[np.dtype('O')]) if dtype_list[type_id] is None: raise DataJointError("Type %s is ambiguous or unknown" % array.dtype) @@ -264,11 +264,13 @@ def read_sparse_array(self): raise DataJointError('datajoint-python does not yet support sparse arrays. Issue (#590)') def read_int(self): - return int.from_bytes(self.read_value('int64'), byteorder='little', signed=True) + return int.from_bytes(self.read_binary(self.read_value('uint16')), byteorder='little', signed=True) @staticmethod def pack_int(v): - return b"\x0a" + v.to_bytes(v.bit_length() // 8 + 1, byteorder='little', signed=True) + n_bytes = v.bit_length() // 8 + 1 + assert n_bytes <= 0xFFFF, 'Integers are limited to 65535 bytes' + return b"\x0a" + np.uint16(n_bytes).tobytes() + v.to_bytes(n_bytes, byteorder='little', signed=True) def read_bool(self): return bool(self.read_value('bool')) @@ -309,7 +311,7 @@ def pack_string(s): def read_bytes(self): return self.read_binary(self.read_value()) - + @staticmethod def pack_bytes(s): return b"\6" + len_u64(s) + s From 4a56d42e813cafff1e6caeb39a2457a29271243d Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Fri, 3 Jan 2020 10:34:05 -0600 Subject: [PATCH 10/13] fix bug in LNX-docker-compose.yml --- LNX-docker-compose.yml | 4 ++-- datajoint/blob.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/LNX-docker-compose.yml b/LNX-docker-compose.yml index f1d2db7e6..4eeefc574 100644 --- a/LNX-docker-compose.yml +++ b/LNX-docker-compose.yml @@ -33,8 +33,8 @@ services: " pip install --user nose nose-cov coveralls .; pip freeze | grep datajoint; - nosetests -vsw tests --with-coverage --cover-package=datajoint; coveralls; + nosetests -vsw tests --with-coverage --cover-package=datajoint; # jupyter notebook; " # ports: @@ -92,4 +92,4 @@ services: - ./tests/nginx/fullchain.pem:/certs/fullchain.pem - ./tests/nginx/privkey.pem:/certs/privkey.pem networks: - main: \ No newline at end of file + main: diff --git a/datajoint/blob.py b/datajoint/blob.py index 512e55e2a..e97a482df 100644 --- a/datajoint/blob.py +++ b/datajoint/blob.py @@ -269,7 +269,7 @@ def read_int(self): @staticmethod def pack_int(v): n_bytes = v.bit_length() // 8 + 1 - assert n_bytes <= 0xFFFF, 'Integers are limited to 65535 bytes' + assert 0 < n_bytes <= 0xFFFF, 'Integers are limited to 65535 bytes' return b"\x0a" + np.uint16(n_bytes).tobytes() + v.to_bytes(n_bytes, byteorder='little', signed=True) def read_bool(self): From 876d62aebdc57aa3c8ad7b678be6b6d6c81abb3c Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Fri, 3 Jan 2020 14:28:35 -0600 Subject: [PATCH 11/13] improve tests for adapted attributes --- tests/schema_adapted.py | 38 ++++++++++++++++---------------- tests/test_adapted_attributes.py | 24 +++++++++++++------- 2 files changed, 35 insertions(+), 27 deletions(-) diff --git a/tests/schema_adapted.py b/tests/schema_adapted.py index 937a4d55f..1260f4431 100644 --- a/tests/schema_adapted.py +++ b/tests/schema_adapted.py @@ -1,5 +1,6 @@ import datajoint as dj import networkx as nx +import json from pathlib import Path import tempfile from datajoint import errors @@ -11,8 +12,8 @@ S3_CONN_INFO, protocol='s3', location='adapted/repo', - stage=tempfile.mkdtemp()) -} + stage=tempfile.mkdtemp())} + dj.config['stores'] = stores_config schema_name = PREFIX + '_test_custom_datatype' @@ -53,37 +54,36 @@ class Connectivity(dj.Manual): errors._switch_filepath_types(True) -class Filepath2GraphAdapter(dj.AttributeAdapter): +class LayoutToFilepath(dj.AttributeAdapter): + """ + An adapted data type that saves a graph layout into fixed filepath + """ attribute_type = 'filepath@repo_s3' @staticmethod - def get(obj): - s = open(obj, "r").read() - return nx.spring_layout( - nx.lollipop_graph(4, 2), seed=int(s)) + def get(path): + with open(path, "r") as f: + return json.load(f) @staticmethod - def put(obj): - path = Path( - dj.config['stores']['repo_s3']['stage'], 'sample.txt') - - f = open(path, "w") - f.write(str(obj*obj)) - f.close() - + def put(layout): + path = Path(dj.config['stores']['repo_s3']['stage'], 'layout.json') + with open(str(path), "w") as f: + json.dump(layout, f) return path -file2graph = Filepath2GraphAdapter() +layout_to_filepath = LayoutToFilepath() @schema -class Position(dj.Manual): +class Layout(dj.Manual): definition = """ - pos_id : int + # stores graph layout + -> Connectivity --- - seed_root: + layout: """ errors._switch_filepath_types(False) diff --git a/tests/test_adapted_attributes.py b/tests/test_adapted_attributes.py index e85fffa79..b0bf20412 100644 --- a/tests/test_adapted_attributes.py +++ b/tests/test_adapted_attributes.py @@ -1,9 +1,9 @@ import datajoint as dj import networkx as nx from itertools import zip_longest -from nose.tools import assert_true, assert_equal +from nose.tools import assert_true, assert_equal, assert_dict_equal from . import schema_adapted as adapted -from .schema_adapted import graph, file2graph +from .schema_adapted import graph, layout_to_filepath def test_adapted_type(): @@ -22,17 +22,25 @@ def test_adapted_type(): def test_adapted_filepath_type(): # https://github.com/datajoint/datajoint-python/issues/684 + dj.errors._switch_adapted_types(True) dj.errors._switch_filepath_types(True) - c = adapted.Position() - Position.insert([{'pos_id': 0, 'seed_root': 3}]) - result = (Position & 'pos_id=0').fetch1('seed_root') - assert_true(isinstance(result, dict)) - assert_equal(0.3761992090175474, result[1][0]) - assert_true(6 == len(result)) + c = adapted.Connectivity() + c.delete() + c.insert1((0, nx.lollipop_graph(4, 2))) + layout = nx.spring_layout(c.fetch1('conn_graph')) + # make json friendly + layout = {str(k): [round(r, ndigits=4) for r in v] for k, v in layout.items()} + t = adapted.Layout() + t.insert1((0, layout)) + result = t.fetch1('layout') + assert_dict_equal(result, layout) + + t.delete() c.delete() + dj.errors._switch_filepath_types(False) dj.errors._switch_adapted_types(False) From 8a3c9a1311b527ed774edf060226cf0e3bb0ec24 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Fri, 3 Jan 2020 16:45:02 -0600 Subject: [PATCH 12/13] update comment to use general data types rather than python-focused --- datajoint/blob.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/datajoint/blob.py b/datajoint/blob.py index e97a482df..2a5e6867f 100644 --- a/datajoint/blob.py +++ b/datajoint/blob.py @@ -1,5 +1,5 @@ """ -(De)serialization methods for python datatypes and numpy.ndarrays with provisions for mutual +(De)serialization methods for basic datatypes and numpy.ndarrays with provisions for mutual compatibility with Matlab-based serialization implemented by mYm. """ @@ -115,7 +115,7 @@ def read_blob(self, n_bytes=None): "P": self.read_sparse_array, # matlab sparse array -- not supported yet "S": self.read_struct, # matlab struct array "C": self.read_cell_array, # matlab cell array - # Python-native + # basic data types "\xFF": self.read_none, # None "\x01": self.read_tuple, # a Sequence (e.g. tuple) "\x02": self.read_list, # a MutableSequence (e.g. list) @@ -123,10 +123,10 @@ def read_blob(self, n_bytes=None): "\x04": self.read_dict, # a Mapping (e.g. dict) "\x05": self.read_string, # a UTF8-encoded string "\x06": self.read_bytes, # a ByteString - "\x0a": self.read_int, # python-native int - "\x0b": self.read_bool, # python-native bool - "\x0c": self.read_complex, # python-native complex - "\x0d": self.read_float, # python-native float + "\x0a": self.read_int, # unbounded scalar int + "\x0b": self.read_bool, # scalar boolean + "\x0c": self.read_complex, # scalar 128-bit complex number + "\x0d": self.read_float, # scalar 64-bit float "F": self.read_recarray, # numpy array with fields, including recarrays "d": self.read_decimal, # a decimal "t": self.read_datetime, # date, time, or datetime From 92f56ab3875a04fac3a25c41848a6db7e33d46a6 Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Tue, 14 Jan 2020 12:42:11 -0600 Subject: [PATCH 13/13] Update release details. --- CHANGELOG.md | 23 +++++++++++------------ docs-parts/intro/Releases_lang1.rst | 17 +++++++++++------ 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 562af2acd..4e87b50f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,23 +1,22 @@ ## Release notes -### 0.12.4 -- Jan 03, 2020 -* Support for simple scalar datatypes in blobs (#690 and PR #706) -* Add support for the `serial` data type in declarations: alias for `bigint unsigned auto_increment` (#713) -* Improve the log table to avoid primary key collisions (#713) -* Improve documentation in README - +### 0.12.4 -- Jan 14, 2020 +* Support for simple scalar datatypes in blobs (#690) PR #709 +* Add support for the `serial` data type in declarations: alias for `bigint unsigned auto_increment` PR #713 +* Improve the log table to avoid primary key collisions PR #713 +* Improve documentation in README PR #713 ### 0.12.3 -- Nov 22, 2019 -* Bugfix #675 (PR #705) networkx 2.4+ is now supported -* Bugfix #698 and #699 (PR #706) display table definition in doc string and help -* Bugfix #701 (PR #702) job reservation works with native python datatype support disabled +* Bugfix - networkx 2.4 causes error in diagrams (#675) PR #705 +* Bugfix - include table definition in doc string and help (#698, #699) PR #706 +* Bugfix - job reservation fails when native python datatype support is disabled (#701) PR #702 ### 0.12.2 -- Nov 11, 2019 -* Bugfix - Convoluted error thrown if there is a reference to a non-existent table attribute (#691) -* Bugfix - Insert into external does not trim leading slash if defined in `dj.config['stores']['']['location']` (#692) +* Bugfix - Convoluted error thrown if there is a reference to a non-existent table attribute (#691) PR #696 +* Bugfix - Insert into external does not trim leading slash if defined in `dj.config['stores']['']['location']` (#692) PR #693 ### 0.12.1 -- Nov 2, 2019 -* Bugfix - AttributeAdapter converts into a string (#684) +* Bugfix - AttributeAdapter converts into a string (#684) PR #688 ### 0.12.0 -- Oct 31, 2019 * Dropped support for Python 3.4 diff --git a/docs-parts/intro/Releases_lang1.rst b/docs-parts/intro/Releases_lang1.rst index 3e6f10782..763ddb92e 100644 --- a/docs-parts/intro/Releases_lang1.rst +++ b/docs-parts/intro/Releases_lang1.rst @@ -1,18 +1,23 @@ +0.12.4 -- Jan 14, 2020 +* Support for simple scalar datatypes in blobs (#690) PR #709 +* Add support for the `serial` data type in declarations: alias for `bigint unsigned auto_increment` PR #713 +* Improve the log table to avoid primary key collisions PR #713 +* Improve documentation in README PR #713 + 0.12.3 -- Nov 22, 2019 ---------------------- * Bugfix - networkx 2.4 causes error in diagrams (#675) PR #705 -* Bugfix - include definition in doc string and help (#698, #699) PR #706 -* Bugfix - job reservation fails when native python datatype support is disabled (#701) PR #702 - +* Bugfix - include table definition in doc string and help (#698, #699) PR #706 +* Bugfix - job reservation fails when native python datatype support is disabled (#701) PR #702 0.12.2 -- Nov 11, 2019 ------------------------- -* Bugfix - Convoluted error thrown if there is a reference to a non-existent table attribute (#691) -* Bugfix - Insert into external does not trim leading slash if defined in `dj.config['stores']['']['location']` (#692) +* Bugfix - Convoluted error thrown if there is a reference to a non-existent table attribute (#691) PR #696 +* Bugfix - Insert into external does not trim leading slash if defined in `dj.config['stores']['']['location']` (#692) PR #693 0.12.1 -- Nov 2, 2019 ------------------------- -* Bugfix - AttributeAdapter converts into a string (#684) +* Bugfix - AttributeAdapter converts into a string (#684) PR #688 0.12.0 -- Oct 31, 2019 -------------------------