diff --git a/CHANGELOG.md b/CHANGELOG.md index 27dd1d75d..4e87b50f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,16 +1,22 @@ ## Release notes -## 0.12.3 -- Nov 22, 2019 -* Bugfix #675 (PR #705) networkx 2.4+ is now supported -* Bugfix #698 and #699 (PR #706) display table definition in doc string and help -* Bugfix #701 (PR #702) job reservation works with native python datatype support disabled +### 0.12.4 -- Jan 14, 2020 +* Support for simple scalar datatypes in blobs (#690) PR #709 +* Add support for the `serial` data type in declarations: alias for `bigint unsigned auto_increment` PR #713 +* Improve the log table to avoid primary key collisions PR #713 +* Improve documentation in README PR #713 + +### 0.12.3 -- Nov 22, 2019 +* Bugfix - networkx 2.4 causes error in diagrams (#675) PR #705 +* Bugfix - include table definition in doc string and help (#698, #699) PR #706 +* Bugfix - job reservation fails when native python datatype support is disabled (#701) PR #702 ### 0.12.2 -- Nov 11, 2019 -* Bugfix - Convoluted error thrown if there is a reference to a non-existent table attribute (#691) -* Bugfix - Insert into external does not trim leading slash if defined in `dj.config['stores']['']['location']` (#692) +* Bugfix - Convoluted error thrown if there is a reference to a non-existent table attribute (#691) PR #696 +* Bugfix - Insert into external does not trim leading slash if defined in `dj.config['stores']['']['location']` (#692) PR #693 ### 0.12.1 -- Nov 2, 2019 -* Bugfix - AttributeAdapter converts into a string (#684) +* Bugfix - AttributeAdapter converts into a string (#684) PR #688 ### 0.12.0 -- Oct 31, 2019 * Dropped support for Python 3.4 diff --git a/LNX-docker-compose.yml b/LNX-docker-compose.yml index f1d2db7e6..4eeefc574 100644 --- a/LNX-docker-compose.yml +++ b/LNX-docker-compose.yml @@ -33,8 +33,8 @@ services: " pip install --user nose nose-cov coveralls .; pip freeze | grep datajoint; - nosetests -vsw tests --with-coverage --cover-package=datajoint; coveralls; + nosetests -vsw tests --with-coverage --cover-package=datajoint; # jupyter notebook; " # ports: @@ -92,4 +92,4 @@ services: - ./tests/nginx/fullchain.pem:/certs/fullchain.pem - ./tests/nginx/privkey.pem:/certs/privkey.pem networks: - main: \ No newline at end of file + main: diff --git a/datajoint/blob.py b/datajoint/blob.py index 390ef04bd..2a5e6867f 100644 --- a/datajoint/blob.py +++ b/datajoint/blob.py @@ -1,5 +1,5 @@ """ -(De)serialization methods for python datatypes and numpy.ndarrays with provisions for mutual +(De)serialization methods for basic datatypes and numpy.ndarrays with provisions for mutual compatibility with Matlab-based serialization implemented by mYm. """ @@ -115,21 +115,25 @@ def read_blob(self, n_bytes=None): "P": self.read_sparse_array, # matlab sparse array -- not supported yet "S": self.read_struct, # matlab struct array "C": self.read_cell_array, # matlab cell array - # Python-native - "\xFF": self.read_none, # None - "\1": self.read_tuple, # a Sequence - "\2": self.read_list, # a MutableSequence - "\3": self.read_set, # a Set - "\4": self.read_dict, # a Mapping - "\5": self.read_string, # a UTF8-encoded string - "\6": self.read_bytes, # a ByteString - "F": self.read_recarray, # numpy array with fields, including recarrays - "d": self.read_decimal, # a decimal - "t": self.read_datetime, # date, time, or datetime - "u": self.read_uuid, # UUID + # basic data types + "\xFF": self.read_none, # None + "\x01": self.read_tuple, # a Sequence (e.g. tuple) + "\x02": self.read_list, # a MutableSequence (e.g. list) + "\x03": self.read_set, # a Set + "\x04": self.read_dict, # a Mapping (e.g. dict) + "\x05": self.read_string, # a UTF8-encoded string + "\x06": self.read_bytes, # a ByteString + "\x0a": self.read_int, # unbounded scalar int + "\x0b": self.read_bool, # scalar boolean + "\x0c": self.read_complex, # scalar 128-bit complex number + "\x0d": self.read_float, # scalar 64-bit float + "F": self.read_recarray, # numpy array with fields, including recarrays + "d": self.read_decimal, # a decimal + "t": self.read_datetime, # date, time, or datetime + "u": self.read_uuid, # UUID }[data_structure_code] except KeyError: - raise DataJointError('Unknown data structure code "%s"' % data_structure_code) + raise DataJointError('Unknown data structure code "%s". Upgrade datajoint.' % data_structure_code) v = call() if n_bytes is not None and self._pos - start != n_bytes: raise DataJointError('Blob length check failed! Invalid blob') @@ -146,13 +150,21 @@ def pack_blob(self, obj): # blob types in the expanded dj0 blob format self.set_dj0() + if not isinstance(obj, (np.ndarray, np.number)): + # python built-in data types + if isinstance(obj, bool): + return self.pack_bool(obj) + if isinstance(obj, int): + return self.pack_int(obj) + if isinstance(obj, complex): + return self.pack_complex(obj) + if isinstance(obj, float): + return self.pack_float(obj) if isinstance(obj, np.ndarray) and obj.dtype.fields: return self.pack_recarray(np.array(obj)) if isinstance(obj, np.number): return self.pack_array(np.array(obj)) - if isinstance(obj, (bool, np.bool, np.bool_)): - return self.pack_array(np.array(obj)) - if isinstance(obj, (float, int, complex)): + if isinstance(obj, (np.bool, np.bool_)): return self.pack_array(np.array(obj)) if isinstance(obj, (datetime.datetime, datetime.date, datetime.time)): return self.pack_datetime(obj) @@ -209,7 +221,7 @@ def pack_array(self, array): if is_complex: array, imaginary = np.real(array), np.imag(array) type_id = (rev_class_id[array.dtype] if array.dtype.char != 'U' - else rev_class_id[np.dtype('O')]) + else rev_class_id[np.dtype('O')]) if dtype_list[type_id] is None: raise DataJointError("Type %s is ambiguous or unknown" % array.dtype) @@ -251,6 +263,36 @@ def pack_recarray(self, array): def read_sparse_array(self): raise DataJointError('datajoint-python does not yet support sparse arrays. Issue (#590)') + def read_int(self): + return int.from_bytes(self.read_binary(self.read_value('uint16')), byteorder='little', signed=True) + + @staticmethod + def pack_int(v): + n_bytes = v.bit_length() // 8 + 1 + assert 0 < n_bytes <= 0xFFFF, 'Integers are limited to 65535 bytes' + return b"\x0a" + np.uint16(n_bytes).tobytes() + v.to_bytes(n_bytes, byteorder='little', signed=True) + + def read_bool(self): + return bool(self.read_value('bool')) + + @staticmethod + def pack_bool(v): + return b"\x0b" + np.array(v, dtype='bool').tobytes() + + def read_complex(self): + return complex(self.read_value('complex128')) + + @staticmethod + def pack_complex(v): + return b"\x0c" + np.array(v, dtype='complex128').tobytes() + + def read_float(self): + return float(self.read_value('float64')) + + @staticmethod + def pack_float(v): + return b"\x0d" + np.array(v, dtype='float64').tobytes() + def read_decimal(self): return Decimal(self.read_string()) @@ -269,7 +311,7 @@ def pack_string(s): def read_bytes(self): return self.read_binary(self.read_value()) - + @staticmethod def pack_bytes(s): return b"\6" + len_u64(s) + s diff --git a/datajoint/version.py b/datajoint/version.py index 18c950f21..1762cf69c 100644 --- a/datajoint/version.py +++ b/datajoint/version.py @@ -1,3 +1,3 @@ -__version__ = "0.12.3" +__version__ = "0.12.4" assert len(__version__) <= 10 # The log table limits version to the 10 characters diff --git a/docs-parts/intro/Releases_lang1.rst b/docs-parts/intro/Releases_lang1.rst index 3e6f10782..763ddb92e 100644 --- a/docs-parts/intro/Releases_lang1.rst +++ b/docs-parts/intro/Releases_lang1.rst @@ -1,18 +1,23 @@ +0.12.4 -- Jan 14, 2020 +* Support for simple scalar datatypes in blobs (#690) PR #709 +* Add support for the `serial` data type in declarations: alias for `bigint unsigned auto_increment` PR #713 +* Improve the log table to avoid primary key collisions PR #713 +* Improve documentation in README PR #713 + 0.12.3 -- Nov 22, 2019 ---------------------- * Bugfix - networkx 2.4 causes error in diagrams (#675) PR #705 -* Bugfix - include definition in doc string and help (#698, #699) PR #706 -* Bugfix - job reservation fails when native python datatype support is disabled (#701) PR #702 - +* Bugfix - include table definition in doc string and help (#698, #699) PR #706 +* Bugfix - job reservation fails when native python datatype support is disabled (#701) PR #702 0.12.2 -- Nov 11, 2019 ------------------------- -* Bugfix - Convoluted error thrown if there is a reference to a non-existent table attribute (#691) -* Bugfix - Insert into external does not trim leading slash if defined in `dj.config['stores']['']['location']` (#692) +* Bugfix - Convoluted error thrown if there is a reference to a non-existent table attribute (#691) PR #696 +* Bugfix - Insert into external does not trim leading slash if defined in `dj.config['stores']['']['location']` (#692) PR #693 0.12.1 -- Nov 2, 2019 ------------------------- -* Bugfix - AttributeAdapter converts into a string (#684) +* Bugfix - AttributeAdapter converts into a string (#684) PR #688 0.12.0 -- Oct 31, 2019 ------------------------- diff --git a/tests/schema_adapted.py b/tests/schema_adapted.py index 937a4d55f..1260f4431 100644 --- a/tests/schema_adapted.py +++ b/tests/schema_adapted.py @@ -1,5 +1,6 @@ import datajoint as dj import networkx as nx +import json from pathlib import Path import tempfile from datajoint import errors @@ -11,8 +12,8 @@ S3_CONN_INFO, protocol='s3', location='adapted/repo', - stage=tempfile.mkdtemp()) -} + stage=tempfile.mkdtemp())} + dj.config['stores'] = stores_config schema_name = PREFIX + '_test_custom_datatype' @@ -53,37 +54,36 @@ class Connectivity(dj.Manual): errors._switch_filepath_types(True) -class Filepath2GraphAdapter(dj.AttributeAdapter): +class LayoutToFilepath(dj.AttributeAdapter): + """ + An adapted data type that saves a graph layout into fixed filepath + """ attribute_type = 'filepath@repo_s3' @staticmethod - def get(obj): - s = open(obj, "r").read() - return nx.spring_layout( - nx.lollipop_graph(4, 2), seed=int(s)) + def get(path): + with open(path, "r") as f: + return json.load(f) @staticmethod - def put(obj): - path = Path( - dj.config['stores']['repo_s3']['stage'], 'sample.txt') - - f = open(path, "w") - f.write(str(obj*obj)) - f.close() - + def put(layout): + path = Path(dj.config['stores']['repo_s3']['stage'], 'layout.json') + with open(str(path), "w") as f: + json.dump(layout, f) return path -file2graph = Filepath2GraphAdapter() +layout_to_filepath = LayoutToFilepath() @schema -class Position(dj.Manual): +class Layout(dj.Manual): definition = """ - pos_id : int + # stores graph layout + -> Connectivity --- - seed_root: + layout: """ errors._switch_filepath_types(False) diff --git a/tests/test_adapted_attributes.py b/tests/test_adapted_attributes.py index e85fffa79..b0bf20412 100644 --- a/tests/test_adapted_attributes.py +++ b/tests/test_adapted_attributes.py @@ -1,9 +1,9 @@ import datajoint as dj import networkx as nx from itertools import zip_longest -from nose.tools import assert_true, assert_equal +from nose.tools import assert_true, assert_equal, assert_dict_equal from . import schema_adapted as adapted -from .schema_adapted import graph, file2graph +from .schema_adapted import graph, layout_to_filepath def test_adapted_type(): @@ -22,17 +22,25 @@ def test_adapted_type(): def test_adapted_filepath_type(): # https://github.com/datajoint/datajoint-python/issues/684 + dj.errors._switch_adapted_types(True) dj.errors._switch_filepath_types(True) - c = adapted.Position() - Position.insert([{'pos_id': 0, 'seed_root': 3}]) - result = (Position & 'pos_id=0').fetch1('seed_root') - assert_true(isinstance(result, dict)) - assert_equal(0.3761992090175474, result[1][0]) - assert_true(6 == len(result)) + c = adapted.Connectivity() + c.delete() + c.insert1((0, nx.lollipop_graph(4, 2))) + layout = nx.spring_layout(c.fetch1('conn_graph')) + # make json friendly + layout = {str(k): [round(r, ndigits=4) for r in v] for k, v in layout.items()} + t = adapted.Layout() + t.insert1((0, layout)) + result = t.fetch1('layout') + assert_dict_equal(result, layout) + + t.delete() c.delete() + dj.errors._switch_filepath_types(False) dj.errors._switch_adapted_types(False) diff --git a/tests/test_blob.py b/tests/test_blob.py index 4520a83c0..225fb775c 100644 --- a/tests/test_blob.py +++ b/tests/test_blob.py @@ -4,7 +4,7 @@ from datetime import datetime from datajoint.blob import pack, unpack from numpy.testing import assert_array_equal -from nose.tools import assert_equal, assert_true, \ +from nose.tools import assert_equal, assert_true, assert_false, \ assert_list_equal, assert_set_equal, assert_tuple_equal, assert_dict_equal @@ -23,9 +23,6 @@ def test_pack(): x = np.random.randn(10) assert_array_equal(x, unpack(pack(x)), "Arrays do not match!") - x = 7j - assert_equal(x, unpack(pack(x)), "Complex scalar does not match") - x = np.float32(np.random.randn(3, 4, 5)) assert_array_equal(x, unpack(pack(x)), "Arrays do not match!") @@ -33,13 +30,34 @@ def test_pack(): assert_array_equal(x, unpack(pack(x)), "Arrays do not match!") x = None - assert_true(x is None, "None did not match") + assert_true(unpack(pack(x)) is None, "None did not match") + + x = -255 + y = unpack(pack(x)) + assert_true(x == y and isinstance(y, int) and not isinstance(y, np.ndarray), "Scalar int did not match") + + x = -25523987234234287910987234987098245697129798713407812347 + y = unpack(pack(x)) + assert_true(x == y and isinstance(y, int) and not isinstance(y, np.ndarray), "Unbounded int did not match") + + x = 7. + y = unpack(pack(x)) + assert_true(x == y and isinstance(y, float) and not isinstance(y, np.ndarray), "Scalar float did not match") + + x = 7j + y = unpack(pack(x)) + assert_true(x == y and isinstance(y, complex) and not isinstance(y, np.ndarray), "Complex scalar did not match") + + x = True + assert_true(unpack(pack(x)) is True, "Scalar bool did not match") x = [None] assert_list_equal(x, unpack(pack(x))) - x = {'name': 'Anonymous', 'age': 15, 99: datetime.now(), 'range': [110, 190], (11,12): None} - assert_dict_equal(x, unpack(pack(x)), "Dict do not match!") + x = {'name': 'Anonymous', 'age': 15, 99: datetime.now(), 'range': [110, 190], (11, 12): None} + y = unpack(pack(x)) + assert_dict_equal(x, y, "Dict do not match!") + assert_false(isinstance(['range'][0], np.ndarray), "Scalar int was coerced into arrray.") x = uuid.uuid4() assert_equal(x, unpack(pack(x)), 'UUID did not match')