datajoint · eywalker · Apr 15, 2020 · Apr 13, 2020 · Apr 13, 2020 · Apr 13, 2020
diff --git a/.travis.yml b/.travis.yml
@@ -11,14 +11,20 @@ env:
 services: 
 - docker
 main: &main
-      stage: Alpine
+      stage: "Tests & Coverage: Alpine"
       os: linux
       dist: xenial # precise, trusty, xenial, bionic
       language: shell
       script:
         - docker-compose -f LNX-docker-compose.yml up --build --exit-code-from app
 jobs:
   include:
+    - stage: "Lint: Syntax"
+      language: python
+      install:
+        - pip install flake8
+      script:
+        - flake8 datajoint --count --select=E9,F63,F7,F82 --show-source --statistics
     - <<: *main
       env:
       - PY_VER: "3.8"
@@ -39,31 +45,15 @@ jobs:
       env:
       - PY_VER: "3.8"
       - MYSQL_VER: "8.0"
-    - <<: *main
-      env:
-      - PY_VER: "3.7"
-      - MYSQL_VER: "8.0"
-    - <<: *main
-      env:
-      - PY_VER: "3.6"
-      - MYSQL_VER: "8.0"
-    - <<: *main
-      env:
-      - PY_VER: "3.5"
-      - MYSQL_VER: "8.0"
     - <<: *main
       env:
       - PY_VER: "3.8"
       - MYSQL_VER: "5.6"
-    - <<: *main
-      env:
-      - PY_VER: "3.7"
-      - MYSQL_VER: "5.6"
-    - <<: *main
-      env:
-      - PY_VER: "3.6"
-      - MYSQL_VER: "5.6"
-    - <<: *main
-      env:
-      - PY_VER: "3.5"
-      - MYSQL_VER: "5.6"
+    - stage: "Lint: Style"
+      language: python
+      install:
+        - pip install flake8
+      script:
+        - |
+          flake8 --ignore=E121,E123,E126,E226,E24,E704,W503,W504,E722,F401,W605 datajoint \
+          --count --max-complexity=62 --max-line-length=127 --statistics
diff --git a/LNX-docker-compose.yml b/LNX-docker-compose.yml
@@ -3,14 +3,54 @@ x-net: &net
   networks:
       - main
 services:
- app:
+ db:
     <<: *net
-    image: datajoint/pydev:${PY_VER}-alpine${ALPINE_VER}
+    image: datajoint/mysql:$MYSQL_VER
+    environment:
+      - MYSQL_ROOT_PASSWORD=simple
+    # ports:
+    #   - "3306:3306"
+    # volumes:
+    #   - ./mysql/data:/var/lib/mysql
+  minio:
+    <<: *net
+    image: minio/minio:$MINIO_VER
+    environment:
+      - MINIO_ACCESS_KEY=datajoint
+      - MINIO_SECRET_KEY=datajoint
+    # ports:
+    #   - "9000:9000"
+    # volumes:
+    #   - ./minio/config:/root/.minio
+    #   - ./minio/data:/data
+    command: server /data
+    healthcheck:
+      test: ["CMD", "curl", "--fail", "http://minio:9000/minio/health/live"]
+      timeout: 5s
+      retries: 60
+      interval: 1s
+  fakeservices.datajoint.io:
+    <<: *net
+    image: raphaelguzman/nginx:v0.0.4
+    environment:
+    - ADD_db_TYPE=DATABASE
+    - ADD_db_ENDPOINT=db:3306
+    - ADD_minio_TYPE=MINIO
+    - ADD_minio_ENDPOINT=minio:9000
+    - ADD_minio_PREFIX=/
+    # ports:
+    #   - "9000:9000"
+    #   - "443:443"
+    #   - "3306:3306"
     depends_on:
       db:
         condition: service_healthy
       minio:
         condition: service_healthy
+  app:
+    <<: *net
+    image: datajoint/pydev:${PY_VER}-alpine${ALPINE_VER}
+    depends_on:
       fakeservices.datajoint.io:
         condition: service_healthy
     environment:
@@ -35,8 +75,7 @@ services:
        "
         pip install --user nose nose-cov coveralls .;
         pip freeze | grep datajoint;
-        coveralls;
-        nosetests -vsw tests --with-coverage --cover-package=datajoint;
+        nosetests -vsw tests --with-coverage --cover-package=datajoint && coveralls;
         # jupyter notebook;
        "
     # ports:
@@ -46,55 +85,5 @@ services:
       - .:/src
       - /tmp/.X11-unix:/tmp/.X11-unix:rw
       # - ./notebooks:/home/dja/notebooks
-  db:
-    <<: *net
-    image: datajoint/mysql:$MYSQL_VER
-    environment:
-      - MYSQL_ROOT_PASSWORD=simple
-    # ports:
-    #   - "3306:3306"
-    # volumes:
-    #   - ./mysql/data:/var/lib/mysql
-  minio:
-    <<: *net
-    image: minio/minio:$MINIO_VER
-    environment:
-      - MINIO_ACCESS_KEY=datajoint
-      - MINIO_SECRET_KEY=datajoint
-    # ports:
-    #   - "9000:9000"
-    # volumes:
-    #   - ./minio/config:/root/.minio
-    #   - ./minio/data:/data
-    command: server /data
-    healthcheck:
-      test: ["CMD", "curl", "--fail", "http://minio:9000/minio/health/live"]
-      timeout: 5s
-      retries: 60
-      interval: 1s
-  fakeservices.datajoint.io:
-    <<: *net
-    image: nginx:alpine
-    environment:
-      - URL=datajoint.io
-      - SUBDOMAINS=fakeservices
-      - MINIO_SERVER=http://minio:9000
-      - MYSQL_SERVER=db:3306
-    entrypoint: /entrypoint.sh
-    healthcheck:
-      test: wget --quiet --tries=1 --spider https://fakeservices.datajoint.io:443/minio/health/live || exit 1
-      timeout: 5s
-      retries: 300
-      interval: 1s
-    # ports:
-    #   - "9000:9000"
-    #   - "443:443"
-    #   - "3306:3306"
-    volumes:
-      - ./tests/nginx/base.conf:/base.conf
-      - ./tests/nginx/nginx.conf:/nginx.conf
-      - ./tests/nginx/entrypoint.sh:/entrypoint.sh
-      - ./tests/nginx/fullchain.pem:/certs/fullchain.pem
-      - ./tests/nginx/privkey.pem:/certs/privkey.pem
 networks:
   main:
diff --git a/datajoint/autopopulate.py b/datajoint/autopopulate.py
@@ -5,7 +5,7 @@
 import random
 import inspect
 from tqdm import tqdm
-from .expression import QueryExpression, AndList, U
+from .expression import QueryExpression, AndList
 from .errors import DataJointError, LostConnectionError
 from .table import FreeTable
 import signal

diff --git a/datajoint/blob.py b/datajoint/blob.py
@@ -73,10 +73,11 @@ def __init__(self, squeeze=False):
 
     def set_dj0(self):
         if not config.get('enable_python_native_blobs'):
-            raise DataJointError('v0.12+ python native blobs disabled. see also: https://github.com/datajoint/datajoint-python#python-native-blobs')
+            raise DataJointError("""v0.12+ python native blobs disabled.
+                See also: https://github.com/datajoint/datajoint-python#python-native-blobs""")
 
         self.protocol = b"dj0\0"  # when using new blob features
- 
+
     def squeeze(self, array, convert_to_scalar=True):
         """
         Simplify the input array - squeeze out all singleton dimensions.
@@ -308,7 +309,7 @@ def read_string(self):
     def pack_string(s):
         blob = s.encode()
         return b"\5" + len_u64(blob) + blob
- 
+
     def read_bytes(self):
         return self.read_binary(self.read_value())
 
@@ -346,7 +347,7 @@ def pack_set(self, t):
 
     def read_dict(self):
         return OrderedDict((self.read_blob(self.read_value()), self.read_blob(self.read_value()))
-                    for _ in range(self.read_value()))
+  for _ in range(self.read_value()))
 
     def pack_dict(self, d):
         return b"\4" + len_u64(d) + b"".join(
@@ -428,7 +429,7 @@ def read_zero_terminated_string(self):
         data = self._blob[self._pos:target].decode()
         self._pos = target + 1
         return data
- 
+
     def read_value(self, dtype='uint64', count=1):
         data = np.frombuffer(self._blob, dtype=dtype, count=count, offset=self._pos)
         self._pos += data.dtype.itemsize * data.size

diff --git a/datajoint/connection.py b/datajoint/connection.py
@@ -69,9 +69,9 @@ def conn(host=None, user=None, password=None, *, init_fun=None, reset=False, use
     :param password: mysql password
     :param init_fun: initialization function
     :param reset: whether the connection should be reset or not
-    :param use_tls: TLS encryption option. Valid options are: True (required), 
-                    False (required no TLS), None (TLS prefered, default), 
-                    dict (Manually specify values per 
+    :param use_tls: TLS encryption option. Valid options are: True (required),
+                    False (required no TLS), None (TLS prefered, default),
+                    dict (Manually specify values per
                     https://dev.mysql.com/doc/refman/5.7/en/connection-options.html
                         #encrypted-connection-options).
     """

diff --git a/datajoint/declare.py b/datajoint/declare.py
@@ -397,8 +397,8 @@ def substitute_special_type(match, category, foreign_key_sql, context):
     elif category in EXTERNAL_TYPES:
         if category == 'FILEPATH' and not _support_filepath_types():
             raise DataJointError("""
-            The filepath data type is disabled until complete validation. 
-            To turn it on as experimental feature, set the environment variable 
+            The filepath data type is disabled until complete validation.
+            To turn it on as experimental feature, set the environment variable
             {env} = TRUE or upgrade datajoint.
             """.format(env=FILEPATH_FEATURE_SWITCH))
         match['store'] = match['type'].split('@', 1)[1]
@@ -444,7 +444,7 @@ def compile_attribute(line, in_key, foreign_key_sql, context):
     else:
         if match['default']:
             quote = (match['default'].split('(')[0].upper() not in CONSTANT_LITERALS
-  and match['default'][0] not in '"\'')
+                     and match['default'][0] not in '"\'')
             match['default'] = 'NOT NULL DEFAULT ' + ('"%s"' if quote else "%s") % match['default']
         else:
             match['default'] = 'NOT NULL'

diff --git a/datajoint/dependencies.py b/datajoint/dependencies.py
@@ -120,4 +120,3 @@ def ancestors(self, full_table_name):
             nx.algorithms.dag.ancestors(self, full_table_name))
         return [full_table_name] + list(reversed(list(
             nx.algorithms.dag.topological_sort(nodes))))
-
diff --git a/datajoint/diagram.py b/datajoint/diagram.py
@@ -183,7 +183,8 @@ def _unite(lst):
         def __add__(self, arg):
             """
             :param arg: either another Diagram or a positive integer.
-            :return: Union of the diagrams when arg is another Diagram or an expansion downstream when arg is a positive integer.
+            :return: Union of the diagrams when arg is another Diagram
+                     or an expansion downstream when arg is a positive integer.
             """
             self = Diagram(self)   # copy
             try:
@@ -204,7 +205,8 @@ def __add__(self, arg):
         def __sub__(self, arg):
             """
             :param arg: either another Diagram or a positive integer.
-            :return: Difference of the diagrams when arg is another Diagram or an expansion upstream when arg is a positive integer.
+            :return: Difference of the diagrams when arg is another Diagram or
+                     an expansion upstream when arg is a positive integer.
             """
             self = Diagram(self)   # copy
             try:

diff --git a/datajoint/expression.py b/datajoint/expression.py
@@ -409,13 +409,14 @@ def preview(self, limit=None, width=None):
         tuples = tuples[:limit]
         columns = heading.names
         widths = {f: min(max([len(f)] +
-            [len(str(e)) for e in tuples[f]] if f in tuples.dtype.names else [len('=BLOB=')]) + 4, width) for f in columns}
+                  [len(str(e)) for e in tuples[f]] if f in tuples.dtype.names else
+                  [len('=BLOB=')]) + 4, width) for f in columns}
         templates = {f: '%%-%d.%ds' % (widths[f], widths[f]) for f in columns}
         return (
             ' '.join([templates[f] % ('*' + f if f in rel.primary_key else f) for f in columns]) + '\n' +
             ' '.join(['+' + '-' * (widths[column] - 2) + '+' for column in columns]) + '\n' +
             '\n'.join(' '.join(templates[f] % (tup[f] if f in tup.dtype.names else '=BLOB=')
-                for f in columns) for tup in tuples) +
+  for f in columns) for tup in tuples) +
             ('\n   ...\n' if has_more else '\n') +
             (' (Total: %d)\n' % len(rel) if config['display.show_tuple_count'] else ''))
 

diff --git a/datajoint/external.py b/datajoint/external.py
@@ -56,7 +56,7 @@ def definition(self):
         size      :bigint unsigned     # size of object in bytes
         attachment_name=null : varchar(255)  # the filename of an attachment
         filepath=null : varchar(1000)  # relative filepath or attachment filename
-        contents_hash=null : uuid      # used for the filepath datatype 
+        contents_hash=null : uuid      # used for the filepath datatype
         timestamp=CURRENT_TIMESTAMP  :timestamp   # automatic timestamp
         """
 
@@ -198,8 +198,8 @@ def upload_attachment(self, local_path):
         self._upload_file(local_path, external_path)
         # insert tracking info
         self.connection.query("""
-        INSERT INTO {tab} (hash, size, attachment_name) 
-        VALUES (%s, {size}, "{attachment_name}") 
+        INSERT INTO {tab} (hash, size, attachment_name)
+        VALUES (%s, {size}, "{attachment_name}")
         ON DUPLICATE KEY UPDATE timestamp=CURRENT_TIMESTAMP""".format(
                 tab=self.full_table_name,
                 size=Path(local_path).stat().st_size,
@@ -338,8 +338,8 @@ def delete(self, *, delete_external_files=None, limit=None, display_progress=Tru
             error_list = []
             for uuid, external_path in items:
                 try:
-                   count = (self & {'hash': uuid}).delete_quick(get_count=True)  # optimize
-                except Exception as err:
+  count = (self & {'hash': uuid}).delete_quick(get_count=True)  # optimize
+                except Exception:
                     pass   # if delete failed, do not remove the external file
                 else:
                     assert count in (0, 1)

diff --git a/datajoint/fetch.py b/datajoint/fetch.py
@@ -12,6 +12,7 @@
 from .settings import config
 from .utils import OrderedDict, safe_write
 
+
 class key:
     """
     object that allows requesting the primary key as an argument in expression.fetch()
@@ -32,7 +33,7 @@ def to_dicts(recarray):
 
 def _get(connection, attr, data, squeeze, download_path):
     """
-    This function is called for every attribute 
+    This function is called for every attribute
 
     :param connection: a dj.Connection object
     :param attr: attribute name from the table's heading
@@ -60,7 +61,7 @@ def _get(connection, attr, data, squeeze, download_path):
         # 4. Otherwise, download the remote file and return the new filepath
         _uuid = uuid.UUID(bytes=data) if attr.is_external else None
         attachment_name = (extern.get_attachment_name(_uuid) if attr.is_external
-                    else data.split(b"\0", 1)[0].decode())
+  else data.split(b"\0", 1)[0].decode())
         local_filepath = Path(download_path) / attachment_name
         if local_filepath.is_file():
             attachment_checksum = _uuid if attr.is_external else hash.uuid_from_buffer(data)
@@ -83,7 +84,7 @@ def _get(connection, attr, data, squeeze, download_path):
         return adapt(str(local_filepath))  # download file from remote store
 
     return adapt(uuid.UUID(bytes=data) if attr.uuid else (
-            blob.unpack(extern.get(uuid.UUID(bytes=data)) if attr.is_external else data, squeeze=squeeze) 
+            blob.unpack(extern.get(uuid.UUID(bytes=data)) if attr.is_external else data, squeeze=squeeze)
             if attr.is_blob else data))
 
 
@@ -159,7 +160,8 @@ def __call__(self, *attrs, offset=None, limit=None, order_by=None, format=None,
         if not (attrs or as_dict) and format is None:
             format = config['fetch_format']  # default to array
             if format not in {"array", "frame"}:
-                raise DataJointError('Invalid entry "{}" in datajoint.config["fetch_format"]: use "array" or "frame"'.format(format))
+                raise DataJointError('Invalid entry "{}" in datajoint.config["fetch_format"]: use "array" or "frame"'.format(
+                    format))
 
         if limit is None and offset is not None:
             warnings.warn('Offset set, but no limit. Setting limit to a large number. '
@@ -194,7 +196,7 @@ def __call__(self, *attrs, offset=None, limit=None, order_by=None, format=None,
                 try:
                     ret = np.array(ret, dtype=record_type)
                 except Exception as e:
-                    raise
+                    raise e
                 for name in heading:
                     ret[name] = list(map(partial(get, heading[name]), ret[name]))
                 if format == "frame":
Original file line number	Diff line number	Diff line change
Expand Up		@@ -120,4 +120,3 @@ def ancestors(self, full_table_name):
		nx.algorithms.dag.ancestors(self, full_table_name))
		return [full_table_name] + list(reversed(list(
		nx.algorithms.dag.topological_sort(nodes))))