add max_allowed_packet and bugfix in insert1 for tuples with null/nan values

Dockerfile

@@ -0,0 +1,28 @@
+FROM ubuntu:14.04
+
+MAINTAINER Fabian Sinz <sinz@bcm.edu>
+
+
+RUN \
+  apt-get update && \
+  apt-get install -y -q \
+    git
+
+RUN \
+  apt-get update && \
+  apt-get install -y -q \
+    python3 \
+    python3-pip \
+    python3-numpy \
+    python3-scipy \
+    python3-matplotlib \
+    python3-pandas \
+    ipython3 \
+    ipython3-notebook \
+    mysql-server \
+    mysql-client
+
+RUN pip3 install git+https://github.com/datajoint/datajoint-python.git
+
+RUN pip3 install git+https://github.com/datajoint/datajoint-addons.git
+

datajoint/__init__.py

@@ -33,18 +33,33 @@ class DataJointError(Exception):
     pass
 
 # ----------- loads local configuration from file ----------------
-from .settings import Config, CONFIGVAR, LOCALCONFIG, logger, log_levels
+from .settings import Config, LOCALCONFIG, GLOBALCONFIG, logger, log_levels
 config = Config()
-local_config_file = os.environ.get(CONFIGVAR, None)
-if local_config_file is None:
-    local_config_file = LOCALCONFIG
-local_config_file = os.path.expanduser(local_config_file)
 
-try:
+
+if os.getenv('DJ_HOST') is not None and os.getenv('DJ_USER') is not None and os.getenv('DJ_PASS') is not None:
+    print("Loading local settings from environment variables")
+    config['database.host'] = os.getenv('DJ_HOST')
+    config['database.user'] = os.getenv('DJ_USER')
+    config['database.password'] = os.getenv('DJ_PASS')
+elif os.path.exists(LOCALCONFIG):
+    local_config_file = os.path.expanduser(LOCALCONFIG)
+    print("Loading local settings from {0:s}".format(local_config_file))
+    logger.log(logging.INFO, "Loading local settings from {0:s}".format(local_config_file))
+    config.load(local_config_file)
+elif os.path.exists(os.path.expanduser('~/') + GLOBALCONFIG):
+    local_config_file = os.path.expanduser('~/') + GLOBALCONFIG
+    print("Loading local settings from {0:s}".format(local_config_file))
     logger.log(logging.INFO, "Loading local settings from {0:s}".format(local_config_file))
     config.load(local_config_file)
-except FileNotFoundError:
-    logger.warn("Local config file {0:s} does not exist! Creating it.".format(local_config_file))
+else:
+    print("""Cannot find configuration settings. Using default configuration. To change that, either
+    * modify the local copy of %s that datajoint just saved for you
+    * put a file named %s with the same configuration format in your home
+    * specify the environment variables DJ_USER, DJ_HOST, DJ_PASS
+          """)
+    local_config_file = os.path.expanduser(LOCALCONFIG)
+    logger.log(logging.INFO, "No config found. Generating {0:s}".format(local_config_file))
     config.save(local_config_file)
 
 logger.setLevel(log_levels[config['loglevel']])

datajoint/base_relation.py

@@ -250,7 +250,8 @@ def make_attribute(name, value):
             raise DataJointError('Empty tuple')
         skip = skip_duplicates
         if skip:
-            primary_key_value = {name: value for name, _, value in attributes if heading[name].in_key}
+            primary_key_value = {name: value for name, _, value in attributes if
+                                 name is not None and heading[name].in_key}
             # if primary key value is empty, auto_populate is probably used
             skip = primary_key_value and (self & primary_key_value)
         if not skip:
@@ -260,7 +261,7 @@ def make_attribute(name, value):
                 sql = 'INSERT IGNORE'
             else:
                 sql = 'INSERT'
-            attributes = (a for a in attributes if a[0])   # omit dropped attributes
+            attributes = (a for a in attributes if a[0] is not None)  # omit dropped attributes
             names, placeholders, values = tuple(zip(*attributes))
             sql += " INTO %s (`%s`) VALUES (%s)" % (
                 self.from_clause, '`,`'.join(names), ','.join(placeholders))

datajoint/connection.py

@@ -2,7 +2,6 @@
 This module hosts the Connection class that manages the connection to the mysql database,
  and the `conn` function that provides access to a persistent connection in datajoint.
 """
-
 from contextlib import contextmanager
 import pymysql as client
 import logging
@@ -59,7 +58,8 @@ def __init__(self, host, user, passwd, init_fun=None):
             port = int(port)
         else:
             port = config['database.port']
-        self.conn_info = dict(host=host, port=port, user=user, passwd=passwd)
+        self.conn_info = dict(host=host, port=port, user=user, passwd=passwd,
+                              max_allowed_packet=1024 ** 3)  # this is a hack to fix problems with pymysql (remove later)
         self._conn = client.connect(init_command=init_fun, **self.conn_info)
         if self.is_connected:
             logger.info("Connected {user}@{host}:{port}".format(**self.conn_info))
@@ -111,7 +111,6 @@ def query(self, query, args=(), as_dict=False):
 
         # Log the query
         logger.debug("Executing SQL:" + query[0:300])
-
         cur.execute(query, args)
 
         return cur
@@ -161,8 +160,8 @@ def commit_transaction(self):
     def transaction(self):
         """
         Context manager for transactions. Opens an transaction and closes it after the with statement.
-        If an error is caught during the transaction, the commits are automatically rolled back. All
-        errors are raised again.
+        If an error is caught during the transaction, the commits are automatically rolled back.
+        All errors are raised again.
 
         Example:
         >>> import datajoint as dj

datajoint/settings.py

@@ -12,7 +12,7 @@
 from enum import Enum
 
 LOCALCONFIG = 'dj_local_conf.json'
-CONFIGVAR = 'DJ_LOCAL_CONF'
+GLOBALCONFIG = '._datajoint_config.json'
 
 validators = collections.defaultdict(lambda: lambda value: True)
 validators['database.port'] = lambda a: isinstance(a, int)

tests/test_nan.py

@@ -15,12 +15,24 @@ class NanTest(dj.Manual):
     """
 
 
-def test_insert_nan():
-    rel = NanTest()
-    a = np.array([0, 1/3, np.nan, np.pi, np.nan])
-    rel.insert(((i, value) for i, value in enumerate(a)))
-    b = rel.fetch.order_by('id')['value']
-    assert_true((np.isnan(a) == np.isnan(b)).all(),
-                'incorrect handling of Nans')
-    assert_true(np.allclose(a[np.logical_not(np.isnan(a))], b[np.logical_not(np.isnan(b))]),
-                'incorrect storage of floats')
+class TestNaNInsert:
+    def __init__(self):
+        self.rel = NanTest()
+        with dj.config(safemode=False):
+            self.rel.delete()
+        a = np.array([0, 1/3, np.nan, np.pi, np.nan])
+        self.rel.insert(((i, value) for i, value in enumerate(a)))
+        self.a = a
+
+
+    def test_insert_nan(self):
+        """Test fetching of null values"""
+        b = self.rel.fetch.order_by('id')['value']
+        assert_true((np.isnan(self.a) == np.isnan(b)).all(),
+                    'incorrect handling of Nans')
+        assert_true(np.allclose(self.a[np.logical_not(np.isnan(self.a))], b[np.logical_not(np.isnan(b))]),
+                    'incorrect storage of floats')
+
+    def test_nulls_do_not_affect_primary_keys(self):
+        """Test against a case that previously caused a bug when skipping existing entries."""
+        self.rel.insert(((i, value) for i, value in enumerate(self.a)), skip_duplicates=True)

tests/test_relational_operand.py

@@ -174,7 +174,8 @@ def test_restrictions_by_lists():
         assert_true(len(w - y) == 0,
                     'incorrect restriction without common attributes')
 
-    def test_datetime(self):
+    @staticmethod
+    def test_datetime():
         """Test date retrieval"""
         date = Experiment().fetch['experiment_date'][0]