diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..e5cb972b4 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,28 @@ +FROM ubuntu:14.04 + +MAINTAINER Fabian Sinz + + +RUN \ + apt-get update && \ + apt-get install -y -q \ + git + +RUN \ + apt-get update && \ + apt-get install -y -q \ + python3 \ + python3-pip \ + python3-numpy \ + python3-scipy \ + python3-matplotlib \ + python3-pandas \ + ipython3 \ + ipython3-notebook \ + mysql-server \ + mysql-client + +RUN pip3 install git+https://github.com/datajoint/datajoint-python.git + +RUN pip3 install git+https://github.com/datajoint/datajoint-addons.git + diff --git a/datajoint/__init__.py b/datajoint/__init__.py index f9ad15b43..7cac45a0e 100644 --- a/datajoint/__init__.py +++ b/datajoint/__init__.py @@ -33,18 +33,33 @@ class DataJointError(Exception): pass # ----------- loads local configuration from file ---------------- -from .settings import Config, CONFIGVAR, LOCALCONFIG, logger, log_levels +from .settings import Config, LOCALCONFIG, GLOBALCONFIG, logger, log_levels config = Config() -local_config_file = os.environ.get(CONFIGVAR, None) -if local_config_file is None: - local_config_file = LOCALCONFIG -local_config_file = os.path.expanduser(local_config_file) -try: + +if os.getenv('DJ_HOST') is not None and os.getenv('DJ_USER') is not None and os.getenv('DJ_PASS') is not None: + print("Loading local settings from environment variables") + config['database.host'] = os.getenv('DJ_HOST') + config['database.user'] = os.getenv('DJ_USER') + config['database.password'] = os.getenv('DJ_PASS') +elif os.path.exists(LOCALCONFIG): + local_config_file = os.path.expanduser(LOCALCONFIG) + print("Loading local settings from {0:s}".format(local_config_file)) + logger.log(logging.INFO, "Loading local settings from {0:s}".format(local_config_file)) + config.load(local_config_file) +elif os.path.exists(os.path.expanduser('~/') + GLOBALCONFIG): + local_config_file = os.path.expanduser('~/') + GLOBALCONFIG + print("Loading local settings from {0:s}".format(local_config_file)) logger.log(logging.INFO, "Loading local settings from {0:s}".format(local_config_file)) config.load(local_config_file) -except FileNotFoundError: - logger.warn("Local config file {0:s} does not exist! Creating it.".format(local_config_file)) +else: + print("""Cannot find configuration settings. Using default configuration. To change that, either + * modify the local copy of %s that datajoint just saved for you + * put a file named %s with the same configuration format in your home + * specify the environment variables DJ_USER, DJ_HOST, DJ_PASS + """) + local_config_file = os.path.expanduser(LOCALCONFIG) + logger.log(logging.INFO, "No config found. Generating {0:s}".format(local_config_file)) config.save(local_config_file) logger.setLevel(log_levels[config['loglevel']]) diff --git a/datajoint/base_relation.py b/datajoint/base_relation.py index d9100a096..8e9260a26 100644 --- a/datajoint/base_relation.py +++ b/datajoint/base_relation.py @@ -250,7 +250,8 @@ def make_attribute(name, value): raise DataJointError('Empty tuple') skip = skip_duplicates if skip: - primary_key_value = {name: value for name, _, value in attributes if heading[name].in_key} + primary_key_value = {name: value for name, _, value in attributes if + name is not None and heading[name].in_key} # if primary key value is empty, auto_populate is probably used skip = primary_key_value and (self & primary_key_value) if not skip: @@ -260,7 +261,7 @@ def make_attribute(name, value): sql = 'INSERT IGNORE' else: sql = 'INSERT' - attributes = (a for a in attributes if a[0]) # omit dropped attributes + attributes = (a for a in attributes if a[0] is not None) # omit dropped attributes names, placeholders, values = tuple(zip(*attributes)) sql += " INTO %s (`%s`) VALUES (%s)" % ( self.from_clause, '`,`'.join(names), ','.join(placeholders)) diff --git a/datajoint/connection.py b/datajoint/connection.py index d1bebdafa..7772eb4a4 100644 --- a/datajoint/connection.py +++ b/datajoint/connection.py @@ -2,7 +2,6 @@ This module hosts the Connection class that manages the connection to the mysql database, and the `conn` function that provides access to a persistent connection in datajoint. """ - from contextlib import contextmanager import pymysql as client import logging @@ -59,7 +58,8 @@ def __init__(self, host, user, passwd, init_fun=None): port = int(port) else: port = config['database.port'] - self.conn_info = dict(host=host, port=port, user=user, passwd=passwd) + self.conn_info = dict(host=host, port=port, user=user, passwd=passwd, + max_allowed_packet=1024 ** 3) # this is a hack to fix problems with pymysql (remove later) self._conn = client.connect(init_command=init_fun, **self.conn_info) if self.is_connected: logger.info("Connected {user}@{host}:{port}".format(**self.conn_info)) @@ -111,7 +111,6 @@ def query(self, query, args=(), as_dict=False): # Log the query logger.debug("Executing SQL:" + query[0:300]) - cur.execute(query, args) return cur @@ -161,8 +160,8 @@ def commit_transaction(self): def transaction(self): """ Context manager for transactions. Opens an transaction and closes it after the with statement. - If an error is caught during the transaction, the commits are automatically rolled back. All - errors are raised again. + If an error is caught during the transaction, the commits are automatically rolled back. + All errors are raised again. Example: >>> import datajoint as dj diff --git a/datajoint/settings.py b/datajoint/settings.py index 0062adde2..8e239dd2b 100644 --- a/datajoint/settings.py +++ b/datajoint/settings.py @@ -12,7 +12,7 @@ from enum import Enum LOCALCONFIG = 'dj_local_conf.json' -CONFIGVAR = 'DJ_LOCAL_CONF' +GLOBALCONFIG = '._datajoint_config.json' validators = collections.defaultdict(lambda: lambda value: True) validators['database.port'] = lambda a: isinstance(a, int) diff --git a/tests/test_nan.py b/tests/test_nan.py index 42754166c..99daa5cbd 100644 --- a/tests/test_nan.py +++ b/tests/test_nan.py @@ -15,12 +15,24 @@ class NanTest(dj.Manual): """ -def test_insert_nan(): - rel = NanTest() - a = np.array([0, 1/3, np.nan, np.pi, np.nan]) - rel.insert(((i, value) for i, value in enumerate(a))) - b = rel.fetch.order_by('id')['value'] - assert_true((np.isnan(a) == np.isnan(b)).all(), - 'incorrect handling of Nans') - assert_true(np.allclose(a[np.logical_not(np.isnan(a))], b[np.logical_not(np.isnan(b))]), - 'incorrect storage of floats') +class TestNaNInsert: + def __init__(self): + self.rel = NanTest() + with dj.config(safemode=False): + self.rel.delete() + a = np.array([0, 1/3, np.nan, np.pi, np.nan]) + self.rel.insert(((i, value) for i, value in enumerate(a))) + self.a = a + + + def test_insert_nan(self): + """Test fetching of null values""" + b = self.rel.fetch.order_by('id')['value'] + assert_true((np.isnan(self.a) == np.isnan(b)).all(), + 'incorrect handling of Nans') + assert_true(np.allclose(self.a[np.logical_not(np.isnan(self.a))], b[np.logical_not(np.isnan(b))]), + 'incorrect storage of floats') + + def test_nulls_do_not_affect_primary_keys(self): + """Test against a case that previously caused a bug when skipping existing entries.""" + self.rel.insert(((i, value) for i, value in enumerate(self.a)), skip_duplicates=True) diff --git a/tests/test_relational_operand.py b/tests/test_relational_operand.py index cc54255b5..2eee7be7d 100644 --- a/tests/test_relational_operand.py +++ b/tests/test_relational_operand.py @@ -174,7 +174,8 @@ def test_restrictions_by_lists(): assert_true(len(w - y) == 0, 'incorrect restriction without common attributes') - def test_datetime(self): + @staticmethod + def test_datetime(): """Test date retrieval""" date = Experiment().fetch['experiment_date'][0]