From 0648bbc94d2058b7e2b563530f09593d621c254a Mon Sep 17 00:00:00 2001 From: Fabian Sinz Date: Mon, 7 Dec 2015 10:55:49 -0600 Subject: [PATCH 01/16] add new substance --- datajoint/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datajoint/connection.py b/datajoint/connection.py index d1bebdafa..bc116c8ce 100644 --- a/datajoint/connection.py +++ b/datajoint/connection.py @@ -59,7 +59,7 @@ def __init__(self, host, user, passwd, init_fun=None): port = int(port) else: port = config['database.port'] - self.conn_info = dict(host=host, port=port, user=user, passwd=passwd) + self.conn_info = dict(host=host, port=port, user=user, passwd=passwd, max_allowed_packet=1024**3) # 1073741824 self._conn = client.connect(init_command=init_fun, **self.conn_info) if self.is_connected: logger.info("Connected {user}@{host}:{port}".format(**self.conn_info)) From b46cd051480438d6c0552f6d089e49b30ed0a5d3 Mon Sep 17 00:00:00 2001 From: Fabian Sinz Date: Tue, 29 Dec 2015 10:08:07 -0600 Subject: [PATCH 02/16] transaction context manager checks for transaction --- datajoint/connection.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/datajoint/connection.py b/datajoint/connection.py index bc116c8ce..eb414a5ac 100644 --- a/datajoint/connection.py +++ b/datajoint/connection.py @@ -161,19 +161,21 @@ def commit_transaction(self): def transaction(self): """ Context manager for transactions. Opens an transaction and closes it after the with statement. - If an error is caught during the transaction, the commits are automatically rolled back. All - errors are raised again. + Only starts a transaction if there is not one going on already (MySQL does not support nested + transactions). If an error is caught during the transaction, the commits are automatically + rolled back. All errors are raised again. Example: >>> import datajoint as dj >>> with dj.conn().transaction as conn: >>> # transaction is open here """ - try: - self.start_transaction() - yield self - except: - self.cancel_transaction() - raise - else: - self.commit_transaction() + if not self.in_transaction: + try: + self.start_transaction() + yield self + except: + self.cancel_transaction() + raise + else: + self.commit_transaction() From 630694881f90284fb57cc6f67afae9037a7359e6 Mon Sep 17 00:00:00 2001 From: Fabian Sinz Date: Tue, 29 Dec 2015 10:16:40 -0600 Subject: [PATCH 03/16] small bugfix in transaction context manager --- datajoint/connection.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/datajoint/connection.py b/datajoint/connection.py index eb414a5ac..c876859ca 100644 --- a/datajoint/connection.py +++ b/datajoint/connection.py @@ -179,3 +179,5 @@ def transaction(self): raise else: self.commit_transaction() + else: + yield self From a92924b820d6f1f942e0cc35578a169f0351107f Mon Sep 17 00:00:00 2001 From: Fabian Sinz Date: Tue, 29 Dec 2015 10:24:49 -0600 Subject: [PATCH 04/16] transaction context manager now throws a waring if in transaction already --- datajoint/connection.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/datajoint/connection.py b/datajoint/connection.py index c876859ca..827de8acd 100644 --- a/datajoint/connection.py +++ b/datajoint/connection.py @@ -2,7 +2,7 @@ This module hosts the Connection class that manages the connection to the mysql database, and the `conn` function that provides access to a persistent connection in datajoint. """ - +import warnings from contextlib import contextmanager import pymysql as client import logging @@ -180,4 +180,6 @@ def transaction(self): else: self.commit_transaction() else: + warnings.warn("""Connection is in a transaction already. MySQL does not support nested transaction. This + transaction call will be ignored. """) yield self From 680a7ed988327571aceb40579e81181a8e27e95a Mon Sep 17 00:00:00 2001 From: Fabian Sinz Date: Wed, 30 Dec 2015 15:27:09 -0600 Subject: [PATCH 05/16] gitlog raises warning when populating with uncommited changes --- datajoint/connection.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/datajoint/connection.py b/datajoint/connection.py index 827de8acd..224c91a91 100644 --- a/datajoint/connection.py +++ b/datajoint/connection.py @@ -170,16 +170,11 @@ def transaction(self): >>> with dj.conn().transaction as conn: >>> # transaction is open here """ - if not self.in_transaction: - try: - self.start_transaction() - yield self - except: - self.cancel_transaction() - raise - else: - self.commit_transaction() - else: - warnings.warn("""Connection is in a transaction already. MySQL does not support nested transaction. This - transaction call will be ignored. """) + try: + self.start_transaction() yield self + except: + self.cancel_transaction() + raise + else: + self.commit_transaction() From 5a9b09e0679bffd34e37f9e712cdbc3128ae9da4 Mon Sep 17 00:00:00 2001 From: Fabian Sinz Date: Thu, 7 Jan 2016 15:26:34 -0600 Subject: [PATCH 06/16] bugfix in insert1 with skip and nan attributes --- datajoint/base_relation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datajoint/base_relation.py b/datajoint/base_relation.py index d9100a096..77598ce34 100644 --- a/datajoint/base_relation.py +++ b/datajoint/base_relation.py @@ -250,7 +250,8 @@ def make_attribute(name, value): raise DataJointError('Empty tuple') skip = skip_duplicates if skip: - primary_key_value = {name: value for name, _, value in attributes if heading[name].in_key} + primary_key_value = {name: value for name, _, value in attributes if + name is not None and heading[name].in_key} # if primary key value is empty, auto_populate is probably used skip = primary_key_value and (self & primary_key_value) if not skip: From 464dcbc2a6c22d91e02c86dab0d68d7f65390718 Mon Sep 17 00:00:00 2001 From: Fabian Sinz Date: Thu, 7 Jan 2016 15:28:01 -0600 Subject: [PATCH 07/16] set max_allowed_packet in connection --- datajoint/connection.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/datajoint/connection.py b/datajoint/connection.py index 224c91a91..2cc9d599d 100644 --- a/datajoint/connection.py +++ b/datajoint/connection.py @@ -59,7 +59,8 @@ def __init__(self, host, user, passwd, init_fun=None): port = int(port) else: port = config['database.port'] - self.conn_info = dict(host=host, port=port, user=user, passwd=passwd, max_allowed_packet=1024**3) # 1073741824 + self.conn_info = dict(host=host, port=port, user=user, passwd=passwd, + max_allowed_packet=1024 ** 3) # 1073741824 self._conn = client.connect(init_command=init_fun, **self.conn_info) if self.is_connected: logger.info("Connected {user}@{host}:{port}".format(**self.conn_info)) @@ -111,7 +112,6 @@ def query(self, query, args=(), as_dict=False): # Log the query logger.debug("Executing SQL:" + query[0:300]) - cur.execute(query, args) return cur @@ -161,9 +161,8 @@ def commit_transaction(self): def transaction(self): """ Context manager for transactions. Opens an transaction and closes it after the with statement. - Only starts a transaction if there is not one going on already (MySQL does not support nested - transactions). If an error is caught during the transaction, the commits are automatically - rolled back. All errors are raised again. + If an error is caught during the transaction, the commits are automatically rolled back. + All errors are raised again. Example: >>> import datajoint as dj From 3c21c9ee847774527209257a4dd3d5bff43d03b4 Mon Sep 17 00:00:00 2001 From: Fabian Sinz Date: Thu, 7 Jan 2016 16:35:57 -0600 Subject: [PATCH 08/16] remove warnings import in connection --- datajoint/connection.py | 1 - 1 file changed, 1 deletion(-) diff --git a/datajoint/connection.py b/datajoint/connection.py index 2cc9d599d..304d41d43 100644 --- a/datajoint/connection.py +++ b/datajoint/connection.py @@ -2,7 +2,6 @@ This module hosts the Connection class that manages the connection to the mysql database, and the `conn` function that provides access to a persistent connection in datajoint. """ -import warnings from contextlib import contextmanager import pymysql as client import logging From 6628efcc6e78490740cff5a30a087112273c56dc Mon Sep 17 00:00:00 2001 From: Fabian Sinz Date: Thu, 7 Jan 2016 17:31:44 -0600 Subject: [PATCH 09/16] be more verbose on changes --- datajoint/base_relation.py | 2 +- datajoint/connection.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/datajoint/base_relation.py b/datajoint/base_relation.py index 77598ce34..8e9260a26 100644 --- a/datajoint/base_relation.py +++ b/datajoint/base_relation.py @@ -261,7 +261,7 @@ def make_attribute(name, value): sql = 'INSERT IGNORE' else: sql = 'INSERT' - attributes = (a for a in attributes if a[0]) # omit dropped attributes + attributes = (a for a in attributes if a[0] is not None) # omit dropped attributes names, placeholders, values = tuple(zip(*attributes)) sql += " INTO %s (`%s`) VALUES (%s)" % ( self.from_clause, '`,`'.join(names), ','.join(placeholders)) diff --git a/datajoint/connection.py b/datajoint/connection.py index 304d41d43..7772eb4a4 100644 --- a/datajoint/connection.py +++ b/datajoint/connection.py @@ -59,7 +59,7 @@ def __init__(self, host, user, passwd, init_fun=None): else: port = config['database.port'] self.conn_info = dict(host=host, port=port, user=user, passwd=passwd, - max_allowed_packet=1024 ** 3) # 1073741824 + max_allowed_packet=1024 ** 3) # this is a hack to fix problems with pymysql (remove later) self._conn = client.connect(init_command=init_fun, **self.conn_info) if self.is_connected: logger.info("Connected {user}@{host}:{port}".format(**self.conn_info)) From afe3c8f7782ffc8b051a84b1ca83f4c35848ec6d Mon Sep 17 00:00:00 2001 From: Fabian Sinz Date: Tue, 12 Jan 2016 12:27:35 -0600 Subject: [PATCH 10/16] add test for null and skip insert bug --- tests/test_nan.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/tests/test_nan.py b/tests/test_nan.py index 42754166c..99daa5cbd 100644 --- a/tests/test_nan.py +++ b/tests/test_nan.py @@ -15,12 +15,24 @@ class NanTest(dj.Manual): """ -def test_insert_nan(): - rel = NanTest() - a = np.array([0, 1/3, np.nan, np.pi, np.nan]) - rel.insert(((i, value) for i, value in enumerate(a))) - b = rel.fetch.order_by('id')['value'] - assert_true((np.isnan(a) == np.isnan(b)).all(), - 'incorrect handling of Nans') - assert_true(np.allclose(a[np.logical_not(np.isnan(a))], b[np.logical_not(np.isnan(b))]), - 'incorrect storage of floats') +class TestNaNInsert: + def __init__(self): + self.rel = NanTest() + with dj.config(safemode=False): + self.rel.delete() + a = np.array([0, 1/3, np.nan, np.pi, np.nan]) + self.rel.insert(((i, value) for i, value in enumerate(a))) + self.a = a + + + def test_insert_nan(self): + """Test fetching of null values""" + b = self.rel.fetch.order_by('id')['value'] + assert_true((np.isnan(self.a) == np.isnan(b)).all(), + 'incorrect handling of Nans') + assert_true(np.allclose(self.a[np.logical_not(np.isnan(self.a))], b[np.logical_not(np.isnan(b))]), + 'incorrect storage of floats') + + def test_nulls_do_not_affect_primary_keys(self): + """Test against a case that previously caused a bug when skipping existing entries.""" + self.rel.insert(((i, value) for i, value in enumerate(self.a)), skip_duplicates=True) From 3b48c84a9f5ece698e547d905ab6c95d13b78783 Mon Sep 17 00:00:00 2001 From: Fabian Sinz Date: Tue, 12 Jan 2016 12:27:53 -0600 Subject: [PATCH 11/16] make test_datetime a static method --- tests/test_relational_operand.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_relational_operand.py b/tests/test_relational_operand.py index cc54255b5..2eee7be7d 100644 --- a/tests/test_relational_operand.py +++ b/tests/test_relational_operand.py @@ -174,7 +174,8 @@ def test_restrictions_by_lists(): assert_true(len(w - y) == 0, 'incorrect restriction without common attributes') - def test_datetime(self): + @staticmethod + def test_datetime(): """Test date retrieval""" date = Experiment().fetch['experiment_date'][0] From 8ed0dbdff7c62c48d62fef276b604b6e46b9e881 Mon Sep 17 00:00:00 2001 From: Fabian Sinz Date: Sun, 17 Jan 2016 21:47:54 -0600 Subject: [PATCH 12/16] add Dockerfile --- Dockerfile | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..3ad0ba1ed --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +FROM ubuntu:14.04 + +MAINTAINER Fabian Sinz + + +RUN \ + apt-get update && \ + apt-get install -y -q \ + git + +RUN \ + apt-get update && \ + apt-get install -y -q \ + python3 \ + python3-pip \ + python3-numpy \ + python3-scipy \ + python3-matplotlib \ + python3-pandas \ + ipython3 \ + ipython3-notebook + +RUN pip3 install git+https://github.com/datajoint/datajoint-python.git + +RUN pip3 install git+https://github.com/datajoint/datajoint-addons.git + From b37ee5722e6a428c1d5420296a3c81c6a10370a5 Mon Sep 17 00:00:00 2001 From: Fabian Sinz Date: Sun, 17 Jan 2016 21:57:09 -0600 Subject: [PATCH 13/16] add mysql to dockerfile --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3ad0ba1ed..e5cb972b4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,9 @@ RUN \ python3-matplotlib \ python3-pandas \ ipython3 \ - ipython3-notebook + ipython3-notebook \ + mysql-server \ + mysql-client RUN pip3 install git+https://github.com/datajoint/datajoint-python.git From e52ef6c55e542b38b08c4646fa2574ad4d02c6d7 Mon Sep 17 00:00:00 2001 From: Fabian Sinz Date: Thu, 21 Jan 2016 10:53:22 -0600 Subject: [PATCH 14/16] make loads of DEFAULT_MODEL python3 compatible --- datajoint/__init__.py | 31 +++++++++++++++++++++++-------- datajoint/settings.py | 2 +- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/datajoint/__init__.py b/datajoint/__init__.py index f9ad15b43..55a801a78 100644 --- a/datajoint/__init__.py +++ b/datajoint/__init__.py @@ -33,18 +33,33 @@ class DataJointError(Exception): pass # ----------- loads local configuration from file ---------------- -from .settings import Config, CONFIGVAR, LOCALCONFIG, logger, log_levels +from .settings import Config, LOCALCONFIG, GLOBALCONFIG, logger, log_levels config = Config() -local_config_file = os.environ.get(CONFIGVAR, None) -if local_config_file is None: - local_config_file = LOCALCONFIG -local_config_file = os.path.expanduser(local_config_file) -try: + +if os.path.exists(LOCALCONFIG): + local_config_file = os.path.expanduser(LOCALCONFIG) + print("Loading local settings from {0:s}".format(local_config_file)) + logger.log(logging.INFO, "Loading local settings from {0:s}".format(local_config_file)) + config.load(local_config_file) +elif os.path.exists(GLOBALCONFIG): + local_config_file = os.path.expanduser('~/') + GLOBALCONFIG + print("Loading local settings from {0:s}".format(local_config_file)) logger.log(logging.INFO, "Loading local settings from {0:s}".format(local_config_file)) config.load(local_config_file) -except FileNotFoundError: - logger.warn("Local config file {0:s} does not exist! Creating it.".format(local_config_file)) +elif os.getenv('DJ_HOST') is not None and os.getenv('DJ_USER') is not None and os.getenv('DJ_PASS') is not None: + print("Loading local settings from environment variables") + config['database.host'] = os.getenv('DJ_HOST') + config['database.user'] = os.getenv('DJ_USER') + config['database.password'] = os.getenv('DJ_PASS') +else: + print("""Cannot find configuration settings. Using default configuration. To change that, either + * modify the local copy of %s that datajoint just saved for you + * put a file named %s with the same configuration format in your home + * specify the environment variables DJ_USER, DJ_HOST, DJ_PASS + """) + local_config_file = os.path.expanduser(LOCALCONFIG) + logger.log(logging.INFO, "No config found. Generating {0:s}".format(local_config_file)) config.save(local_config_file) logger.setLevel(log_levels[config['loglevel']]) diff --git a/datajoint/settings.py b/datajoint/settings.py index 0062adde2..8e239dd2b 100644 --- a/datajoint/settings.py +++ b/datajoint/settings.py @@ -12,7 +12,7 @@ from enum import Enum LOCALCONFIG = 'dj_local_conf.json' -CONFIGVAR = 'DJ_LOCAL_CONF' +GLOBALCONFIG = '._datajoint_config.json' validators = collections.defaultdict(lambda: lambda value: True) validators['database.port'] = lambda a: isinstance(a, int) From 1e993a6a39e1199356a360dfab3b698b18c1c80f Mon Sep 17 00:00:00 2001 From: Fabian Sinz Date: Thu, 21 Jan 2016 11:01:24 -0600 Subject: [PATCH 15/16] modify order of config loading (ENV, local, global) --- datajoint/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/datajoint/__init__.py b/datajoint/__init__.py index 55a801a78..2d71ab568 100644 --- a/datajoint/__init__.py +++ b/datajoint/__init__.py @@ -37,7 +37,12 @@ class DataJointError(Exception): config = Config() -if os.path.exists(LOCALCONFIG): +if os.getenv('DJ_HOST') is not None and os.getenv('DJ_USER') is not None and os.getenv('DJ_PASS') is not None: + print("Loading local settings from environment variables") + config['database.host'] = os.getenv('DJ_HOST') + config['database.user'] = os.getenv('DJ_USER') + config['database.password'] = os.getenv('DJ_PASS') +elif os.path.exists(LOCALCONFIG): local_config_file = os.path.expanduser(LOCALCONFIG) print("Loading local settings from {0:s}".format(local_config_file)) logger.log(logging.INFO, "Loading local settings from {0:s}".format(local_config_file)) @@ -47,11 +52,6 @@ class DataJointError(Exception): print("Loading local settings from {0:s}".format(local_config_file)) logger.log(logging.INFO, "Loading local settings from {0:s}".format(local_config_file)) config.load(local_config_file) -elif os.getenv('DJ_HOST') is not None and os.getenv('DJ_USER') is not None and os.getenv('DJ_PASS') is not None: - print("Loading local settings from environment variables") - config['database.host'] = os.getenv('DJ_HOST') - config['database.user'] = os.getenv('DJ_USER') - config['database.password'] = os.getenv('DJ_PASS') else: print("""Cannot find configuration settings. Using default configuration. To change that, either * modify the local copy of %s that datajoint just saved for you From 1efb50c52d888da1a5cf48fc6693fe5d07176529 Mon Sep 17 00:00:00 2001 From: Fabian Sinz Date: Thu, 21 Jan 2016 19:04:20 -0600 Subject: [PATCH 16/16] bugfix in settings loading --- datajoint/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datajoint/__init__.py b/datajoint/__init__.py index 2d71ab568..7cac45a0e 100644 --- a/datajoint/__init__.py +++ b/datajoint/__init__.py @@ -47,7 +47,7 @@ class DataJointError(Exception): print("Loading local settings from {0:s}".format(local_config_file)) logger.log(logging.INFO, "Loading local settings from {0:s}".format(local_config_file)) config.load(local_config_file) -elif os.path.exists(GLOBALCONFIG): +elif os.path.exists(os.path.expanduser('~/') + GLOBALCONFIG): local_config_file = os.path.expanduser('~/') + GLOBALCONFIG print("Loading local settings from {0:s}".format(local_config_file)) logger.log(logging.INFO, "Loading local settings from {0:s}".format(local_config_file))