Minor fixes to IBL registration client

ibllib/__init__.py

@@ -2,7 +2,7 @@
 import logging
 import warnings
 
-__version__ = '2.32.4'
+__version__ = '2.32.5'
 warnings.filterwarnings('always', category=DeprecationWarning, module='ibllib')
 
 # if this becomes a full-blown library we should let the logging configuration to the discretion of the dev

ibllib/io/session_params.py

@@ -169,9 +169,9 @@ def merge_params(a, b, copy=False):
         if k == 'sync':
             assert k not in a or a[k] == b[k], 'multiple sync fields defined'
         if isinstance(b[k], list):
-            prev = a.get(k, [])
+            prev = list(a.get(k, []))
             # For procedures and projects, remove duplicates
-            to_add = b[k] if k == 'tasks' else set(prev) ^ set(b[k])
+            to_add = b[k] if k == 'tasks' else set(b[k]) - set(prev)
             a[k] = prev + list(to_add)
         elif isinstance(b[k], dict):
             a[k] = {**a.get(k, {}), **b[k]}

ibllib/oneibl/registration.py

@@ -221,7 +221,7 @@ def register_session(self, ses_path, file_list=True, projects=None, procedures=N
             # assert len({md['IBLRIG_VERSION_TAG'] for md in settings}) == 1
 
             users = []
-            for user in filter(None, map(lambda x: x.get('PYBPOD_CREATOR'), settings)):
+            for user in filter(lambda x: x and x[1], map(lambda x: x.get('PYBPOD_CREATOR'), settings)):
                 user = self.assert_exists(user[0], 'users')  # user is list of [username, uuid]
                 users.append(user['username'])
 
@@ -246,7 +246,7 @@ def register_session(self, ses_path, file_list=True, projects=None, procedures=N
             if poo_counts:
                 json_field['POOP_COUNT'] = int(sum(poo_counts))
 
-        if not session:  # Create session and weighings
+        if not len(session):  # Create session and weighings
             ses_ = {'subject': subject['nickname'],
                     'users': users or [subject['responsible_user']],
                     'location': settings[0]['PYBPOD_BOARD'],
@@ -273,9 +273,13 @@ def register_session(self, ses_path, file_list=True, projects=None, procedures=N
                 self.register_weight(subject['nickname'], md['SUBJECT_WEIGHT'],
                                      date_time=md['SESSION_DATETIME'], user=user)
         else:  # if session exists update a few key fields
-            data = {'procedures': procedures, 'projects': projects}
+            data = {'procedures': procedures, 'projects': projects,
+                    'n_correct_trials': n_correct_trials, 'n_trials': n_trials}
             if task_protocols:
                 data['task_protocol'] = '/'.join(task_protocols)
+            if end_time:
+                data['end_time'] = self.ensure_ISO8601(end_time)
+
             session = self.one.alyx.rest('sessions', 'partial_update', id=session_id[0], data=data)
             if json_field:
                 session['json'] = self.one.alyx.json_field_update('sessions', session['id'], data=json_field)
@@ -401,14 +405,16 @@ def _get_session_times(fn, md, ses_data):
     """
     if isinstance(md, dict):
         start_time = _start_time = isostr2date(md['SESSION_DATETIME'])
+        end_time = isostr2date(md['SESSION_END_TIME']) if md.get('SESSION_END_TIME') else None
     else:
         start_time = isostr2date(md[0]['SESSION_DATETIME'])
         _start_time = isostr2date(md[-1]['SESSION_DATETIME'])
+        end_time = isostr2date(md[-1]['SESSION_END_TIME']) if md[-1].get('SESSION_END_TIME') else None
         assert isinstance(ses_data, (list, tuple)) and len(ses_data) == len(md)
         assert len(md) == 1 or start_time < _start_time
         ses_data = ses_data[-1]
-    if not ses_data:
-        return start_time, None
+    if not ses_data or end_time is not None:
+        return start_time, end_time
     c = ses_duration_secs = 0
     for sd in reversed(ses_data):
         ses_duration_secs = (sd['behavior_data']['Trial end timestamp'] -

ibllib/tests/test_io.py

@@ -555,6 +555,23 @@ def test_get_collections_repeat_protocols(self):
         collections = session_params.get_collections(tasks, flat=True)
         self.assertEqual(collections, {'raw_passive_data_bis', 'raw_passive_data', 'raw_behavior_data'})
 
+    def test_merge_params(self):
+        """Test for ibllib.io.session_params.merge_params functions."""
+        a = self.fixture
+        b = {'procedures': ['Imaging', 'Injection'], 'tasks': [{'fooChoiceWorld': {'collection': 'bar'}}]}
+        c = session_params.merge_params(a, b, copy=True)
+        self.assertCountEqual(['Imaging', 'Behavior training/tasks', 'Injection'], c['procedures'])
+        self.assertCountEqual(['passiveChoiceWorld', 'ephysChoiceWorld', 'fooChoiceWorld'], (list(x)[0] for x in c['tasks']))
+        # Ensure a and b not modified
+        self.assertNotEqual(set(c['procedures']), set(a['procedures']))
+        self.assertNotEqual(set(a['procedures']), set(b['procedures']))
+        # Test without copy
+        session_params.merge_params(a, b, copy=False)
+        self.assertCountEqual(['Imaging', 'Behavior training/tasks', 'Injection'], a['procedures'])
+        # Test assertion on duplicate sync
+        b['sync'] = {'foodaq': {'collection': 'raw_sync_data'}}
+        self.assertRaises(AssertionError, session_params.merge_params, a, b)
+
 
 class TestRawDaqLoaders(unittest.TestCase):
     """Tests for raw_daq_loaders module"""

ibllib/tests/test_oneibl.py

@@ -429,16 +429,20 @@ def test_registration_session(self):
                               query_type='remote')[0]
         ses_info = self.one.alyx.rest('sessions', 'read', id=eid)
         self.assertTrue(ses_info['procedures'] == ['Behavior training/tasks'])
-        self.one.alyx.rest('sessions', 'delete', id=eid)
-        # re-register the session as unknown protocol this time
+        # re-register the session as unknown protocol, this time without removing session first
         self.settings['PYBPOD_PROTOCOL'] = 'gnagnagna'
+        # also add an end time
+        start = datetime.datetime.fromisoformat(self.settings['SESSION_DATETIME'])
+        self.settings['SESSION_START_TIME'] = rc.ensure_ISO8601(start)
+        self.settings['SESSION_END_TIME'] = rc.ensure_ISO8601(start + datetime.timedelta(hours=1))
         with open(settings_file, 'w') as fid:
             json.dump(self.settings, fid)
         rc.register_session(self.session_path)
         eid = self.one.search(subject=self.subject, date_range=['2018-04-01', '2018-04-01'],
                               query_type='remote')[0]
         ses_info = self.one.alyx.rest('sessions', 'read', id=eid)
         self.assertTrue(ses_info['procedures'] == [])
+        self.assertEqual(self.settings['SESSION_END_TIME'], ses_info['end_time'])
         self.one.alyx.rest('sessions', 'delete', id=eid)
 
     def test_register_chained_session(self):
@@ -457,12 +461,13 @@ def test_register_chained_session(self):
 
         # Save experiment description
         session_params.write_params(self.session_path, experiment_description)
-
+        self.settings['POOP_COUNT'] = 10
         with open(behaviour_paths[1].joinpath('_iblrig_taskSettings.raw.json'), 'w') as fid:
             json.dump(self.settings, fid)
 
         settings = self.settings.copy()
         settings['PYBPOD_PROTOCOL'] = '_iblrig_tasks_passiveChoiceWorld'
+        settings['POOP_COUNT'] = 53
         start_time = (datetime.datetime.fromisoformat(settings['SESSION_DATETIME']) -
                       datetime.timedelta(hours=1, minutes=2, seconds=12))
         settings['SESSION_DATETIME'] = start_time.isoformat()
@@ -475,7 +480,9 @@ def test_register_chained_session(self):
         ses_info = self.one.alyx.rest('sessions', 'read', id=session['id'])
         self.assertCountEqual(experiment_description['procedures'], ses_info['procedures'])
         self.assertCountEqual(experiment_description['projects'], ses_info['projects'])
-        self.assertCountEqual({'IS_MOCK': False, 'IBLRIG_VERSION': None}, ses_info['json'])
+        # Poo count should be sum of values in both settings files
+        expected = {'IS_MOCK': False, 'IBLRIG_VERSION': '5.4.1', 'POOP_COUNT': 63}
+        self.assertDictEqual(expected, ses_info['json'])
         self.assertEqual('2018-04-01T11:46:14.795526', ses_info['start_time'])
         # Test task protocol
         expected = '_iblrig_tasks_passiveChoiceWorld5.4.1/_iblrig_tasks_ephysChoiceWorld5.4.1'

release_notes.md

@@ -13,6 +13,9 @@
 #### 2.32.4
 - Add support for variations of the biaseCW task in the json task description
 
+#### 2.32.5
+- Minor fixes to IBL registion client, including use of SESSION_END_TIME key
+
 ## Release Notes 2.31
 
 ### features