Merge pull request #756 from cortex-lab/dev

Release 1.5.1

alyx/data/fixtures/data.datasettype.json

@@ -703,17 +703,6 @@
       "filename_pattern": "spikes.depths*.npy"
     }
   },
-  {
-    "model": "data.datasettype",
-    "pk": "f010bc2d-0211-41e7-930f-08c927a78d82",
-    "fields": {
-      "json": null,
-      "name": "trials.itiDuration",
-      "created_by": null,
-      "description": "Intertrial interval duration for each trial",
-      "filename_pattern": "*trials.itiDuration.*"
-    }
-  },
   {
     "model": "data.datasettype",
     "pk": "f7dbfad8-1cfc-459e-874f-4065cf9def86",

alyx/data/tests_rest.py

@@ -365,6 +365,121 @@ def _assert_registration(self, r, data):
         self.assertEqual(d1['file_records'][0]['relative_path'],
                          op.join(data['path'], 'a.c.e2'))
 
+    def test_register_existence_options(self):
+
+        self.post(reverse('datarepository-list'), {'name': 'ibl1', 'hostname': 'iblhost1',
+                                                   'globus_is_personal': 'True'})
+        self.post(reverse('datarepository-list'), {'name': 'ibl2', 'hostname': 'iblhost2',
+                                                   'globus_is_personal': 'True'})
+        # server repo
+        self.post(reverse('datarepository-list'), {'name': 'ibl3', 'hostname': 'iblhost3',
+                                                   'globus_is_personal': 'False'})
+
+        self.post(reverse('lab-list'), {'name': 'ibl',
+                                        'repositories': ['ibl1', 'ibl2', 'ibl3']})
+
+        # Case 1, server_only = False, no repo name specified
+        # Expect: 3 file records (one for each repo),
+        # all with exists = False (preserves old behavior)
+        data = {'path': '%s/2018-01-01/002/dir' % self.subject,
+                'filenames': 'a.a.e1',  # this is the repository name
+                'server_only': False,
+                'labs': ['ibl']
+                }
+
+        r = self.client.post(reverse('register-file'), data)
+        r = self.ar(r, 201)[0]
+        frs = r['file_records']
+        self.assertTrue(len(frs) == 3)
+        for fr in frs:
+            self.assertEqual(fr['exists'], False)
+
+        # Case 2, server_only = True, no repo name specified
+        # Expect: 1 file record for data repo with globus is personal == True,
+        # with exists = True (preserves old behavior)
+
+        data = {'path': '%s/2018-01-01/002/dir' % self.subject,
+                'filenames': 'a.a.e2',  # this is the repository name
+                'server_only': True,
+                'labs': ['ibl']
+                }
+
+        r = self.client.post(reverse('register-file'), data)
+        r = self.ar(r, 201)[0]
+        frs = r['file_records']
+        self.assertTrue(len(frs) == 1)
+        self.assertEqual(frs[0]['exists'], True)
+        self.assertEqual(frs[0]['data_repository'], 'ibl3')
+
+        # Case 3, server_only = False, repo name specified
+        # Expect: 3 file records, specified repo with exists = True,
+        # the others False (preserves old behavior)
+        data = {'path': '%s/2018-01-01/002/dir' % self.subject,
+                'filenames': 'a.b.e1',  # this is the repository name
+                'name': 'ibl1',
+                'server_only': False,
+                'labs': ['ibl']
+                }
+
+        r = self.client.post(reverse('register-file'), data)
+        r = self.ar(r, 201)[0]
+        frs = r['file_records']
+        self.assertTrue(len(frs) == 3)
+        for fr in frs:
+            self.assertEqual(fr['exists'], fr['data_repository'] == 'ibl1')
+
+        # # Case 4, server_only = False, repo name specified, exists = False
+        # # Expect: 3 file records, all with exists = False
+        data = {'path': '%s/2018-01-01/002/dir' % self.subject,
+                'filenames': 'a.b.e2',  # this is the repository name
+                'name': 'ibl1',
+                'server_only': False,
+                'exists': False,
+                'labs': ['ibl']
+                }
+
+        r = self.client.post(reverse('register-file'), data)
+        r = self.ar(r, 201)[0]
+        frs = r['file_records']
+        self.assertTrue(len(frs) == 3)
+        for fr in frs:
+            self.assertEqual(fr['exists'], False)
+
+        # Case 5, server_only = True, repo name specified
+        # Expect: 2 file records, 1 for specified repo and 1 for server repo,
+        # all with exists = True (preserves old behavior)
+        data = {'path': '%s/2018-01-01/002/dir' % self.subject,
+                'filenames': 'a.c.e1',  # this is the repository name
+                'name': 'ibl1',
+                'server_only': True,
+                'labs': ['ibl']
+                }
+
+        r = self.client.post(reverse('register-file'), data)
+        r = self.ar(r, 201)[0]
+        frs = r['file_records']
+        self.assertTrue(len(frs) == 2)
+        for fr in frs:
+            self.assertEqual(fr['exists'], True)
+
+        # # Case 5, server_only = True, repo name specified, exists = False
+        # # Expect: 2 file records, 1 for specified repo and 1 for server repo,
+        # all with exists = False
+        data = {'path': '%s/2018-01-01/002/dir' % self.subject,
+                'filenames': 'a.c.e2',  # this is the repository name
+                'name': 'ibl1',
+                'server_only': True,
+                'exists': False,
+                'labs': ['ibl']
+                }
+
+        r = self.client.post(reverse('register-file'), data)
+        r = self.ar(r, 201)[0]
+        frs = r['file_records']
+        self.assertTrue(len(frs) == 2)
+        for fr in frs:
+            self.assertEqual(fr['exists'], False)
+
     def test_register_with_revision(self):
         self.post(reverse('datarepository-list'), {'name': 'drb2', 'hostname': 'hostb2'})
         self.post(reverse('lab-list'), {'name': 'labb', 'repositories': ['drb2']})

alyx/data/views.py

@@ -416,6 +416,8 @@ def create(self, request):
 
         # flag to discard file records creation on local repositories, defaults to False
         server_only = request.data.get('server_only', False)
+        if isinstance(server_only, str):
+            server_only = server_only == 'True'
 
         default = request.data.get('default', True)
         # Need to explicitly cast string to a bool
@@ -432,6 +434,14 @@ def create(self, request):
         if server_only:
             exists_in = repositories
 
+        # # If exists is specified to be false then we set the exists_in back to None
+        exists = request.data.get('exists', True)
+        # Need to explicitly cast string to a bool
+        if isinstance(exists, str):
+            exists = exists == 'True'
+        if not exists:
+            exists_in = (None,)
+
         session = _get_session(
             subject=subject, date=date, number=session_number, user=user)
         assert session

requirements_frozen.txt

@@ -1,12 +1,13 @@
 asgiref==3.5.2
 backports.zoneinfo==0.2.1
-boto3==1.24.71
-botocore==1.27.71
-certifi==2022.6.15.1
+boto3==1.24.81
+botocore==1.27.81
+certifi==2022.9.24
 cffi==1.15.1
 charset-normalizer==2.1.1
 click==8.1.3
 colorlog==6.7.0
+contourpy==1.0.5
 coreapi==2.3.3
 coreschema==0.0.4
 coverage==6.4.4
@@ -15,7 +16,7 @@ cryptography==36.0.2
 cycler==0.11.0
 Django==4.1.1
 django-admin-list-filter-dropdown==1.0.3
-django-admin-rangefilter==0.8.8
+django-admin-rangefilter==0.9.0
 django-autocomplete-light==3.9.4
 django-cleanup==6.0.0
 django-filter==21.1
@@ -27,16 +28,16 @@ django-reversion==5.0.2
 django-storages==1.13.1
 django-structlog==3.0.1
 django-test-without-migrations==0.6
-djangorestframework==3.13.1
+djangorestframework==3.14.0
 docopt==0.6.2
 docutils==0.19
 drfdocs==0.0.11
 flake8==5.0.4
-fonttools==4.37.1
+fonttools==4.37.3
 globus-cli==3.8.0
 globus-sdk==3.11.0
 iblutil==1.3.3
-idna==3.3
+idna==3.4
 importlib-metadata==4.12.0
 itypes==1.2.0
 Jinja2==3.1.2
@@ -46,30 +47,32 @@ llvmlite==0.39.1
 lxml==4.9.1
 Markdown==3.4.1
 MarkupSafe==2.1.1
-matplotlib==3.5.3
+matplotlib==3.6.0
 mccabe==0.7.0
 numba==0.56.2
 numpy==1.23.3
-ONE-api==1.14.0
+ONE-api==1.15.0
 packaging==21.3
-pandas==1.4.4
+pandas==1.5.0
 Pillow==9.2.0
 psycopg2-binary==2.9.3
 pyarrow==9.0.0
 pycodestyle==2.9.1
 pycparser==2.21
 pyflakes==2.5.0
-PyJWT==2.4.0
+PyJWT==2.5.0
 pyparsing==3.0.9
 python-dateutil==2.8.2
 python-magic==0.4.27
 pytz==2022.2.1
+PyYAML==6.0
 requests==2.28.1
 s3transfer==0.6.0
 six==1.16.0
-sqlparse==0.4.2
+sqlparse==0.4.3
 structlog==22.1.0
 tqdm==4.64.1
+types-cryptography==3.3.23
 typing_extensions==4.3.0
 uritemplate==4.1.1
 urllib3==1.26.12

scripts/sync_ucl/prune_cortexlab.py

@@ -234,7 +234,7 @@
 # finds eids that have tasks on both ibl and cortex lab database
 overlap_eids = set(cortex_eids).intersection(ibl_eids)
 
-dfields = ('id', 'name', 'session', 'arguments')
+dfields = ('id', 'name', 'session')
 task_cortex = Task.objects.using('cortexlab').filter(session__in=overlap_eids).exclude(name__in=task_names_to_exclude)
 cids = task_cortex.values_list(*dfields)