Merge pull request #716 from cortex-lab/dev

Dev

.gitignore

@@ -29,3 +29,5 @@ alyx/templates/ibl_reports
 alyx/alyx/settings_secret.py
 alyx/alyx/settings_lab.py
 alyx/alyx/settings.py
+
+alyx/.idea/

alyx/actions/views.py

@@ -18,6 +18,7 @@
 
 from alyx.base import base_json_filter, BaseFilterSet
 from subjects.models import Subject
+from experiments.views import _filter_qs_with_brain_regions
 from .water_control import water_control, to_date
 from .models import (
     BaseAction, Session, WaterAdministration, WaterRestriction,
@@ -208,10 +209,7 @@ def atlas(self, queryset, name, value):
         returns sessions containing at least one channel in the given brain region.
         Hierarchical tree search"
         """
-        from experiments.models import BrainRegion
-        brs = BrainRegion.objects.filter(**{name: value}).get_descendants(include_self=True)
-        return queryset.filter(
-            probe_insertion__trajectory_estimate__channels__brain_region__in=brs).distinct()
+        return _filter_qs_with_brain_regions(self, queryset, name, value)
 
     def has_histology(self, queryset, name, value):
         """returns sessions whose subjects have an histology session available"""
@@ -314,7 +312,7 @@ class SessionAPIList(generics.ListCreateAPIView):
         -   chained lookups: `/sessions/?extended_qc=qc_pct__gte;0.5;qc_bool;True`,
     -   **performance_gte**, **performance_lte**: percentage of successful trials gte/lte
     -   **brain_region**: returns a session if any channel name icontains the value:
-        `/sessions?brain_region=vis`
+        `/sessions?brain_region=visual cortex`
     -   **atlas_acronym**: returns a session if any of its channels name exactly matches the value
         `/sessions?atlas_acronym=SSp-m4`, cf Allen CCFv2017
     -   **atlas_id**: returns a session if any of its channels id matches the provided value:

alyx/alyx/settings.py → alyx/alyx/settings_ci.py

@@ -21,22 +21,24 @@
     from .settings_secret_template import *  # noqa
 
 # Lab-specific settings
-from .settings_lab import *  # noqa
+try:
+    from .settings_lab import *  # noqa
+except ImportError:
+    from .settings_lab_template import *  # noqa
 
 en_formats.DATETIME_FORMAT = "d/m/Y H:i"
 DATE_INPUT_FORMATS = ('%d/%m/%Y',)
 
 
-if 'TRAVIS' in os.environ:
+if 'GITHUB_ACTIONS' in os.environ:
     DATABASES = {
         'default': {
             'ENGINE': 'django.db.backends.postgresql_psycopg2',
-	    # 'ENGINE': 'django_prometheus.db.backends.postgresql',
-            'NAME': 'travisci',
+            'NAME': 'githubactions',
             'USER': 'postgres',
-            'PASSWORD': '',
+            'PASSWORD': 'postgres',
             'HOST': 'localhost',
-            'PORT': '',
+            'PORT': '5432',
         }
     }
 
@@ -74,23 +76,23 @@
     },
     'handlers': {
         'file': {
-            'level': 'INFO',
+            'level': 'DEBUG',
             'class': 'logging.handlers.RotatingFileHandler',
             'filename': '/var/log/alyx.log',
             'maxBytes': 16777216,
             'formatter': 'simple'
         },
         'console': {
-            'level': 'INFO',
+            'level': 'DEBUG',
             'class': 'logging.StreamHandler',
             'formatter': 'simple'
         },
         'json_file': {
-            'level': 'WARNING',
+            'level': 'DEBUG',
             'class': 'logging.handlers.RotatingFileHandler',
             'filename': '/var/log/alyx_json.log',
-	    'maxBytes': 16777216,
-	    'backupCount': 3,
+            'maxBytes': 16777216,
+            'backupCount': 3,
             'formatter': 'json_formatter',
         },
     },
@@ -102,12 +104,12 @@
         },
         'django_structlog': {
             'handlers': ['json_file'],
-            'level': 'WARNING',
+            'level': 'INFO',
         }
     },
     'root': {
         'handlers': ['file', 'console'],
-        'level': 'INFO',
+        'level': 'WARNING',
         'propagate': True,
     }
 }
@@ -121,7 +123,7 @@
 # See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = False
+DEBUG = True
 
 # Production settings:
 if not DEBUG:
@@ -150,7 +152,6 @@
     'django.contrib.sessions',
     'django.contrib.messages',
     'django.contrib.staticfiles',
-    # 'django_prometheus',
     'mptt',
     'polymorphic',
     'rangefilter',
@@ -170,7 +171,6 @@
 )
 
 MIDDLEWARE = (
-    # 'django_prometheus.middleware.PrometheusBeforeMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
     'django.middleware.common.CommonMiddleware',
     'django.middleware.csrf.CsrfViewMiddleware',
@@ -180,7 +180,6 @@
     'django.middleware.security.SecurityMiddleware',
     'alyx.base.QueryPrintingMiddleware',
     'django_structlog.middlewares.RequestMiddleware',
-    # 'django_prometheus.middleware.PrometheusAfterMiddleware',
 )
 
 ROOT_URLCONF = 'alyx.urls'
@@ -240,10 +239,12 @@
 STATIC_ROOT = os.path.join(BASE_DIR, 'static/')
 STATIC_URL = '/static/'
 
-MEDIA_ROOT = 'backups/uploaded/'
+MEDIA_ROOT = os.path.realpath(os.path.join(BASE_DIR, '../uploaded/'))
+MEDIA_ROOT = '/backups/uploaded/'
 MEDIA_URL = '/uploaded/'
 
 TABLES_ROOT = os.path.realpath(os.path.join(BASE_DIR, '../tables/'))
+TABLES_ROOT = '/backups/tables/'
 
 UPLOADED_IMAGE_WIDTH = 800
 

alyx/data/admin.py

@@ -99,9 +99,9 @@ class DatasetAdmin(BaseExperimentalDataAdmin):
     ordering = ('-created_datetime',)
 
     def get_queryset(self, request):
-        qs = super(DatasetAdmin, self).get_queryset(request)
-        qs = qs.select_related('dataset_type', 'session', 'session__subject', 'created_by')
-        return qs
+        queryset = super(DatasetAdmin, self).get_queryset(request)
+        queryset = queryset.select_related('session', 'session__subject', 'created_by')
+        return queryset
 
     def dataset_type_(self, obj):
         return obj.dataset_type.name
@@ -121,17 +121,17 @@ def subject(self, obj):
         return obj.session.subject.nickname
 
     def _online(self, obj):
-        return obj.online
+        return obj.is_online
     _online.short_description = 'On server'
     _online.boolean = True
 
     def _protected(self, obj):
-        return obj.protected
+        return obj.is_protected
     _protected.short_description = 'Protected'
     _protected.boolean = True
 
     def _public(self, obj):
-        return obj.public
+        return obj.is_public
     _public.short_description = 'Public'
     _public.boolean = True
 

alyx/data/fixtures/data.datasettype.json

@@ -1461,5 +1461,38 @@
     "description": "Times of licks as detected from DLC tongue traces.",
     "filename_pattern": "licks.times.*"
     }
+  },
+      {
+  "model": "data.datasettype",
+  "pk": "828c691b-1f94-4454-9477-a29e2e445b27",
+  "fields": {
+    "json": null,
+    "name": "electrodeSites.localCoordinates",
+    "created_by": null,
+    "description": "Location of each channel relative to probe coordinate system (µm): (first) dimension is on the width of the shank; Second is the depth where 0 is the deepest site, and positive above this. Straight mapping to the raw electrophysiology binary files",
+    "filename_pattern": "electrodeSites.localCoordinates*.npy"
+    }
+  },
+       {
+  "model": "data.datasettype",
+  "pk": "e9d47477-f4ee-4421-b2d0-817e7e889521",
+  "fields": {
+    "json": null,
+    "name": "electrodeSites.mlapdv",
+    "created_by": null,
+    "description": "3d location of the channels relative to bregma following ephys alignment - mediolateral; anterior-posterior; dorsoventral coordinates (um) straight mapping to the raw electrophysiology binary file channels",
+    "filename_pattern": "electrodeSites.mlapdv*.npy"
+    }
+  },
+       {
+  "model": "data.datasettype",
+  "pk": "e1cda381-5a32-4f76-927b-43bb7fbfc874",
+  "fields": {
+    "json": null,
+    "name": "electrodeSites.brainLocationIds_ccf_2017",
+    "created_by": null,
+    "description": "Brain location id of channels following ephys alignment obtained from 25um resolution 2017 Allen Common Coordinate Framework - straight mapping to the raw electrophysiology binary file channels.",
+    "filename_pattern": "electrodeSites.brainLocationIds_ccf_2017*.npy"
+    }
   }
 ]

alyx/data/management/commands/files.py

@@ -115,7 +115,7 @@ def handle(self, *args, **options):
             dtypes = ['ephysData.raw.ap', 'ephysData.raw.lf', 'ephysData.raw.nidq',
                       '_iblrig_Camera.raw', '_kilosort_raw.output']
             frecs = FileRecord.objects.filter(
-                ~Q(data_repository__name__icontains='flatiron'),
+                data_repository__globus_is_personal=True,
                 dataset__dataset_type__name__in=dtypes,
                 exists=True,
                 dataset__session__start_time__date__lte=before,