Dev

Smartscope/core/db_manipulations.py

@@ -86,6 +86,13 @@ def update_target_label(model:models.Model,objects_ids:List[str],value:str,metho
         for obj in new_objs:
             Classifier(object_id=obj, method_name=method,content_type=content_type, label=value).save()
 
+def update_target_status(model:models.Model,objects_ids:List[str],value:str, *args, **kwargs):
+    objs = list(model.objects.filter(pk__in=objects_ids))
+    with transaction.atomic():
+        for obj in objs:
+            obj.status = value
+            obj.save()
+
 
 def set_or_update_refined_finder(object_id, stage_x, stage_y, stage_z):
     from .models.target_label import Finder
@@ -124,7 +131,8 @@ def group_holes_for_BIS(hole_models, max_radius=4, min_group_size=1, queue_all=F
         f'grouping params, max radius = {max_radius}, min group size = {min_group_size}, queue all = {queue_all}, max iterations = {iterations}, score_weight = {score_weight}')
     # Extract coordinated for the holes
     prefetch_related_objects(hole_models, 'finders')
-    coords = np.array([[list(h.finders.all())[0].stage_x, list(h.finders.all())[0].stage_y] for h in hole_models])
+    coords = []
+    coords = np.array([h.stage_coords for h in hole_models])
     input_number = len(hole_models)
     # Generate distance matrix
     cd = cdist(coords, coords)
@@ -275,7 +283,7 @@ def add_high_mag(grid, parent):
 
 def select_n_squares(parent, n):
     squares = np.array(parent.squaremodel_set.all().filter(selected=False, status=None).order_by('area'))
-    squares = [s for s in squares if s.is_good()]
+    squares = [s for s in squares if s.is_good() and not s.is_out_of_range()]
     if len(squares) == 0:
         return
     split_squares = np.array_split(squares, n)
@@ -295,7 +303,7 @@ def select_n_holes(parent, n, is_bis=False):
     holes = list(parent.holemodel_set.filter(
         **filter_fields).order_by('dist_from_center'))
 
-    holes = [h for h in holes if h.is_good()]
+    holes = [h for h in holes if h.is_good() and not h.is_out_of_range()]
 
     if n <= 0:
         with transaction.atomic():
@@ -333,13 +341,13 @@ def select_n_areas(parent, n, is_bis=False):
     if n <= 0:
         with transaction.atomic():
             for t in targets:
-                if t.is_good() and not t.is_excluded()[0]:
+                if t.is_good() and not t.is_excluded()[0] and not t.is_out_of_range():
                     update(t, selected=True, status='queued')
         return
 
     clusters = dict()
     for t in targets:
-        if not t.is_good():
+        if not t.is_good() or t.is_out_of_range():
             continue
         excluded, label = t.is_excluded()
         if excluded:

Smartscope/core/grid/run_square.py

@@ -54,7 +54,7 @@ def process_square_image(square, grid, microscope_id):
             if is_bis:
                 holes = list(HoleModel.display.filter(square_id=square.square_id))
                 holes = group_holes_for_BIS(
-                    [h for h in holes if h.is_good() and not h.is_excluded()[0]],
+                    [h for h in holes if h.is_good() and not h.is_excluded()[0] and not h.is_out_of_range()],
                     max_radius=grid.params_id.bis_max_distance,
                     min_group_size=grid.params_id.min_bis_group_size
                 )

Smartscope/core/interfaces/microscope.py

@@ -34,6 +34,8 @@ class AtlasSettings(BaseModel):
     maxY:int = Field(alias='atlas_max_tiles_Y')
     spotSize:int = Field(alias='spot_size')
     c2:float = Field(alias='c2_perc')
+    atlas_to_search_offset_x:float
+    atlas_to_search_offset_y:float
     atlas_c2_aperture: Optional[int] = None
 
     class Config:

Smartscope/core/interfaces/tfsserialem_interface.py

@@ -13,21 +13,35 @@ class Aperture:
     CONDENSER_3:int=3
     OBJECTIVE:int = 2
 
-def change_aperture_temporarily(function: Callable, aperture:Aperture, aperture_size:Optional[int]):
+def change_aperture_temporarily(function: Callable, aperture:Aperture, aperture_size:Optional[int], wait:int=3):
     def wrapper(*args, **kwargs):
         inital_aperture_size = int(sem.ReportApertureSize(aperture))
         if inital_aperture_size == aperture_size or aperture_size is None:
-            return function(*args, **kwargs) 
+            return function(*args, **kwargs)
+        msg = f'Changing condenser aperture {aperture} from {inital_aperture_size} to {aperture_size} and waiting {wait}s.'
+        sem.Echo(msg)
+        logger.info(msg)
         sem.SetApertureSize(aperture,aperture_size)
+        time.sleep(wait)
         function(*args, **kwargs)
+        msg = f'Resetting condenser aperture to {inital_aperture_size}.'
+        sem.Echo(msg)
+        logger.info(msg)
         sem.SetApertureSize(aperture,inital_aperture_size)
     return wrapper    
 
-def remove_objective_aperture(function: Callable):
+def remove_objective_aperture(function: Callable, wait:int=3):
     def wrapper(*args, **kwargs):
+        msg = 'Removing objective aperture.'
+        sem.Echo(msg)
+        logger.info(msg)
         sem.RemoveAperture(Aperture.OBJECTIVE)
         function(*args, **kwargs)
+        msg = f'Reinserting objective aperture and waiting {wait}s.'
+        sem.Echo(msg)
+        logger.info(msg)
         sem.ReInsertAperture(Aperture.OBJECTIVE)
+        time.sleep(wait)
     return wrapper
 
 class TFSSerialemInterface(SerialemInterface):

Smartscope/core/main_commands.py

@@ -122,13 +122,8 @@ def regroup_bis(grid_id, square_id):
     holes_for_grouping = []
     other_holes = []
     for h in filtered_holes:
-
-        # h.bis_group = None
-        # h.bis_type = None
-        if h.is_good() and not h.is_excluded()[0]:
+        if h.is_good() and not h.is_excluded()[0] and not h.is_out_of_range():
             holes_for_grouping.append(h)
-        # else:
-        #     other_holes.append(h)
 
     logger.info(f'Filtered holes = {len(filtered_holes)}\nHoles for grouping = {len(holes_for_grouping)}')
 
@@ -194,4 +189,56 @@ def download_testfiles(overwrite=False):
         p.wait()
     print('Done.')
 
-
+def get_atlas_to_search_offset(detector_name,maximum=0):
+    if isinstance(maximum, str):
+        maximum = int(maximum)
+    detector = Detector.objects.filter(name__contains=detector_name).first()
+    if detector is None:
+        print(f'Could not find detector with name {detector_name}')
+        return
+    completed_square = SquareModel.just_labels.filter(status='completed',grid_id__session_id__detector_id__pk=detector.pk)
+    count = completed_square.count()
+    no_finder = 0
+    total_done = 0
+    if maximum > 0 and count > maximum:
+        print(f'Found {count} completed squares, limiting to {maximum}')
+        count=maximum
+    else:
+        print(f'Found {count} completed squares')
+
+    for square in completed_square:
+        if total_done == count:
+            break
+        finders = square.finders.all()
+        recenter = finders.values_list('stage_x', 'stage_y').filter(method_name='Recentering').first()
+        original = finders.values_list('stage_x', 'stage_y').filter(method_name='AI square finder').first()
+        if any([recenter is None, original is None]):
+            no_finder+=1
+            total_done+=1
+            print(f'Progress: {total_done/count*100:.0f}%, Skipped: {no_finder}', end='\r')
+            continue
+        diff = np.array(original) - np.array(recenter)
+        if not 'array' in locals():
+            array = diff.reshape(1,2)
+            print(f'Progress: {total_done/count*100:.0f}%, Skipped: {no_finder}', end='\r')
+            total_done+=1
+            continue
+        total_done+=1
+        array = np.append(array, diff.reshape(1,2), axis=0)
+        print(f'Progress: {total_done/count*100:.0f}%, Skipped: {no_finder}', end='\r')
+    print(f'Progress: {total_done/count*100:.0f}%, Skipped: {no_finder}')
+    mean = np.mean(array, axis=0).round(2)
+    std = np.std(array, axis=0).round(2)
+    print(f'Calculated offset from {total_done-no_finder} squares:\n\t X: {mean[0]} +/- {std[0]} um\n\t Y: {mean[1]} +/-  {std[1]} um \n')
+    set_values = 'unset'
+    while set_values not in ['y', 'n']:
+        set_values = input(f'atlas_to_search_offset_x={mean[0]} um\natlas_to_search_offset_y={mean[1]} um\n\nSet values of for {detector}? (y/n)')
+    if set_values == 'y':
+        detector.atlas_to_search_offset_x = mean[0]
+        detector.atlas_to_search_offset_y = mean[1]
+        detector.save()
+        print('Saved')
+        return
+    print('Skip setting values')
+
+

Smartscope/core/models/custom_paths.py

@@ -0,0 +1,26 @@
+from .base_model import *
+
+class CustomGroupPath(BaseModel):
+    group = models.OneToOneField(
+        Group,
+        null=True,
+        on_delete=models.SET_NULL,
+        to_field='name'
+    )
+    path = models.CharField(max_length=300)
+
+    class Meta(BaseModel.Meta):
+        db_table = 'customgrouppath'
+
+    def __str__(self):
+        return f'{self.group.name}: {self.path}'
+
+class CustomUserPath(BaseModel):
+    user = models.OneToOneField(User, null=True, on_delete=models.SET_NULL, to_field='username')
+    path = models.CharField(max_length=300)
+
+    class Meta(BaseModel.Meta):
+        db_table = 'customuserpath'
+
+    def __str__(self):
+        return f'{self.user.username}: {self.path}'

Smartscope/core/models/screening_session.py

@@ -1,21 +1,56 @@
+import logging
+
 from .base_model import *
+
+from .custom_paths import CustomUserPath, CustomGroupPath
 from Smartscope.lib.image.smartscope_storage import SmartscopeStorage
 from Smartscope import __version__ as SmartscopeVersion
 
 # from .microscope import Microscope
 # from .detector import Detector
+logger = logging.getLogger(__name__)
 
 class ScreeningSessionManager(models.Manager):
     def get_queryset(self):
         return super().get_queryset().prefetch_related('microscope_id')\
             .prefetch_related('detector_id')
+
 
+def root_directories(session):
+    root_directories = []
+    if settings.USE_CUSTOM_PATHS:
+        # if session.custom_path is not None:
+        #     root_directories.append(session.custom_path.path)
+        custom_group_path = CustomGroupPath.objects.filter(group=session.group).first()
+        if custom_group_path is not None:
+            root_directories.append(custom_group_path.path)
+        custom_user_path = CustomUserPath.objects.filter(user=session.user).first()
+        if custom_user_path is not None:
+            root_directories.append(custom_user_path.path)
+    if settings.USE_STORAGE:
+        root_directories.append(settings.AUTOSCREENDIR)
+        if (groupname:=session.group.name) is not None:
+            root_directories.append(os.path.join(settings.AUTOSCREENDIR,groupname))
+    if settings.USE_LONGTERMSTORAGE:
+        root_directories.append(settings.AUTOSCREENSTORAGE)
+        if (groupname:=session.group.name) is not None:
+            root_directories.append(os.path.join(settings.AUTOSCREENSTORAGE,groupname))
+    ###FIX AWS STORAGE
+    return root_directories
+
+def find_screening_session(root_directories,directory_name):
+    for directory in root_directories:
+        logger.debug(f'Looking for {directory_name} in {directory}')
+        if os.path.isdir(os.path.join(directory,directory_name)):
+            return os.path.join(directory,directory_name)
+    raise FileNotFoundError(f'Could not find {directory_name} in {root_directories}')
 
 class ScreeningSession(BaseModel):
     from .microscope import Microscope
     from .detector import Detector
 
     session = models.CharField(max_length=30)
+    user = models.ForeignKey(User, null=True, default=None, on_delete=models.SET_NULL, to_field='username')
     group = models.ForeignKey(
         Group,
         null=True,
@@ -49,28 +84,9 @@ def directory(self):
         if (directory:=cache.get(cache_key)) is not None:
             logger.info(f'Session {self} directory from cache.')
             return directory
-
-        if settings.USE_STORAGE:
-            cwd = os.path.join(settings.AUTOSCREENDIR, self.working_dir)
-            if os.path.isdir(cwd):
-                cache.set(cache_key,cwd,timeout=21600)
-                return cwd
-
-        if settings.USE_LONGTERMSTORAGE:
-            cwd_storage = os.path.join(settings.AUTOSCREENSTORAGE, self.working_dir)
-            if os.path.isdir(cwd_storage):
-                cache.set(cache_key,cwd_storage,timeout=21600)
-                return cwd_storage
-
-        if settings.USE_AWS:
-            storage = SmartscopeStorage()
-            if storage.dir_exists(self.working_dir):
-                cache.set(cache_key,self.working_dir,timeout=21600)
-                return self.working_dir
-
-        if settings.USE_STORAGE:
-            cache.set(cache_key,cwd,timeout=21600)
-            return cwd
+        cwd = find_screening_session(root_directories(self),self.working_directory)
+        cache.set(cache_key,cwd,timeout=10800)
+        return cwd
 
     @property
     def stop_file(self):
@@ -87,25 +103,28 @@ def currentGrid(self):
         return self.autoloadergrid_set.all().order_by('position')\
             .exclude(status='complete').first()
 
-    @property
-    def storage(self):
-        return os.path.join(settings.AUTOSCREENSTORAGE, self.working_dir)
+    # @property
+    # def storage(self):
+    #     return os.path.join(settings.AUTOSCREENSTORAGE, self.working_dir)
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         if not self.session_id:
             if not self.date:
                 self.date = datetime.today().strftime('%Y%m%d')
             self.session_id = generate_unique_id(extra_inputs=[self.date, self.session])
+
+    @property
+    def working_directory(self):
+        return f'{self.date}_{self.session}'
 
     def save(self, *args, **kwargs):
         self.session = self.session.replace(' ', '_')
         if not self.version:
             self.version = SmartscopeVersion
-        self.working_dir = os.path.join(self.group.name, f'{self.date}_{self.session}')
         super().save(*args, **kwargs)
         return self
 
     def __str__(self):
-        return f'{self.date}_{self.session}'
+        return self.working_directory
 

Smartscope/core/models/session.py

@@ -34,6 +34,7 @@
 from .screening_session import *
 from .square import *
 from .target import *
+from .change_log import *
 
 
 '''