Permission issue on figURL curation populate

[rpswenson]

After updating to figurl-0.2.22 sortingview-0.13.2 I got this error when running sgs.FigURLCuration.populate([(sgs.FigURLCurationSelection & figurl_key).proj()]) :

Cell In[9], line 84
     81 sgs.FigURLCurationSelection.insert_selection(key)
     82 figurl_key = {
     83     "figurl_curation_id": str(key['figurl_curation_id'])}
---> 84 sgs.FigURLCuration.populate([(sgs.FigURLCurationSelection & figurl_key).proj()])
     86 #sgs.FigURLCuration.populate(figurl_key)

File ~/datajoint-python/datajoint/autopopulate.py:247, in AutoPopulate.populate(self, suppress_errors, return_exception_objects, reserve_jobs, order, limit, max_calls, display_progress, processes, make_kwargs, *restrictions)
    241 if processes == 1:
    242     for key in (
    243         tqdm(keys, desc=self.__class__.__name__)
    244         if display_progress
    245         else keys
    246     ):
--> 247         status = self._populate1(key, jobs, **populate_kwargs)
    248         if status is True:
    249             success_list.append(1)

File ~/datajoint-python/datajoint/autopopulate.py:314, in AutoPopulate._populate1(self, key, jobs, suppress_errors, return_exception_objects, make_kwargs)
    312 self.__class__._allow_insert = True
    313 try:
--> 314     make(dict(key), **(make_kwargs or {}))
    315 except (KeyboardInterrupt, SystemExit, Exception) as error:
    316     try:

File ~/spyglass/src/spyglass/spikesorting/v1/figurl_curation.py:155, in FigURLCuration.make(self, key)
    150 unit_metrics = _reformat_metrics(metric_dict)
    152 # TODO: figure out a way to specify the similarity metrics
    153 
    154 # Generate the figURL
--> 155 key["url"] = _generate_figurl(
    156     R=recording,
    157     S=sorting,
    158     initial_curation_uri=curation_uri,
    159     recording_label=recording_label,
    160     sorting_label=sorting_label,
    161     unit_metrics=unit_metrics,
    162 )
    164 # INSERT
    165 self.insert1(key, skip_duplicates=True)

File ~/spyglass/src/spyglass/spikesorting/v1/figurl_curation.py:205, in _generate_figurl(R, S, initial_curation_uri, recording_label, sorting_label, unit_metrics, segment_duration_sec, snippet_ms_before, snippet_ms_after, max_num_snippets_per_segment, channel_neighborhood_size, raster_plot_subsample_max_firing_rate, spike_amplitudes_subsample_max_firing_rate)
    201 sorting = S
    203 sampling_frequency = recording.get_sampling_frequency()
--> 205 this_view = SpikeSortingView.create(
    206     recording=recording,
    207     sorting=sorting,
    208     segment_duration_sec=segment_duration_sec,
    209     snippet_len=(
    210         int(snippet_ms_before * sampling_frequency / 1000),
    211         int(snippet_ms_after * sampling_frequency / 1000),
    212     ),
    213     max_num_snippets_per_segment=max_num_snippets_per_segment,
    214     channel_neighborhood_size=channel_neighborhood_size,
    215 )
    217 # Assemble the views in a layout. Can be replaced with other layouts.
    218 raster_max_fire = raster_plot_subsample_max_firing_rate

File ~/miniforge3/envs/spyglass/lib/python3.9/site-packages/sortingview/SpikeSortingView/SpikeSortingView.py:41, in SpikeSortingView.create(recording, sorting, segment_duration_sec, snippet_len, max_num_snippets_per_segment, channel_neighborhood_size, bandpass_filter)
     30 @staticmethod
     31 def create(
     32     *,
   (...)
     39     bandpass_filter: bool = False,
     40 ):
---> 41     data_uri = prepare_spikesortingview_data(
     42         recording=recording,
     43         sorting=sorting,
     44         segment_duration_sec=segment_duration_sec,
     45         snippet_len=snippet_len,
     46         max_num_snippets_per_segment=max_num_snippets_per_segment,
     47         channel_neighborhood_size=channel_neighborhood_size,
     48         bandpass_filter=bandpass_filter,
     49     )
     50     return SpikeSortingView(data_uri)

File ~/miniforge3/envs/spyglass/lib/python3.9/site-packages/sortingview/SpikeSortingView/prepare_spikesortingview_data.py:143, in prepare_spikesortingview_data(recording, sorting, segment_duration_sec, snippet_len, max_num_snippets_per_segment, channel_neighborhood_size, bandpass_filter)
    141             index = index + num
    142             f.create_dataset(f"segment/{iseg}/unit/{unit_id}/subsampled_spike_snippets", data=spike_snippets)
--> 143 uri = kcl.store_file_local(output_file_name)
    144 return uri

File ~/miniforge3/envs/spyglass/lib/python3.9/site-packages/kachery_cloud/store_file_local.py:21, in store_file_local(filename, label, reference, store_file_dir, store_file_prefix)
     19 if not os.path.isabs(filename):
     20     filename = os.path.abspath(filename)
---> 21 sha1 = _compute_file_hash(filename, algorithm='sha1')
     22 if store_file_prefix is None:
     23     uri = f'[sha1://{](sha1://{sha1)[sha1](sha1://{sha1)}'

File ~/miniforge3/envs/spyglass/lib/python3.9/site-packages/kachery_cloud/store_file_local.py:84, in _compute_file_hash(path, algorithm)
     82 ret = hashsum.hexdigest()
     83 if size0 > 1000 * 1000:
---> 84     set_mutable_local(f'@compute_sha1_cache/{_sha1_of_string(path)}', json.dumps({
     85         'path': path,
     86         'size': size0,
     87         'mtime': os.stat(path).st_mtime,
     88         'sha1': ret
     89     }))
     90 return ret

File ~/miniforge3/envs/spyglass/lib/python3.9/site-packages/kachery_cloud/mutable_local.py:15, in set_mutable_local(key, value)
     13 if not os.path.exists(parent_dir):
     14     _makedirs(parent_dir)
---> 15 with open(file_name, 'w') as f:
     16     f.write(value)

PermissionError: [Errno 13] Permission denied: '/stelmo/nwb/kachery_storage/mutables/@compute_sha1_cache/e8d9b91a62d82444cb5bcea32799fbf76c1b6530'

Here is the state of my permissions when checking them with dj.conn().query('show grants for current_user();').fetchall() :

(('GRANT SELECT, REFERENCES, SHOW DATABASES ON . TO rswenson@%',),
('GRANT ALL PRIVILEGES ON common\\_%.* TO rswenson@%',),
('GRANT ALL PRIVILEGES ON decoding\\_%.* TO rswenson@%',),
('GRANT ALL PRIVILEGES ON lfp\\_%.* TO rswenson@%',),
('GRANT ALL PRIVILEGES ON linearization\\_%.* TO rswenson@%',),
('GRANT ALL PRIVILEGES ON mua\\_%.* TO rswenson@%',),
('GRANT ALL PRIVILEGES ON position\\_%.* TO rswenson@%',),
('GRANT ALL PRIVILEGES ON position_linearization\\_%.* TO rswenson@%',),
('GRANT ALL PRIVILEGES ON ripple\\_%.* TO rswenson@%',),
('GRANT ALL PRIVILEGES ON rswenson\\_%.* TO rswenson@%',),
('GRANT ALL PRIVILEGES ON spikesorting\\_%.* TO rswenson@%',),
('GRANT ALL PRIVILEGES ON waveform\\_%.* TO rswenson@%',),
('GRANT dj_user@% TO rswenson@%',))

[samuelbray32]

For learning purposes: Erno 13 ios file systems error outside of spyglass or anything python that means you don't have permission to do something in that folder (in this case that's writing files)

I don't know why that folder was open to me but I changed the permission for it now

[rpswenson]

I've got a different one now?

PermissionError: [Errno 13] Permission denied: '/stelmo/nwb/kachery_storage/sha1/ca/01'

[edeno]

Hm, perhaps new versions of kachery or sorting view are creating new directories that they weren't before?

[rpswenson]

Hm, perhaps new versions of kachery or sorting view are creating new directories that they weren't before?

@edeno do you know how we might be able to fix this?

[edeno]

Okay so I traced it down to being related to #882.

We have:

drwxrwsrwx     15 loren       kachery-users     18 Apr  4 01:52 .kachery-cloud
drwxrwsr-x      4 loren       kachery-users      6 Apr 11  2023 .kachery_cloud
drwxr-sr-x      3 sambray     root               5 Mar 19 16:05 kachery-cloud
drwxrwsr-x      2 bsharma     root               4 Jan 25 11:39 kachery_cloud
drwxr-sr-x      2 sambray     root               4 Feb 28 15:38 .kachery_storage
drwxrwsr-x      2 sc4712      root               2 Apr  3 13:02 kachery-storage
drwxrwxrwx      5 cbroz       root               7 Apr  8 14:42 kachery_storage

Our current config has:

spyglass/src/spyglass/settings.py

Lines 72 to 76 in 5d57902

	"kachery": {
	"cloud": ".kachery-cloud",
	"storage": "kachery_storage",
	"temp": "tmp",
	},

kachery_storage should have loren as the owner and kachery-users as the group so I ran:

sudo chown -R loren:kachery-users /stelmo/nwb/kachery_storage

Ideally we should probably not have kachery-storage, .kachery_storage, kachery_cloud, kachery-cloud, and .kachery_cloud

I maybe should not have made this operation recursive, but I think it'll still be okay.

@rpswenson could you try again?

[rpswenson]

@edeno I'm still getting the same [Errno 13] Permission denied: '/stelmo/nwb/kachery_storage/sha1/ca/01'
I'm up to date on spyglass, is there something else I need to update for the fix to work?

[edeno]

No, this is a file system related problem. I need to update the write permissions. Can you try one more time?

[rpswenson]

Ok, it no longer runs into that issue. Thank you!

[CBroz1]

We have:

drwxrwsrwx     15 loren       kachery-users     18 Apr  4 01:52 .kachery-cloud
drwxrwsr-x      4 loren       kachery-users      6 Apr 11  2023 .kachery_cloud
drwxr-sr-x      3 sambray     root               5 Mar 19 16:05 kachery-cloud
drwxrwsr-x      2 bsharma     root               4 Jan 25 11:39 kachery_cloud
drwxr-sr-x      2 sambray     root               4 Feb 28 15:38 .kachery_storage
drwxrwsr-x      2 sc4712      root               2 Apr  3 13:02 kachery-storage
drwxrwxrwx      5 cbroz       root               7 Apr  8 14:42 kachery_storage

Oof 😣 - we have quite a few. Can these be collapsed? Can we converge on .kachery-X for both?