Skip to content

Commit

Permalink
Overwriting DirectoryStore.__setitem__ is not needed anymore
Browse files Browse the repository at this point in the history
  • Loading branch information
madsbk committed Sep 9, 2022
1 parent 91cb3e8 commit a4703df
Showing 1 changed file with 0 additions and 60 deletions.
60 changes: 0 additions & 60 deletions python/kvikio/zarr.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,11 @@
# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved.
# See file LICENSE for terms.

import errno
import os
import os.path
import shutil
import uuid

import cupy
import zarr.storage
from zarr.util import retry_call

import kvikio
from kvikio._lib.arr import asarray
Expand Down Expand Up @@ -55,59 +51,3 @@ def _tofile(self, a, fn):
assert written == a.nbytes
else:
super()._tofile(a.obj, fn)

def __setitem__(self, key, value):
"""
We have to overwrite this because `DirectoryStore.__setitem__`
converts `value` to a NumPy array always
"""
key = self._normalize_key(key)

# coerce to flat, contiguous buffer (ideally without copying)
arr = asarray(value)
if arr.contiguous:
value = arr
else:
if arr.cuda:
# value = cupy.ascontiguousarray(value)
value = arr.reshape(-1, order="A")
else:
# can flatten without copy
value = arr.reshape(-1, order="A")

# destination path for key
file_path = os.path.join(self.path, key)

# ensure there is no directory in the way
if os.path.isdir(file_path):
shutil.rmtree(file_path)

# ensure containing directory exists
dir_path, file_name = os.path.split(file_path)
if os.path.isfile(dir_path):
raise KeyError(key)
if not os.path.exists(dir_path):
try:
os.makedirs(dir_path)
except OSError as e:
if e.errno != errno.EEXIST:
raise KeyError(key)

# write to temporary file
# note we're not using tempfile.NamedTemporaryFile to avoid
# restrictive file permissions
temp_name = file_name + "." + uuid.uuid4().hex + ".partial"
temp_path = os.path.join(dir_path, temp_name)
try:
self._tofile(value, temp_path)

# move temporary file into place;
# make several attempts at writing the temporary file to get past
# potential antivirus file locking issues
retry_call(
os.replace, (temp_path, file_path), exceptions=(PermissionError,)
)
finally:
# clean up if temp file still exists for whatever reason
if os.path.exists(temp_path): # pragma: no cover
os.remove(temp_path)

0 comments on commit a4703df

Please sign in to comment.