Skip to content

Commit

Permalink
iotools test
Browse files Browse the repository at this point in the history
  • Loading branch information
@willmcgugan
willmcgugan committed Nov 22, 2016
1 parent dca0697 commit f68aa26
Show file tree
Hide file tree
Showing 6 changed files with 152 additions and 102 deletions.
6 changes: 5 additions & 1 deletion fs/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,10 @@ def __exit__(self, exc_type, exc_value, traceback):

@property
def walk(self):
"""
Get a :class:`fs.walker.BoundWlker` object for this filesystem.
"""
return Walker.bind(self)

# ---------------------------------------------------------------- #
Expand Down Expand Up @@ -225,7 +229,7 @@ def appendtext(self, path, text):
"""
if not isinstance(text, six.text_type):
raise ValueError('must be unicode str')
raise ValueError('must be unicode string')
with self._lock:
with self.open(path, 'at') as append_file:
append_file.write(text)
Expand Down
107 changes: 40 additions & 67 deletions fs/iotools.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
import io
from io import SEEK_SET, SEEK_CUR

from .mode import Mode


class RawWrapper(object):
"""Convert a Python 2 style file-like object in to a IO object."""
Expand Down Expand Up @@ -33,17 +35,21 @@ def seek(self, offset, whence=SEEK_SET):
return self._f.seek(offset, whence)

def readable(self):
if hasattr(self._f, 'readable'):
return self._f.readable()
return 'r' in self.mode
return getattr(
self._f,
'readable',
Mode(self.mode).reading
)

def writable(self):
if hasattr(self._f, 'writeable'):
return self._f.writeable()
return 'w' in self.mode
return getattr(
self._f,
'writable',
Mode(self.mode).writing
)

def seekable(self):
if hasattr(self._f, 'seekable'):
if self.is_io:
return self._f.seekable()
try:
self.seek(0, SEEK_CUR)
Expand All @@ -59,10 +65,8 @@ def truncate(self, size=None):
return self._f.truncate(size)

def write(self, data):
if self.is_io:
return self._f.write(data)
self._f.write(data)
return len(data)
count = self._f.write(data)
return len(data) if count is None else count

def read(self, n=-1):
if n == -1:
Expand Down Expand Up @@ -105,7 +109,7 @@ def __iter__(self):


def make_stream(name,
f,
bin_file,
mode='r',
buffering=-1,
encoding=None,
Expand All @@ -114,27 +118,27 @@ def make_stream(name,
line_buffering=False,
**kwargs):
"""Take a Python 2.x binary file and return an IO Stream."""
r = 'r' in mode
w = 'w' in mode
a = 'a' in mode
reading = 'r' in mode
writing = 'w' in mode
appending = 'a' in mode
binary = 'b' in mode
if '+' in mode:
r = True
w = True
reading = True
writing = True

io_object = RawWrapper(f, mode=mode, name=name)
io_object = RawWrapper(bin_file, mode=mode, name=name)
if buffering >= 0:
if r and w:
if reading and writing:
io_object = io.BufferedRandom(
io_object,
buffering or io.DEFAULT_BUFFER_SIZE
)
elif r:
elif reading:
io_object = io.BufferedReader(
io_object,
buffering or io.DEFAULT_BUFFER_SIZE
)
elif w or a:
elif writing or appending:
io_object = io.BufferedWriter(
io_object,
buffering or io.DEFAULT_BUFFER_SIZE
Expand All @@ -152,55 +156,24 @@ def make_stream(name,
return io_object


def decode_binary(data, encoding=None, errors=None, newline=None):
"""Decode bytes as though read from a text file."""
return io.TextIOWrapper(
io.BytesIO(data),
encoding=encoding,
errors=errors,
newline=newline
).read()


def line_iterator(f, size=None):
def line_iterator(readable_file, size=None):
"""A not terribly efficient char by char line iterator."""
read = f.read
read = readable_file.read
line = []
append = line.append
c = True
join = b''.join
is_terminator = b'\n'.__contains__ # True for '\n' and also for ''
byte = b'1'
if size is None or size < 0:
while c:
c = read(1)
append(c)
if is_terminator(c):
yield join(line)
while byte:
byte = read(1)
line.append(byte)
if byte in b'\n':
yield b''.join(line)
del line[:]

else:
while c:
c = read(1)
append(c)
if is_terminator(c) or len(line) >= size:
yield join(line)
while byte and size:
byte = read(1)
size -= len(byte)
line.append(byte)
if byte in b'\n' or not size:
yield b''.join(line)
del line[:]


if __name__ == "__main__": # pragma: nocover
print("Reading a binary file")
bin_file = open('tests/data/UTF-8-demo.txt', 'rb')
with make_stream('UTF-8-demo.txt', bin_file, 'rb') as f:
print(repr(f))
print(type(f.read(200)))

print("Reading a text file")
bin_file = open('tests/data/UTF-8-demo.txt', 'rb')
with make_stream('UTF-8-demo.txt', bin_file, 'rt') as f:
print(repr(f))
print(type(f.read(200)))

print("Reading a buffered binary file")
bin_file = open('tests/data/UTF-8-demo.txt', 'rb')
with make_stream('UTF-8-demo.txt', bin_file, 'rb', buffering=0) as f:
print(repr(f))
print(type(f.read(200)))
54 changes: 28 additions & 26 deletions fs/memoryfs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,25 +57,22 @@ def _seek_lock(self):
yield
self.pos = self._bytes_io.tell()

def _on_modify(self):
def on_modify(self):
"""Called when file data is modified."""
self.modified_time = time.time()
self._memory_fs._on_modify_file(
self._path,
self.modified_time
)

def _on_access(self):
def on_access(self):
"""Called when file is accessed."""
self.accessed_time = time.time()
self._memory_fs._on_access_file(
self._path,
self.accessed_time
)

# def fileno(self):
# raise io.UnsupportedOperation('fileno')

def flush(self):
pass

Expand All @@ -92,7 +89,7 @@ def next(self):

def readline(self, *args, **kwargs):
with self._seek_lock():
self._on_access()
self.on_access()
return self._bytes_io.readline(*args, **kwargs)

def close(self):
Expand All @@ -105,7 +102,7 @@ def read(self, size=None):
if size is None:
size = -1
with self._seek_lock():
self._on_access()
self.on_access()
return self._bytes_io.read(size)

def readlines(self):
Expand All @@ -114,25 +111,25 @@ def readlines(self):

def seek(self, *args, **kwargs):
with self._seek_lock():
self._on_access()
self.on_access()
return self._bytes_io.seek(*args, **kwargs)

def tell(self):
return self.pos

def truncate(self, *args, **kwargs):
with self._seek_lock():
self._on_modify()
self.on_modify()
self._bytes_io.truncate(*args, **kwargs)

def write(self, data):
with self._seek_lock():
self._on_modify()
self.on_modify()
self._bytes_io.write(data)

def writelines(self, sequence):
with self._seek_lock():
self._on_modify()
self.on_modify()
self._bytes_io.writelines(sequence)

def __enter__(self):
Expand All @@ -150,12 +147,12 @@ def __init__(self, resource_type, name):
self._dir = OrderedDict()
self._open_files = []
self._bytes_file = None
self._lock = RLock()
self.lock = RLock()

_t = time.time()
self.created_time = _t
self.accessed_time = _t
self.modified_time = _t
current_time = time.time()
self.created_time = current_time
self.accessed_time = current_time
self.modified_time = current_time

if not self.is_dir:
self._bytes_file = io.BytesIO()
Expand All @@ -170,7 +167,7 @@ def is_dir(self):

@property
def size(self):
with self._lock:
with self.lock:
if self.is_dir:
return 0
else:
Expand Down Expand Up @@ -206,7 +203,11 @@ def remove_open_file(self, memory_file):
@six.python_2_unicode_compatible
class MemoryFS(FS):
"""
An in-memory filesystem.
An filesystem that stores all file and directory information in
memory. This makes them very fast, but non-permanent.
Memory filesystems are useful for caches, temporary data stores,
unit testing, etc.
"""

Expand All @@ -226,8 +227,8 @@ def __init__(self):
"""
self._meta = self._meta.copy()
self._lock = RLock()
self.root = self._make_dir_entry(ResourceType.directory, '')
super(MemoryFS, self).__init__()

def __repr__(self):
return "MemoryFS()"
Expand Down Expand Up @@ -256,13 +257,13 @@ def _on_close_file(self, mem_file, path):
if dir_entry is not None:
dir_entry.remove_open_file(mem_file)

def _on_access_file(self, path, time):
def _on_access_file(self, path, _time):
dir_entry = self._get_dir_entry(path)
dir_entry.accessed_time = time
dir_entry.accessed_time = _time

def _on_modify_file(self, path, time):
def _on_modify_file(self, path, _time):
dir_entry = self._get_dir_entry(path)
dir_entry.accessed_time = dir_entry.modified_time = time
dir_entry.accessed_time = dir_entry.modified_time = _time

def getinfo(self, path, namespaces=None):
namespaces = namespaces or ()
Expand Down Expand Up @@ -356,8 +357,9 @@ def openbin(self, path, mode="r", buffering=-1, **options):
memory_fs=self,
bytes_io=file_dir_entry.bytes_file,
mode=mode,
lock=file_dir_entry._lock
lock=file_dir_entry.lock
)

file_dir_entry.add_open_file(mem_file)
return mem_file

Expand All @@ -373,7 +375,7 @@ def openbin(self, path, mode="r", buffering=-1, **options):
memory_fs=self,
bytes_io=file_dir_entry.bytes_file,
mode=mode,
lock=file_dir_entry._lock
lock=file_dir_entry.lock
)
file_dir_entry.add_open_file(mem_file)
return mem_file
Expand All @@ -394,7 +396,7 @@ def remove(self, path):

parent_dir_entry.remove_entry(file_name)

def removedir(self, path, force=False):
def removedir(self, path):
_path = self.validatepath(path)

if _path == '/':
Expand Down

0 comments on commit f68aa26

Please sign in to comment.