Skip to content
Permalink
Browse files

Use bytearray instead of array.array

I'm just gonna leave this here:

    >>> timeit.timeit("array.array(u'B', itertools.repeat(0, 4096))",
                      number=10000,
                      setup="import array; import itertools")
    3.422340818680823

    >>> timeit.timeit("a=bytearray(4096)", number=10000)
    0.0034010084345936775

Yes, you're seeing it right. 1000x faster. Especially for the block
group searches, which we are doing a lot of separately each time, this
makes a huge difference.

For a real life benchmark... On a large filesystem with >40k blockgroups
I have this reduces the time to retrieve and list all the items about
them (all info in disk cache) from 18 to 3 seconds.
  • Loading branch information
knorrie committed Mar 19, 2017
1 parent 1807c6c commit 9b6407d164eaa79ffe0494fa0ec82632128c783f
Showing with 11 additions and 18 deletions.
  1. +1 −3 btrfs/crc32c.py
  2. +10 −15 btrfs/ioctl.py
@@ -16,8 +16,6 @@
# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
# Boston, MA 02110-1301 USA

import array

table = (
0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4,
0xc79a971f, 0x35f1141c, 0x26a1e7e8, 0xd4ca64eb,
@@ -87,7 +85,7 @@


def name_hash(data):
if not isinstance(data, (array.array, bytearray)):
if not isinstance(data, (bytes, bytearray)):
data = bytes(data, 'utf-8')
crc = 4294967294 # (u32)~1
for char in data:
@@ -19,7 +19,6 @@
from collections import namedtuple
import array
import fcntl
import itertools
import platform
import struct
import uuid
@@ -93,10 +92,6 @@ def _IOWR(_type, nr, _struct):
import btrfs.ctree # noqa


def create_buf(size=4096):
return array.array(u'B', itertools.repeat(0, size))


ioctl_fs_info_args = struct.Struct('=QQ16sLLL980x')
IOC_FS_INFO = _IOR(BTRFS_IOCTL_MAGIC, 31, ioctl_fs_info_args)

@@ -114,7 +109,7 @@ def __str__(self):


def fs_info(fd):
buf = create_buf(ioctl_fs_info_args.size)
buf = bytearray(ioctl_fs_info_args.size)
fcntl.ioctl(fd, IOC_FS_INFO, buf)
return FsInfo(buf)

@@ -136,7 +131,7 @@ def __str__(self):


def dev_info(fd, devid):
buf = create_buf(ioctl_dev_info_args.size)
buf = bytearray(ioctl_dev_info_args.size)
ioctl_dev_info_args.pack_into(buf, 0, devid, b'', 0, 0, b'')
fcntl.ioctl(fd, IOC_DEV_INFO, buf)
return DevInfo(buf)
@@ -160,7 +155,7 @@ def __str__(self):


def dev_stats(fd, devid, reset=False):
buf = create_buf(ioctl_get_dev_stats.size)
buf = bytearray(ioctl_get_dev_stats.size)
ioctl_get_dev_stats.pack_into(buf, 0, devid, 5, int(reset), 0, 0, 0, 0, 0)
fcntl.ioctl(fd, IOC_GET_DEV_STATS, buf)
return DevStats(buf)
@@ -190,15 +185,15 @@ def __str__(self):


def space_args(fd):
buf = create_buf(ioctl_space_args.size)
buf = bytearray(ioctl_space_args.size)
fcntl.ioctl(fd, IOC_SPACE_INFO, buf)
return SpaceArgs(*ioctl_space_args.unpack(buf))


def space_info(fd):
args = space_args(fd)
buf_size = ioctl_space_args.size + ioctl_space_info.size * args.total_spaces
buf = create_buf(buf_size)
buf = bytearray(buf_size)
ioctl_space_args.pack_into(buf, 0, args.total_spaces, 0)
fcntl.ioctl(fd, IOC_SPACE_INFO, buf)
return [SpaceInfo(buf, pos)
@@ -241,9 +236,9 @@ def search_v2(fd, tree, min_key=None, max_key=None,
if _v2:
if buf_size is None:
buf_size = 16384
buf = create_buf(ioctl_search_args_v2.size + buf_size)
buf = bytearray(ioctl_search_args_v2.size + buf_size)
else:
buf = create_buf(4096)
buf = bytearray(4096)
while min_key <= max_key and result_nr_items != 0 and wanted_nr_items > 0:
pos = 0
ioctl_search_key.pack_into(buf, pos, tree,
@@ -283,9 +278,9 @@ def search_v2(fd, tree, min_key=None, max_key=None,

def logical_to_ino(fd, vaddr, bufsize=4096):
bufsize = min(bufsize, 65536)
inodes_buf = create_buf(bufsize)
inodes_buf = array.array(u'B', bytearray(bufsize))
inodes_ptr = inodes_buf.buffer_info()[0]
args = create_buf(ioctl_logical_ino_args.size)
args = bytearray(ioctl_logical_ino_args.size)
ioctl_logical_ino_args.pack_into(args, 0, vaddr, bufsize, inodes_ptr)
fcntl.ioctl(fd, IOC_LOGICAL_INO, args)
bytes_left, bytes_missing, elem_cnt, elem_missed = data_container.unpack_from(inodes_buf, 0)
@@ -303,7 +298,7 @@ def logical_to_ino(fd, vaddr, bufsize=4096):


def ino_lookup(fd, treeid=0, objectid=FIRST_FREE_OBJECTID):
args = create_buf(ioctl_ino_lookup_args.size)
args = bytearray(ioctl_ino_lookup_args.size)
ioctl_ino_lookup_args.pack_into(args, 0, treeid, objectid, b'')
fcntl.ioctl(fd, IOC_INO_LOOKUP, args)
treeid, _, name_bytes = ioctl_ino_lookup_args.unpack_from(args, 0)

0 comments on commit 9b6407d

Please sign in to comment.
You can’t perform that action at this time.