Permalink
Browse files

crypto.low_level extension: build blake2 like zstd, see borgbackup#3415

  • Loading branch information...
ThomasWaldmann committed Dec 17, 2017
1 parent b56f6cd commit 4a2fec32b6429109e9425f1fb54998f3e903a781
Showing with 88 additions and 18 deletions.
  1. +15 −17 setup.py
  2. +72 −0 setup_b2.py
  3. +1 −1 src/borg/algorithms/blake2-libselect.h
@@ -14,13 +14,17 @@

import setup_lz4
import setup_zstd
import setup_b2

# True: use the shared liblz4 (>= 1.7.0 / r129) from the system, False: use the bundled lz4 code
prefer_system_liblz4 = True

# True: use the shared libzstd (>= 1.3.0) from the system, False: use the bundled zstd code
prefer_system_libzstd = True

# True: use the shared libb2 from the system, False: use the bundled blake2 code
prefer_system_libb2 = True

min_python = (3, 5)
my_python = sys.version_info

@@ -150,19 +154,9 @@ def detect_openssl(prefixes):
return prefix


def detect_libb2(prefixes):
for prefix in prefixes:
filename = os.path.join(prefix, 'include', 'blake2.h')
if os.path.exists(filename):
with open(filename, 'r') as fd:
if 'blake2b_init' in fd.read():
return prefix


include_dirs = []
library_dirs = []
define_macros = []
crypto_libraries = ['crypto']

possible_openssl_prefixes = ['/usr', '/usr/local', '/usr/local/opt/openssl', '/usr/local/ssl', '/usr/local/openssl',
'/usr/local/borg', '/opt/local', '/opt/pkg', ]
@@ -191,13 +185,13 @@ def detect_libb2(prefixes):
'/usr/local/borg', '/opt/local', '/opt/pkg', ]
if os.environ.get('BORG_LIBB2_PREFIX'):
possible_libb2_prefixes.insert(0, os.environ.get('BORG_LIBB2_PREFIX'))
libb2_prefix = detect_libb2(possible_libb2_prefixes)
if libb2_prefix:
libb2_prefix = setup_b2.b2_system_prefix(possible_libb2_prefixes)
if prefer_system_libb2 and libb2_prefix:
print('Detected and preferring libb2 over bundled BLAKE2')
include_dirs.append(os.path.join(libb2_prefix, 'include'))
library_dirs.append(os.path.join(libb2_prefix, 'lib'))
crypto_libraries.append('b2')
define_macros.append(('BORG_USE_LIBB2', 'YES'))
libb2_system = True
else:
libb2_system = False

possible_libzstd_prefixes = ['/usr', '/usr/local', '/usr/local/opt/libzstd', '/usr/local/libzstd',
'/usr/local/borg', '/opt/local', '/opt/pkg', ]
@@ -778,10 +772,14 @@ def run(self):
compress_ext_kwargs = setup_zstd.zstd_ext_kwargs(bundled_path='src/borg/algorithms/zstd',
system_prefix=libzstd_prefix, system=libzstd_system,
multithreaded=False, legacy=False, **compress_ext_kwargs)
crypto_ext_kwargs = dict(sources=[crypto_ll_source, crypto_helpers], libraries=['crypto'],
include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros)
crypto_ext_kwargs = setup_b2.b2_ext_kwargs(bundled_path='src/borg/algorithms/blake2',
system_prefix=libb2_prefix, system=libb2_system,
**crypto_ext_kwargs)
ext_modules += [
Extension('borg.compress', **compress_ext_kwargs),
Extension('borg.crypto.low_level', [crypto_ll_source, crypto_helpers], libraries=crypto_libraries,
include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros),
Extension('borg.crypto.low_level', **crypto_ext_kwargs),
Extension('borg.hashindex', [hashindex_source]),
Extension('borg.item', [item_source]),
Extension('borg.chunker', [chunker_source]),
@@ -0,0 +1,72 @@
# Support code for building a C extension with blake2 files
#
# Copyright (c) 2016-present, Gregory Szorc (original code for zstd)
# 2017-present, Thomas Waldmann (mods to make it more generic, code for blake2)
# All rights reserved.
#
# This software may be modified and distributed under the terms
# of the BSD license. See the LICENSE file for details.

import os

# b2 files, structure as seen in BLAKE2 (reference implementation) project repository:

b2_sources = [
'ref/blake2b-ref.c',
]

b2_includes = [
'ref',
]


def b2_system_prefix(prefixes):
for prefix in prefixes:
filename = os.path.join(prefix, 'include', 'blake2.h')
if os.path.exists(filename):
with open(filename, 'r') as fd:
if 'blake2b_init' in fd.read():
return prefix


def b2_ext_kwargs(bundled_path, system_prefix=None, system=False, **kwargs):
"""amend kwargs with b2 stuff for a distutils.extension.Extension initialization.
bundled_path: relative (to this file) path to the bundled library source code files
system_prefix: where the system-installed library can be found
system: True: use the system-installed shared library, False: use the bundled library code
kwargs: distutils.extension.Extension kwargs that should be amended
returns: amended kwargs
"""
def multi_join(paths, *path_segments):
"""apply os.path.join on a list of paths"""
return [os.path.join(*(path_segments + (path, ))) for path in paths]

use_system = system and system_prefix is not None

sources = kwargs.get('sources', [])
if not use_system:
sources += multi_join(b2_sources, bundled_path)

include_dirs = kwargs.get('include_dirs', [])
if use_system:
include_dirs += multi_join(['include'], system_prefix)
else:
include_dirs += multi_join(b2_includes, bundled_path)

library_dirs = kwargs.get('library_dirs', [])
if use_system:
library_dirs += multi_join(['lib'], system_prefix)

libraries = kwargs.get('libraries', [])
if use_system:
libraries += ['b2', ]

extra_compile_args = kwargs.get('extra_compile_args', [])
if not use_system:
extra_compile_args += [] # not used yet

ret = dict(**kwargs)
ret.update(dict(sources=sources, extra_compile_args=extra_compile_args,
include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries))
return ret
@@ -1,5 +1,5 @@
#ifdef BORG_USE_LIBB2
#include <blake2.h>
#else
#include "blake2/ref/blake2b-ref.c"
#include "blake2/ref/blake2.h"
#endif

0 comments on commit 4a2fec3

Please sign in to comment.