Permalink
Browse files

Merge pull request borgbackup#3451 from ThomasWaldmann/blake2-ext-bui…

…lding

crypto.low_level ext building (blake2 part)
  • Loading branch information...
ThomasWaldmann committed Dec 18, 2017
2 parents 1e9c90d + 4a2fec3 commit b627944d904c06f83922d5a39a6e1d1a0cebab6a
@@ -14,13 +14,17 @@

import setup_lz4
import setup_zstd
import setup_b2

# True: use the shared liblz4 (>= 1.7.0 / r129) from the system, False: use the bundled lz4 code
prefer_system_liblz4 = True

# True: use the shared libzstd (>= 1.3.0) from the system, False: use the bundled zstd code
prefer_system_libzstd = True

# True: use the shared libb2 from the system, False: use the bundled blake2 code
prefer_system_libb2 = True

min_python = (3, 5)
my_python = sys.version_info

@@ -150,19 +154,9 @@ def detect_openssl(prefixes):
return prefix


def detect_libb2(prefixes):
for prefix in prefixes:
filename = os.path.join(prefix, 'include', 'blake2.h')
if os.path.exists(filename):
with open(filename, 'r') as fd:
if 'blake2b_init' in fd.read():
return prefix


include_dirs = []
library_dirs = []
define_macros = []
crypto_libraries = ['crypto']

possible_openssl_prefixes = ['/usr', '/usr/local', '/usr/local/opt/openssl', '/usr/local/ssl', '/usr/local/openssl',
'/usr/local/borg', '/opt/local', '/opt/pkg', ]
@@ -191,13 +185,13 @@ def detect_libb2(prefixes):
'/usr/local/borg', '/opt/local', '/opt/pkg', ]
if os.environ.get('BORG_LIBB2_PREFIX'):
possible_libb2_prefixes.insert(0, os.environ.get('BORG_LIBB2_PREFIX'))
libb2_prefix = detect_libb2(possible_libb2_prefixes)
if libb2_prefix:
libb2_prefix = setup_b2.b2_system_prefix(possible_libb2_prefixes)
if prefer_system_libb2 and libb2_prefix:
print('Detected and preferring libb2 over bundled BLAKE2')
include_dirs.append(os.path.join(libb2_prefix, 'include'))
library_dirs.append(os.path.join(libb2_prefix, 'lib'))
crypto_libraries.append('b2')
define_macros.append(('BORG_USE_LIBB2', 'YES'))
libb2_system = True
else:
libb2_system = False

possible_libzstd_prefixes = ['/usr', '/usr/local', '/usr/local/opt/libzstd', '/usr/local/libzstd',
'/usr/local/borg', '/opt/local', '/opt/pkg', ]
@@ -778,10 +772,14 @@ def run(self):
compress_ext_kwargs = setup_zstd.zstd_ext_kwargs(bundled_path='src/borg/algorithms/zstd',
system_prefix=libzstd_prefix, system=libzstd_system,
multithreaded=False, legacy=False, **compress_ext_kwargs)
crypto_ext_kwargs = dict(sources=[crypto_ll_source, crypto_helpers], libraries=['crypto'],
include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros)
crypto_ext_kwargs = setup_b2.b2_ext_kwargs(bundled_path='src/borg/algorithms/blake2',
system_prefix=libb2_prefix, system=libb2_system,
**crypto_ext_kwargs)
ext_modules += [
Extension('borg.compress', **compress_ext_kwargs),
Extension('borg.crypto.low_level', [crypto_ll_source, crypto_helpers], libraries=crypto_libraries,
include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros),
Extension('borg.crypto.low_level', **crypto_ext_kwargs),
Extension('borg.hashindex', [hashindex_source]),
Extension('borg.item', [item_source]),
Extension('borg.chunker', [chunker_source]),
@@ -0,0 +1,72 @@
# Support code for building a C extension with blake2 files
#
# Copyright (c) 2016-present, Gregory Szorc (original code for zstd)
# 2017-present, Thomas Waldmann (mods to make it more generic, code for blake2)
# All rights reserved.
#
# This software may be modified and distributed under the terms
# of the BSD license. See the LICENSE file for details.

import os

# b2 files, structure as seen in BLAKE2 (reference implementation) project repository:

b2_sources = [
'ref/blake2b-ref.c',
]

b2_includes = [
'ref',
]


def b2_system_prefix(prefixes):
for prefix in prefixes:
filename = os.path.join(prefix, 'include', 'blake2.h')
if os.path.exists(filename):
with open(filename, 'r') as fd:
if 'blake2b_init' in fd.read():
return prefix


def b2_ext_kwargs(bundled_path, system_prefix=None, system=False, **kwargs):
"""amend kwargs with b2 stuff for a distutils.extension.Extension initialization.
bundled_path: relative (to this file) path to the bundled library source code files
system_prefix: where the system-installed library can be found
system: True: use the system-installed shared library, False: use the bundled library code
kwargs: distutils.extension.Extension kwargs that should be amended
returns: amended kwargs
"""
def multi_join(paths, *path_segments):
"""apply os.path.join on a list of paths"""
return [os.path.join(*(path_segments + (path, ))) for path in paths]

use_system = system and system_prefix is not None

sources = kwargs.get('sources', [])
if not use_system:
sources += multi_join(b2_sources, bundled_path)

include_dirs = kwargs.get('include_dirs', [])
if use_system:
include_dirs += multi_join(['include'], system_prefix)
else:
include_dirs += multi_join(b2_includes, bundled_path)

library_dirs = kwargs.get('library_dirs', [])
if use_system:
library_dirs += multi_join(['lib'], system_prefix)

libraries = kwargs.get('libraries', [])
if use_system:
libraries += ['b2', ]

extra_compile_args = kwargs.get('extra_compile_args', [])
if not use_system:
extra_compile_args += [] # not used yet

ret = dict(**kwargs)
ret.update(dict(sources=sources, extra_compile_args=extra_compile_args,
include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries))
return ret
@@ -1,5 +1,5 @@
#ifdef BORG_USE_LIBB2
#include <blake2.h>
#else
#include "blake2/blake2b-ref.c"
#include "blake2/ref/blake2.h"
#endif
@@ -1,14 +1,14 @@
/*
BLAKE2 reference source code package - reference C implementations
Copyright 2012, Samuel Neves <sneves@dei.uc.pt>. You may use this under the
terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at
your option. The terms of these licenses can be found at:
- CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0
- OpenSSL license : https://www.openssl.org/source/license.html
- Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0
More information about the BLAKE2 hash function can be found at
https://blake2.net.
*/
@@ -158,4 +158,3 @@ static BLAKE2_INLINE void secure_zero_memory(void *v, size_t n)
}

#endif

@@ -193,4 +193,3 @@ extern "C" {
#endif

#endif

@@ -1,14 +1,14 @@
/*
BLAKE2 reference source code package - reference C implementations
Copyright 2012, Samuel Neves <sneves@dei.uc.pt>. You may use this under the
terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at
your option. The terms of these licenses can be found at:
- CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0
- OpenSSL license : https://www.openssl.org/source/license.html
- Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0
More information about the BLAKE2 hash function can be found at
https://blake2.net.
*/
@@ -377,4 +377,3 @@ int main( void )
return -1;
}
#endif

0 comments on commit b627944

Please sign in to comment.