Skip to content

Commit

Permalink
Add a version of std::function that includes a few optimizations in A…
Browse files Browse the repository at this point in the history 
…BI V2.

Patch by Jordan Soyke (jsoyke@google.com)
Reviewed as D55045

The result of running the benchmarks and comparing them can be found
here: https://gist.github.com/EricWF/a77fd42ec87fc98da8039e26d0349498

llvm-svn: 348812
  • Loading branch information
@EricWF
EricWF committed Dec 11, 2018
1 parent ba005aa commit 08e231d
Show file tree
Hide file tree
Showing 2 changed files with 305 additions and 0 deletions.
2 changes: 2 additions & 0 deletions libcxx/include/__config
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,8 @@
// Use the smallest possible integer type to represent the index of the variant.
// Previously libc++ used "unsigned int" exclusivly.
# define _LIBCPP_ABI_VARIANT_INDEX_TYPE_OPTIMIZATION
// Unstable attempt to provide a more optimized std::function
# define _LIBCPP_ABI_OPTIMIZED_FUNCTION
#elif _LIBCPP_ABI_VERSION == 1
# if !defined(_LIBCPP_OBJECT_FORMAT_COFF)
// Enable compiling copies of now inline methods into the dylib to support
Expand Down
303 changes: 303 additions & 0 deletions libcxx/include/functional
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1859,14 +1859,317 @@ template <class _Rp, class... _ArgTypes> class __value_func<_Rp(_ArgTypes...)>
#endif // _LIBCPP_NO_RTTI
};

// Storage for a functor object, to be used with __policy to manage copy and
// destruction.
union __policy_storage
{
mutable char __small[sizeof(void*) * 2];
void* __large;
};

// True if _Fun can safely be held in __policy_storage.__small.
template <typename _Fun>
struct __use_small_storage
: public _VSTD::integral_constant<
bool, sizeof(_Fun) <= sizeof(__policy_storage) &&
alignof(_Fun) <= alignof(__policy_storage) &&
_VSTD::is_trivially_copy_constructible<_Fun>::value &&
_VSTD::is_trivially_destructible<_Fun>::value> {};

// Policy contains information about how to copy, destroy, and move the
// underlying functor. You can think of it as a vtable of sorts.
struct __policy
{
// Used to copy or destroy __large values. null for trivial objects.
void* (*const __clone)(const void*);
void (*const __destroy)(void*);

// True if this is the null policy (no value).
const bool __is_null;

// The target type. May be null if RTTI is disabled.
const std::type_info* const __type_info;

// Returns a pointer to a static policy object suitable for the functor
// type.
template <typename _Fun>
_LIBCPP_INLINE_VISIBILITY static const __policy* __create()
{
return __choose_policy<_Fun>(__use_small_storage<_Fun>());
}

_LIBCPP_INLINE_VISIBILITY
static const __policy* __create_empty()
{
static const _LIBCPP_CONSTEXPR __policy __policy_ = {nullptr, nullptr,
true,
#ifndef _LIBCPP_NO_RTTI
&typeid(void)
#else
nullptr
#endif
};
return &__policy_;
}

private:
template <typename _Fun> static void* __large_clone(const void* __s)
{
const _Fun* __f = static_cast<const _Fun*>(__s);
return __f->__clone();
}

template <typename _Fun> static void __large_destroy(void* __s)
{
typedef allocator_traits<typename _Fun::_Alloc> __alloc_traits;
typedef typename __rebind_alloc_helper<__alloc_traits, _Fun>::type
_FunAlloc;
_Fun* __f = static_cast<_Fun*>(__s);
_FunAlloc __a(__f->__allocator());
__f->destroy();
__a.deallocate(__f, 1);
}

template <typename _Fun>
_LIBCPP_INLINE_VISIBILITY static const __policy*
__choose_policy(/* is_small = */ false_type)
{
static const _LIBCPP_CONSTEXPR __policy __policy_ = {
&__large_clone<_Fun>, &__large_destroy<_Fun>, false,
#ifndef _LIBCPP_NO_RTTI
&typeid(typename _Fun::_Target)
#else
nullptr
#endif
};
return &__policy_;
}

template <typename _Fun>
_LIBCPP_INLINE_VISIBILITY static const __policy*
__choose_policy(/* is_small = */ true_type)
{
static const _LIBCPP_CONSTEXPR __policy __policy_ = {
nullptr, nullptr, false,
#ifndef _LIBCPP_NO_RTTI
&typeid(typename _Fun::_Target)
#else
nullptr
#endif
};
return &__policy_;
}
};

// Used to choose between perfect forwarding or pass-by-value. Pass-by-value is
// faster for types that can be passed in registers.
template <typename _Tp>
using __fast_forward =
typename _VSTD::conditional<_VSTD::is_scalar<_Tp>::value, _Tp, _Tp&&>::type;

// __policy_invoker calls an instance of __alloc_func held in __policy_storage.

template <class _Fp> struct __policy_invoker;

template <class _Rp, class... _ArgTypes>
struct __policy_invoker<_Rp(_ArgTypes...)>
{
typedef _Rp (*__Call)(const __policy_storage*,
__fast_forward<_ArgTypes>...);

__Call __call_;

// Creates an invoker that throws bad_function_call.
_LIBCPP_INLINE_VISIBILITY
__policy_invoker() : __call_(&__call_empty) {}

// Creates an invoker that calls the given instance of __func.
template <typename _Fun>
_LIBCPP_INLINE_VISIBILITY static __policy_invoker __create()
{
return __policy_invoker(&__call_impl<_Fun>);
}

private:
_LIBCPP_INLINE_VISIBILITY
explicit __policy_invoker(__Call __c) : __call_(__c) {}

static _Rp __call_empty(const __policy_storage*,
__fast_forward<_ArgTypes>...)
{
__throw_bad_function_call();
}

template <typename _Fun>
static _Rp __call_impl(const __policy_storage* __buf,
__fast_forward<_ArgTypes>... __args)
{
_Fun* __f = reinterpret_cast<_Fun*>(__use_small_storage<_Fun>::value
? &__buf->__small
: __buf->__large);
return (*__f)(_VSTD::forward<_ArgTypes>(__args)...);
}
};

// __policy_func uses a __policy and __policy_invoker to create a type-erased,
// copyable functor.

template <class _Fp> class __policy_func;

template <class _Rp, class... _ArgTypes> class __policy_func<_Rp(_ArgTypes...)>
{
// Inline storage for small objects.
__policy_storage __buf_;

// Calls the value stored in __buf_. This could technically be part of
// policy, but storing it here eliminates a level of indirection inside
// operator().
typedef __function::__policy_invoker<_Rp(_ArgTypes...)> __invoker;
__invoker __invoker_;

// The policy that describes how to move / copy / destroy __buf_. Never
// null, even if the function is empty.
const __policy* __policy_;

public:
_LIBCPP_INLINE_VISIBILITY
__policy_func() : __policy_(__policy::__create_empty()) {}

template <class _Fp, class _Alloc>
_LIBCPP_INLINE_VISIBILITY __policy_func(_Fp&& __f, const _Alloc& __a)
: __policy_(__policy::__create_empty())
{
typedef __alloc_func<_Fp, _Alloc, _Rp(_ArgTypes...)> _Fun;
typedef allocator_traits<_Alloc> __alloc_traits;
typedef typename __rebind_alloc_helper<__alloc_traits, _Fun>::type
_FunAlloc;

if (__function::__not_null(__f))
{
__invoker_ = __invoker::template __create<_Fun>();
__policy_ = __policy::__create<_Fun>();

_FunAlloc __af(__a);
if (__use_small_storage<_Fun>())
{
::new ((void*)&__buf_.__small)
_Fun(_VSTD::move(__f), _Alloc(__af));
}
else
{
typedef __allocator_destructor<_FunAlloc> _Dp;
unique_ptr<_Fun, _Dp> __hold(__af.allocate(1), _Dp(__af, 1));
::new ((void*)__hold.get())
_Fun(_VSTD::move(__f), _Alloc(__af));
__buf_.__large = __hold.release();
}
}
}

_LIBCPP_INLINE_VISIBILITY
__policy_func(const __policy_func& __f)
: __buf_(__f.__buf_), __invoker_(__f.__invoker_),
__policy_(__f.__policy_)
{
if (__policy_->__clone)
__buf_.__large = __policy_->__clone(__f.__buf_.__large);
}

_LIBCPP_INLINE_VISIBILITY
__policy_func(__policy_func&& __f)
: __buf_(__f.__buf_), __invoker_(__f.__invoker_),
__policy_(__f.__policy_)
{
if (__policy_->__destroy)
{
__f.__policy_ = __policy::__create_empty();
__f.__invoker_ = __invoker();
}
}

_LIBCPP_INLINE_VISIBILITY
~__policy_func()
{
if (__policy_->__destroy)
__policy_->__destroy(__buf_.__large);
}

_LIBCPP_INLINE_VISIBILITY
__policy_func& operator=(__policy_func&& __f)
{
*this = nullptr;
__buf_ = __f.__buf_;
__invoker_ = __f.__invoker_;
__policy_ = __f.__policy_;
__f.__policy_ = __policy::__create_empty();
__f.__invoker_ = __invoker();
return *this;
}

_LIBCPP_INLINE_VISIBILITY
__policy_func& operator=(nullptr_t)
{
const __policy* __p = __policy_;
__policy_ = __policy::__create_empty();
__invoker_ = __invoker();
if (__p->__destroy)
__p->__destroy(__buf_.__large);
return *this;
}

_LIBCPP_INLINE_VISIBILITY
_Rp operator()(_ArgTypes&&... __args) const
{
return __invoker_.__call_(_VSTD::addressof(__buf_),
_VSTD::forward<_ArgTypes>(__args)...);
}

_LIBCPP_INLINE_VISIBILITY
void swap(__policy_func& __f)
{
_VSTD::swap(__invoker_, __f.__invoker_);
_VSTD::swap(__policy_, __f.__policy_);
_VSTD::swap(__buf_, __f.__buf_);
}

_LIBCPP_INLINE_VISIBILITY
explicit operator bool() const _NOEXCEPT
{
return !__policy_->__is_null;
}

#ifndef _LIBCPP_NO_RTTI
_LIBCPP_INLINE_VISIBILITY
const std::type_info& target_type() const _NOEXCEPT
{
return *__policy_->__type_info;
}

template <typename _Tp>
_LIBCPP_INLINE_VISIBILITY const _Tp* target() const _NOEXCEPT
{
if (__policy_->__is_null || typeid(_Tp) != *__policy_->__type_info)
return nullptr;
if (__policy_->__clone) // Out of line storage.
return reinterpret_cast<const _Tp*>(__buf_.__large);
else
return reinterpret_cast<const _Tp*>(&__buf_.__small);
}
#endif // _LIBCPP_NO_RTTI
};

} // __function

template<class _Rp, class ..._ArgTypes>
class _LIBCPP_TEMPLATE_VIS function<_Rp(_ArgTypes...)>
: public __function::__maybe_derive_from_unary_function<_Rp(_ArgTypes...)>,
public __function::__maybe_derive_from_binary_function<_Rp(_ArgTypes...)>
{
#ifndef _LIBCPP_ABI_OPTIMIZED_FUNCTION
typedef __function::__value_func<_Rp(_ArgTypes...)> __func;
#else
typedef __function::__policy_func<_Rp(_ArgTypes...)> __func;
#endif

__func __f_;

Expand Down

0 comments on commit 08e231d

Please sign in to comment.