Skip to content
Permalink
Browse files

obs-outputs: Add support for and use mbedTLS for SSL

This diff adds mbedTLS support to the obs-outputs plugin.  PolarSSL and
mbedTLS have grown so different between 2015-or-so when libRTMP was
written, and now it's no longer feasible to just use the USE_POLARSSL
flag.

This commit adds a WITH_RTMPS tri-state CMake variable (auto/on/off),
set to "Auto" by default.  "Auto" will use RTMPS if mbedTLS is found,
otherwise will disable RTMPS.  "On" will make it require mbedTLS,
otherwise fails configuration, and "Off" disables RTMPS support
altogether.

Closes #1360
  • Loading branch information
compiler-errors authored and jp9000 committed Jul 3, 2018
1 parent 4e07ac1 commit e67e2e12e61a5f5e111afc445b2f3256b8034c9b
@@ -0,0 +1,143 @@
# Once done these will be defined:
#
# LIBMBEDTLS_FOUND
# LIBMBEDTLS_INCLUDE_DIRS
# LIBMBEDTLS_LIBRARIES
#
# For use in OBS:
#
# MBEDTLS_INCLUDE_DIR

find_package(PkgConfig QUIET)
if (PKG_CONFIG_FOUND)
pkg_check_modules(_MBEDTLS QUIET mbedtls)
endif()

if(CMAKE_SIZEOF_VOID_P EQUAL 8)
set(_lib_suffix 64)
else()
set(_lib_suffix 32)
endif()

# If we're on MacOS or Linux, please try to statically-link mbedtls.
if(STATIC_MBEDTLS AND (APPLE OR UNIX))
set(_MBEDTLS_LIBRARIES libmbedtls.a)
set(_MBEDCRYPTO_LIBRARIES libmbedcrypto.a)
set(_MBEDX509_LIBRARIES libmbedx509.a)
endif()

find_path(MBEDTLS_INCLUDE_DIR
NAMES mbedtls/ssl.h
HINTS
ENV mbedtlsPath${_lib_suffix}
ENV mbedtlsPath
ENV DepsPath${_lib_suffix}
ENV DepsPath
${mbedtlsPath${_lib_suffix}}
${mbedtlsPath}
${DepsPath${_lib_suffix}}
${DepsPath}
${_MBEDTLS_INCLUDE_DIRS}
PATHS
/usr/include /usr/local/include /opt/local/include /sw/include
PATH_SUFFIXES
include)

find_library(MBEDTLS_LIB
NAMES ${_MBEDTLS_LIBRARIES} mbedtls libmbedtls
HINTS
ENV mbedtlsPath${_lib_suffix}
ENV mbedtlsPath
ENV DepsPath${_lib_suffix}
ENV DepsPath
${mbedtlsPath${_lib_suffix}}
${mbedtlsPath}
${DepsPath${_lib_suffix}}
${DepsPath}
${_MBEDTLS_LIBRARY_DIRS}
PATHS
/usr/lib /usr/local/lib /opt/local/lib /sw/lib
PATH_SUFFIXES
lib${_lib_suffix} lib
libs${_lib_suffix} libs
bin${_lib_suffix} bin
../lib${_lib_suffix} ../lib
../libs${_lib_suffix} ../libs
../bin${_lib_suffix} ../bin)

find_library(MBEDCRYPTO_LIB
NAMES ${_MBEDCRYPTO_LIBRARIES} mbedcrypto libmbedcrypto
HINTS
ENV mbedcryptoPath${_lib_suffix}
ENV mbedcryptoPath
ENV DepsPath${_lib_suffix}
ENV DepsPath
${mbedcryptoPath${_lib_suffix}}
${mbedcryptoPath}
${DepsPath${_lib_suffix}}
${DepsPath}
${_MBEDCRYPTO_LIBRARY_DIRS}
PATHS
/usr/lib /usr/local/lib /opt/local/lib /sw/lib
PATH_SUFFIXES
lib${_lib_suffix} lib
libs${_lib_suffix} libs
bin${_lib_suffix} bin
../lib${_lib_suffix} ../lib
../libs${_lib_suffix} ../libs
../bin${_lib_suffix} ../bin)

find_library(MBEDX509_LIB
NAMES ${_MBEDX509_LIBRARIES} mbedx509 libmbedx509
HINTS
ENV mbedx509Path${_lib_suffix}
ENV mbedx509Path
ENV DepsPath${_lib_suffix}
ENV DepsPath
${mbedx509Path${_lib_suffix}}
${mbedx509Path}
${DepsPath${_lib_suffix}}
${DepsPath}
${_MBEDX509_LIBRARY_DIRS}
PATHS
/usr/lib /usr/local/lib /opt/local/lib /sw/lib
PATH_SUFFIXES
lib${_lib_suffix} lib
libs${_lib_suffix} libs
bin${_lib_suffix} bin
../lib${_lib_suffix} ../lib
../libs${_lib_suffix} ../libs
../bin${_lib_suffix} ../bin)

# Sometimes mbedtls is split between three libs, and sometimes it isn't.
# If it isn't, let's check if the symbols we need are all in MBEDTLS_LIB.
if(MBEDTLS_LIB AND NOT MBEDCRYPTO_LIB AND NOT MBEDX509_LIB)
set(CMAKE_REQUIRED_LIBRARIES ${MBEDTLS_LIB})
set(CMAKE_REQUIRED_INCLUDES ${MBEDTLS_INCLUDE_DIR})
check_symbol_exists(mbedtls_x509_crt_init "mbedtls/x509_crt.h" MBEDTLS_INCLUDES_X509)
check_symbol_exists(mbedtls_sha256_init "mbedtls/sha256.h" MBEDTLS_INCLUDES_CRYPTO)
unset(CMAKE_REQUIRED_INCLUDES)
unset(CMAKE_REQUIRED_LIBRARIES)
endif()

# If we find all three libraries, then go ahead.
if(MBEDTLS_LIB AND MBEDCRYPTO_LIB AND MBEDX509_LIB)
set(LIBMBEDTLS_INCLUDE_DIRS ${MBEDTLS_INCLUDE_DIR})
set(LIBMBEDTLS_LIBRARIES ${MBEDTLS_LIB} ${MBEDCRYPTO_LIB} ${MBEDX509_LIB})
set(MBEDTLS_INCLUDE_DIRS ${LIBMBEDTLS_INCLUDE_DIRS})
set(MBEDTLS_LIBRARIES ${LIBMBEDTLS_LIBRARIES})

# Otherwise, if we find MBEDTLS_LIB, and it has both CRYPTO and x509
# within the single lib (i.e. a windows build environment), then also
# feel free to go ahead.
elseif(MBEDTLS_LIB AND MBEDTLS_INCLUDES_CRYPTO AND MBEDTLS_INCLUDES_X509)
set(LIBMBEDTLS_INCLUDE_DIRS ${MBEDTLS_INCLUDE_DIR})
set(LIBMBEDTLS_LIBRARIES ${MBEDTLS_LIB})
set(MBEDTLS_INCLUDE_DIRS ${LIBMBEDTLS_INCLUDE_DIRS})
set(MBEDTLS_LIBRARIES ${LIBMBEDTLS_LIBRARIES})
endif()

# Now we've accounted for the 3-vs-1 library case:
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(Libmbedtls DEFAULT_MSG MBEDTLS_LIBRARIES MBEDTLS_INCLUDE_DIRS)
mark_as_advanced(MBEDTLS_INCLUDE_DIR MBEDTLS_LIBRARIES MBEDTLS_INCLUDE_DIRS)

This file was deleted.

@@ -1,20 +1,25 @@
project(obs-outputs)

option(USE_SSL "Enable rtmps support with OpenSSL" OFF)
set(WITH_RTMPS AUTO CACHE STRING "Enable RTMPS support with mbedTLS")
set_property(CACHE WITH_RTMPS PROPERTY STRINGS AUTO ON OFF)

if (USE_SSL)
find_package(SSL QUIET)
option(STATIC_MBEDTLS "Statically link mbedTLS into binary" OFF)

if (WITH_RTMPS OR (WITH_RTMPS STREQUAL "AUTO"))
find_package(MbedTLS QUIET)
find_package(ZLIB QUIET)
endif()

if (SSL_FOUND AND ZLIB_FOUND)
add_definitions(-DCRYPTO -DUSE_OPENSSL)
include_directories(${SSL_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIRS})
if (LIBMBEDTLS_FOUND AND ZLIB_FOUND)
add_definitions(-DCRYPTO -DUSE_MBEDTLS)
include_directories(${LIBMBEDTLS_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIRS})
else()
if (USE_SSL)
message(WARNING "SSL enabled by user, but OpenSSL was not found")
if(WITH_RTMPS STREQUAL "AUTO")
message(WARNING "mbedTLS was not found, RTMPS will be auto-disabled")
elseif (WITH_RTMPS)
message(FATAL_ERROR "RTMPS enabled by user, but mbedTLS was not found")
endif()
unset(SSL_LIBRARIES)
unset(LIBMBEDTLS_LIBRARIES)
unset(ZLIB_LIBRARIES)
add_definitions(-DNO_CRYPTO)
endif()
@@ -84,6 +89,12 @@ if(WIN32)
ws2_32
winmm
Iphlpapi)

if (WITH_RTMPS OR (WITH_RTMPS STREQUAL "AUTO"))
SET(obs-outputs_PLATFORM_DEPS
${obs-outputs_PLATFORM_DEPS}
crypt32)
endif()
endif()

if(MSVC)
@@ -92,6 +103,16 @@ if(MSVC)
w32-pthreads)
endif()

if(APPLE AND (WITH_RTMPS OR (WITH_RTMPS STREQUAL "AUTO")))
find_library(FOUNDATION_FRAMEWORK Foundation)
find_library(SECURITY_FRAMEWORK Security)

set(obs-outputs_PLATFORM_DEPS
${obs-outputs_PLATFORM_DEPS}
${FOUNDATION_FRAMEWORK}
${SECURITY_FRAMEWORK})
endif()

set(obs-outputs_librtmp_HEADERS
librtmp/amf.h
librtmp/bytes.h
@@ -133,7 +154,7 @@ set(obs-outputs_SOURCES
flv-output.c
flv-mux.c
net-if.c)

add_library(obs-outputs MODULE
${ftl_SOURCES}
${ftl_HEADERS}
@@ -143,7 +164,7 @@ add_library(obs-outputs MODULE
${obs-outputs_librtmp_HEADERS})
target_link_libraries(obs-outputs
libobs
${SSL_LIBRARIES}
${LIBMBEDTLS_LIBRARIES}
${ZLIB_LIBRARIES}
${ftl_IMPORTS}
${obs-outputs_PLATFORM_DEPS})
@@ -11,3 +11,4 @@ ConnectionReset="The connection was reset by the peer. This usually indicates in
HostNotFound="Hostname not found. Make sure you entered a valid streaming server and your internet connection / DNS are working correctly."
NoData="Hostname found, but no data of the requested type. This can occur if you have bound to an IPv6 address and your streaming service only has IPv4 addresses (see Settings / Advanced)."
AddressNotAvailable="Address not available. You may have tried to bind to an invalid IP address (see Settings / Advanced)."
SSLCertVerifyFailed="The RTMP server sent an invalid SSL certificate."
@@ -21,7 +21,59 @@
* http://www.gnu.org/copyleft/lgpl.html
*/

#ifdef USE_POLARSSL
#if defined(USE_MBEDTLS)
#include <mbedtls/dhm.h>
#include <mbedtls/bignum.h>
typedef mbedtls_mpi* MP_t;
#define MP_new(m) m = malloc(sizeof(mbedtls_mpi)); mbedtls_mpi_init(m)
#define MP_set_w(mpi, w) mbedtls_mpi_lset(mpi, w)
#define MP_cmp(u, v) mbedtls_mpi_cmp_mpi(u, v)
#define MP_set(u, v) mbedtls_mpi_copy(u, v)
#define MP_sub_w(mpi, w) mbedtls_mpi_sub_int(mpi, mpi, w)
#define MP_cmp_1(mpi) mbedtls_mpi_cmp_int(mpi, 1)
#define MP_modexp(r, y, q, p) mbedtls_mpi_exp_mod(r, y, q, p, NULL)
#define MP_free(mpi) mbedtls_mpi_free(mpi); free(mpi)
#define MP_gethex(u, hex, res) MP_new(u); res = mbedtls_mpi_read_string(u, 16, hex) == 0
#define MP_bytes(u) mbedtls_mpi_size(u)
#define MP_setbin(u,buf,len) mbedtls_mpi_write_binary(u,buf,len)
#define MP_getbin(u,buf,len) MP_new(u); mbedtls_mpi_read_binary(u,buf,len)

typedef struct MDH
{
MP_t p;
MP_t g;
MP_t pub_key;
MP_t priv_key;
long length;
mbedtls_dhm_context ctx;
} MDH;

#define MDH_new() calloc(1,sizeof(MDH))
#define MDH_free(vp) {MDH *_dh = vp; mbedtls_dhm_free(&_dh->ctx); MP_free(_dh->p); MP_free(_dh->g); MP_free(_dh->pub_key); MP_free(_dh->priv_key); free(_dh);}

static int MDH_generate_key(MDH *dh)
{
unsigned char out[2];
MP_set(&dh->ctx.P, dh->p);
MP_set(&dh->ctx.G, dh->g);
dh->ctx.len = 128;
mbedtls_dhm_make_public(&dh->ctx, 1024, out, 1, mbedtls_ctr_drbg_random, &RTMP_TLS_ctx->ctr_drbg);
MP_new(dh->pub_key);
MP_new(dh->priv_key);
MP_set(dh->pub_key, &dh->ctx.GX);
MP_set(dh->priv_key, &dh->ctx.X);
return 1;
}

static int MDH_compute_key(uint8_t *secret, size_t len, MP_t pub, MDH *dh)
{
MP_set(&dh->ctx.GY, pub);
size_t olen;
mbedtls_dhm_calc_secret(&dh->ctx, secret, len, &olen, NULL, NULL);
return 0;
}

#elif defined(USE_POLARSSL)
#include <polarssl/dhm.h>
typedef mpi * MP_t;
#define MP_new(m) m = malloc(sizeof(mpi)); mpi_init(m)
@@ -271,7 +323,7 @@ DHGetPublicKey(MDH *dh, uint8_t *pubkey, size_t nPubkeyLen)
if (!dh || !dh->pub_key)
return 0;

len = MP_bytes(dh->pub_key);
len = (int)MP_bytes(dh->pub_key);
if (len <= 0 || len > (int) nPubkeyLen)
return 0;

0 comments on commit e67e2e1

Please sign in to comment.
You can’t perform that action at this time.