HACK: NSS Support

Author: dwbuiten

configure

@@ -203,6 +203,7 @@ External library support:
   --enable-libxavs         enable AVS encoding via xavs [no]
   --enable-libxvid         enable Xvid encoding via xvidcore,
                            native MPEG-4/Xvid encoder exists [no]
+  --enable-nss             enable NSS [no]
   --enable-openssl         enable openssl [no]
   --enable-x11grab         enable X11 grabbing [no]
   --enable-zlib            enable zlib [autodetect]
@@ -1055,6 +1056,7 @@ EXTERNAL_LIBRARY_LIST="
     libx264
     libxavs
     libxvid
+    nss
     openssl
     x11grab
     zlib
@@ -1819,7 +1821,7 @@ sctp_protocol_deps="struct_sctp_event_subscribe"
 sctp_protocol_select="network"
 srtp_protocol_select="rtp_protocol"
 tcp_protocol_select="network"
-tls_protocol_deps_any="openssl gnutls"
+tls_protocol_deps_any="openssl gnutls nss"
 tls_protocol_select="tcp_protocol"
 udp_protocol_select="network"
 
@@ -3569,6 +3571,7 @@ enabled libx264    && require  libx264 x264.h x264_encoder_encode -lx264 &&
                         die "ERROR: libx264 version must be >= 0.118."; }
 enabled libxavs    && require  libxavs xavs.h xavs_encoder_encode -lxavs
 enabled libxvid    && require  libxvid xvid.h xvid_global -lxvidcore
+enabled nss        && require_pkg_config nss nss.h NSS_Init
 enabled openssl    && { check_lib openssl/ssl.h SSL_library_init -lssl -lcrypto ||
                         check_lib openssl/ssl.h SSL_library_init -lssl32 -leay32 ||
                         check_lib openssl/ssl.h SSL_library_init -lssl -lcrypto -lws2_32 -lgdi32 ||

libavformat/tls.c

@@ -46,6 +46,21 @@
         if (c->ctx) \
             SSL_CTX_free(c->ctx); \
     } while (0)
+#elif CONFIG_NSS
+#include <nspr.h>
+#include <nss.h>
+#include <prinit.h>
+#include <prio.h>
+#include <private/pprio.h>
+#include <prnetdb.h>
+#include <ssl.h>
+#define TLS_read(c, buf, size) PR_Read(c->nssfd, buf, size)
+#define TLS_write(c, buf, size) PR_Write(c->nssfd, buf, size)
+#define TLS_shutdown(c) PR_Close(c->nssfd)
+#define TLS_free(c) do { \
+        if (c->nssai) \
+            PR_FreeAddrInfo(c->nssai); \
+        } while (0)
 #endif
 #include "network.h"
 #include "os_support.h"
@@ -63,10 +78,23 @@ typedef struct {
 #elif CONFIG_OPENSSL
     SSL_CTX *ctx;
     SSL *ssl;
+#elif CONFIG_NSS
+    PRAddrInfo *nssai;
+    PRFileDesc *nssfd;
+    PRStatus cstatus;
+    SECStatus status;
 #endif
     int fd;
 } TLSContext;
 
+#if CONFIG_NSS
+static SECStatus nss_auth_cert_hook(void *arg, PRFileDesc * socket,
+                                    PRBool checksig, PRBool isserver)
+{
+    return SECSuccess;
+}
+#endif
+
 static int do_tls_poll(URLContext *h, int ret)
 {
     TLSContext *c = h->priv_data;
@@ -90,6 +118,22 @@ static int do_tls_poll(URLContext *h, int ret)
         av_log(h, AV_LOG_ERROR, "%s\n", ERR_error_string(ERR_get_error(), NULL));
         return AVERROR(EIO);
     }
+#elif CONFIG_NSS
+    PRPollDesc prdesc;
+
+    prdesc.fd = c->nssfd;
+    prdesc.in_flags = PR_POLL_READ | PR_POLL_WRITE;
+
+    ret = PR_Poll(&prdesc, 1, PR_INTERVAL_NO_TIMEOUT);
+    if (ret < 1) {
+        av_log(h, AV_LOG_ERROR, "Cannot poll.\n");
+        return AVERROR(EIO);
+    }
+
+    if (prdesc.out_flags & PR_POLL_READ)
+        p.events = POLLIN;
+    else if (prdesc.out_flags & PR_POLL_WRITE)
+        p.events = POLLOUT;
 #endif
     if (h->flags & AVIO_FLAG_NONBLOCK)
         return AVERROR(EAGAIN);
@@ -191,6 +235,92 @@ static int tls_open(URLContext *h, const char *uri, int flags)
         if ((ret = do_tls_poll(h, ret)) < 0)
             goto fail;
     }
+#elif CONFIG_NSS
+    PR_Init(PR_USER_THREAD, PR_PRIORITY_NORMAL, 0);
+    
+    c->status = NSS_NoDB_Init(NULL);
+    if (c->status != SECSuccess) {
+        av_log(h, AV_LOG_ERROR, "Cannot snitialize NSS.\n");
+        ret = AVERROR(EIO);
+        goto fail;
+    }
+
+    c->status = NSS_SetDomesticPolicy();
+    if (c->status != SECSuccess) {
+        av_log(h, AV_LOG_ERROR, "Cannot initialize NSS ciphers.\n");
+        ret = AVERROR(EIO);
+        goto fail;
+    }
+
+    c->nssfd = SSL_ImportFD(NULL, PR_ImportTCPSocket(c->fd));
+    if (!c->nssfd) {
+        av_log(h, AV_LOG_ERROR, "Cannot import URL descriptor with NSS.\n");
+        ret = AVERROR(EIO);
+        goto fail;
+    }
+
+    c->status = SSL_OptionSet(c->nssfd, SSL_SECURITY, PR_TRUE);
+    if (c->status != SECSuccess) {
+        av_log(h, AV_LOG_ERROR, "Cannot enable SSL security.\n");
+        ret = AVERROR(EIO);
+        goto fail;
+    }
+
+    c->status = SSL_OptionSet(c->nssfd, SSL_HANDSHAKE_AS_CLIENT, PR_TRUE);
+    if (c->status != SECSuccess) {
+        av_log(h, AV_LOG_ERROR, "Cannot enable client-style handshake.\n");
+        ret = AVERROR(EIO);
+        goto fail;
+    }
+
+    c->status = SSL_AuthCertificateHook(c->nssfd, nss_auth_cert_hook, NULL);
+    if (c->status != SECSuccess) {
+        av_log(h, AV_LOG_ERROR,
+               "Cannot register cetrificate auth callback function with NSS.\n");
+        ret = AVERROR(EIO);
+        goto fail;
+    }
+
+    if (!numerichost) {
+       ret = SSL_SetURL(c->nssfd, host);
+       if (ret < 0) {
+           av_log(h, AV_LOG_ERROR, "Cannot set hostname with NSS.\n");
+           goto fail;
+       }
+    }
+/*
+    c->nssai = PR_GetAddrInfoByName(host, PR_AF_INET, NULL); <-- wrong struct anyway
+    if (!c->nssai) {
+        av_log(h, AV_LOG_ERROR, "Could not get address info with NSS.\n");
+        ret = AVERROR(EIO);
+        goto fail
+    }
+
+    c->cstatus = PR_Connect(c->nssfd, c->nssai, PR_INTERVAL_NO_TIMEOUT);
+    if (c->cstatus != PR_SUCCESS) {
+        av_log(h, AV_LOG_ERROR, "Could not create TLS/SSL session.\n");
+        ret = AVERROR(EIO);
+        goto fail;
+    }*/
+
+    c->status = SSL_ResetHandshake(c->nssfd, PR_FALSE);
+    if (c->status != SECSuccess) {
+        av_log(h, AV_LOG_ERROR, "Cannot reset HandShake.\n");
+        ret = AVERROR(EIO);
+        goto fail;
+    }
+
+    c->status = SSL_ForceHandshake(c->nssfd);
+    if (c->status != SECSuccess) {
+        av_log(h, AV_LOG_ERROR, "Could not negotiate TLS/SSL session.\n");
+        ret = AVERROR(EIO);
+        goto fail;
+    }
+
+//    while(1) {
+//        if ((ret = do_tls_poll(h, 0)) < 0)
+//            goto fail;
+//    }
 #endif
     return 0;
 fail: