From 9fcd703812bb166abdbe2ff6c8b404d1c7e7d308 Mon Sep 17 00:00:00 2001 From: zoff99 Date: Fri, 19 Jan 2018 22:59:42 +0100 Subject: [PATCH] Improve video key frame sending. This change does not include the addition of VP9. We do that in a separate pull request. Changes: * fix the video bug (video frames larger than 65KBytes) by sending full frame length in alternate header field * improve video frame reconstruction logic with slots * configure video encoder and decoder to be multihtreaded * set error resilience flags on video codec * change encoder and decoder softdeadline --- testing/BUILD.bazel | 4 +- toxav/bwcontroller.c | 99 ++--- toxav/bwcontroller.h | 2 +- toxav/rtp.c | 874 +++++++++++++++++++++++++++++++------------ toxav/rtp.h | 63 +++- toxav/rtp_test.cpp | 2 +- toxav/toxav.c | 90 +++-- toxav/video.c | 292 ++++++++++++--- toxav/video.h | 5 +- 9 files changed, 1028 insertions(+), 403 deletions(-) diff --git a/testing/BUILD.bazel b/testing/BUILD.bazel index afb1a8aa8e..6cc7018ee7 100644 --- a/testing/BUILD.bazel +++ b/testing/BUILD.bazel @@ -34,9 +34,9 @@ cc_binary( "//c-toxcore/toxav", "//c-toxcore/toxav:monolith", "//c-toxcore/toxcore", - "@portaudio", - "@sndfile", "@opencv//:core", "@opencv//:highgui", + "@portaudio", + "@sndfile", ], ) diff --git a/toxav/bwcontroller.c b/toxav/bwcontroller.c index 9d1ad9ced1..defb3fdef7 100644 --- a/toxav/bwcontroller.c +++ b/toxav/bwcontroller.c @@ -32,9 +32,10 @@ #include #define BWC_PACKET_ID 196 -#define BWC_SEND_INTERVAL_MS 1000 -#define BWC_REFRESH_INTERVAL_MS 10000 +#define BWC_SEND_INTERVAL_MS 950 /* 0.95s */ +#define BWC_REFRESH_INTERVAL_MS 2000 /* 2.00s */ #define BWC_AVG_PKT_COUNT 20 +#define BWC_AVG_LOSS_OVER_CYCLES_COUNT 30 struct BWController_s { void (*mcb)(BWController *, uint32_t, float, void *); @@ -56,6 +57,8 @@ struct BWController_s { uint32_t packet_length_array[BWC_AVG_PKT_COUNT]; RingBuffer *rb; } rcvpkt; /* To calculate average received packet (this means split parts, not the full message!) */ + + uint32_t packet_loss_counted_cycles; }; struct BWCMessage { @@ -71,23 +74,23 @@ BWController *bwc_new(Messenger *m, uint32_t friendnumber, void *udata) { BWController *retu = (BWController *)calloc(sizeof(struct BWController_s), 1); - + LOGGER_DEBUG(m->log, "Creating bandwidth controller"); retu->mcb = mcb; retu->mcb_data = udata; retu->m = m; retu->friend_number = friendnumber; retu->cycle.last_sent_timestamp = retu->cycle.last_refresh_timestamp = current_time_monotonic(); retu->rcvpkt.rb = rb_new(BWC_AVG_PKT_COUNT); + retu->cycle.lost = 0; + retu->cycle.recv = 0; + retu->packet_loss_counted_cycles = 0; /* Fill with zeros */ - int i = 0; - - for (; i < BWC_AVG_PKT_COUNT; i ++) { - rb_write(retu->rcvpkt.rb, retu->rcvpkt.packet_length_array + i); + for (int i = 0; i < BWC_AVG_PKT_COUNT; i++) { + rb_write(retu->rcvpkt.rb, &retu->rcvpkt.packet_length_array[i]); } m_callback_rtp_packet(m, friendnumber, BWC_PACKET_ID, bwc_handle_data, retu); - return retu; } @@ -98,47 +101,21 @@ void bwc_kill(BWController *bwc) } m_callback_rtp_packet(bwc->m, bwc->friend_number, BWC_PACKET_ID, nullptr, nullptr); - rb_kill(bwc->rcvpkt.rb); free(bwc); } -void bwc_feed_avg(BWController *bwc, uint32_t bytes) -{ - uint32_t *p; - - rb_read(bwc->rcvpkt.rb, (void **) &p); - rb_write(bwc->rcvpkt.rb, p); - - *p = bytes; -} - void bwc_add_lost(BWController *bwc, uint32_t bytes_lost) { if (!bwc) { return; } - if (!bytes_lost) { - uint32_t *t_avg[BWC_AVG_PKT_COUNT], c = 1; - - rb_data(bwc->rcvpkt.rb, (void **) t_avg); - - int i = 0; - - for (; i < BWC_AVG_PKT_COUNT; i ++) { - bytes_lost += *(t_avg[i]); - - if (*(t_avg[i])) { - c++; - } - } - - bytes_lost /= c; + if (bytes_lost > 0) { + LOGGER_DEBUG(bwc->m->log, "BWC lost(1): %d", (int)bytes_lost); + bwc->cycle.lost += bytes_lost; + send_update(bwc); } - - bwc->cycle.lost += bytes_lost; - send_update(bwc); } void bwc_add_recv(BWController *bwc, uint32_t recv_bytes) @@ -147,37 +124,35 @@ void bwc_add_recv(BWController *bwc, uint32_t recv_bytes) return; } + bwc->packet_loss_counted_cycles++; bwc->cycle.recv += recv_bytes; send_update(bwc); } - void send_update(BWController *bwc) { - if (current_time_monotonic() - bwc->cycle.last_refresh_timestamp > BWC_REFRESH_INTERVAL_MS) { - - bwc->cycle.lost /= 10; - bwc->cycle.recv /= 10; - bwc->cycle.last_refresh_timestamp = current_time_monotonic(); - } else if (current_time_monotonic() - bwc->cycle.last_sent_timestamp > BWC_SEND_INTERVAL_MS) { + if (bwc->packet_loss_counted_cycles > BWC_AVG_LOSS_OVER_CYCLES_COUNT && + current_time_monotonic() - bwc->cycle.last_sent_timestamp > BWC_SEND_INTERVAL_MS) { + bwc->packet_loss_counted_cycles = 0; if (bwc->cycle.lost) { - LOGGER_DEBUG(bwc->m->log, "%p Sent update rcv: %u lost: %u", - bwc, bwc->cycle.recv, bwc->cycle.lost); - - uint8_t p_msg[sizeof(struct BWCMessage) + 1]; - struct BWCMessage *b_msg = (struct BWCMessage *)(p_msg + 1); - - p_msg[0] = BWC_PACKET_ID; - b_msg->lost = net_htonl(bwc->cycle.lost); - b_msg->recv = net_htonl(bwc->cycle.recv); - - if (-1 == m_send_custom_lossy_packet(bwc->m, bwc->friend_number, p_msg, sizeof(p_msg))) { - LOGGER_WARNING(bwc->m->log, "BWC send failed (len: %d)! std error: %s", sizeof(p_msg), strerror(errno)); + LOGGER_DEBUG(bwc->m->log, "%p Sent update rcv: %u lost: %u percent: %f %%", + bwc, bwc->cycle.recv, bwc->cycle.lost, + (((float) bwc->cycle.lost / (bwc->cycle.recv + bwc->cycle.lost)) * 100.0f)); + uint8_t bwc_packet[sizeof(struct BWCMessage) + 1]; + struct BWCMessage *msg = (struct BWCMessage *)(bwc_packet + 1); + bwc_packet[0] = BWC_PACKET_ID; // set packet ID + msg->lost = net_htonl(bwc->cycle.lost); + msg->recv = net_htonl(bwc->cycle.recv); + + if (-1 == m_send_custom_lossy_packet(bwc->m, bwc->friend_number, bwc_packet, sizeof(bwc_packet))) { + LOGGER_WARNING(bwc->m->log, "BWC send failed (len: %d)! std error: %s", sizeof(bwc_packet), strerror(errno)); } } bwc->cycle.last_sent_timestamp = current_time_monotonic(); + bwc->cycle.lost = 0; + bwc->cycle.recv = 0; } } @@ -185,9 +160,9 @@ static int on_update(BWController *bwc, const struct BWCMessage *msg) { LOGGER_DEBUG(bwc->m->log, "%p Got update from peer", bwc); - /* Peer must respect time boundary */ - if (current_time_monotonic() < bwc->cycle.last_recv_timestamp + BWC_SEND_INTERVAL_MS) { - LOGGER_DEBUG(bwc->m->log, "%p Rejecting extra update", bwc); + /* Peers sent update too soon */ + if (bwc->cycle.last_recv_timestamp + BWC_SEND_INTERVAL_MS > current_time_monotonic()) { + LOGGER_INFO(bwc->m->log, "%p Rejecting extra update", bwc); return -1; } @@ -196,9 +171,9 @@ static int on_update(BWController *bwc, const struct BWCMessage *msg) uint32_t recv = net_ntohl(msg->recv); uint32_t lost = net_ntohl(msg->lost); - LOGGER_DEBUG(bwc->m->log, "recved: %u lost: %u", recv, lost); - if (lost && bwc->mcb) { + LOGGER_DEBUG(bwc->m->log, "recved: %u lost: %u percentage: %f %%", recv, lost, + (((float) lost / (recv + lost)) * 100.0f)); bwc->mcb(bwc, bwc->friend_number, ((float) lost / (recv + lost)), bwc->mcb_data); diff --git a/toxav/bwcontroller.h b/toxav/bwcontroller.h index 43475252df..be5eb19132 100644 --- a/toxav/bwcontroller.h +++ b/toxav/bwcontroller.h @@ -27,9 +27,9 @@ typedef struct BWController_s BWController; BWController *bwc_new(Messenger *m, uint32_t friendnumber, void (*mcb)(BWController *, uint32_t, float, void *), void *udata); + void bwc_kill(BWController *bwc); -void bwc_feed_avg(BWController *bwc, uint32_t bytes); void bwc_add_lost(BWController *bwc, uint32_t bytes); void bwc_add_recv(BWController *bwc, uint32_t bytes); diff --git a/toxav/rtp.c b/toxav/rtp.c index 06bc6df542..8daf89fa2b 100644 --- a/toxav/rtp.c +++ b/toxav/rtp.c @@ -33,11 +33,557 @@ #include #include +enum { + /** + * The number of milliseconds we want to keep a keyframe in the buffer for, + * even though there are no free slots for incoming frames. + */ + VIDEO_KEEP_KEYFRAME_IN_BUFFER_FOR_MS = 15, +}; + +// allocate_len is NOT including header! +static struct RTPMessage *new_message(const struct RTPHeader *header, size_t allocate_len, const uint8_t *data, + uint16_t data_length) +{ + assert(allocate_len >= data_length); + struct RTPMessage *msg = (struct RTPMessage *)calloc(sizeof(struct RTPMessage) + allocate_len, 1); + + if (msg == nullptr) { + return nullptr; + } + + msg->len = data_length; // result without header + msg->header = *header; + memcpy(msg->data, data, msg->len); + return msg; +} + +enum { + /** + * Instruct the caller to clear slot 0. + */ + GET_SLOT_RESULT_DROP_OLDEST_SLOT = -1, + /** + * Instruct the caller to drop the incoming packet. + */ + GET_SLOT_RESULT_DROP_INCOMING = -2, +}; + +/** + * Find the next free slot in work_buffer for the incoming data packet. + * + * - If the data packet belongs to a frame thats already in the work_buffer then + * use that slot. + * - If there is no free slot return GET_SLOT_RESULT_DROP_OLDEST_SLOT. + * - If the data packet is too old return GET_SLOT_RESULT_DROP_INCOMING. + * + * If there is a keyframe beeing assembled in slot 0, keep it a bit longer and + * do not kick it out right away if all slots are full instead kick out the new + * incoming interframe. + */ +static int8_t get_slot(Logger *log, struct RTPWorkBufferList *wkbl, bool is_keyframe, + const struct RTPHeader *header, bool is_multipart) +{ + if (is_multipart) { + // This RTP message is part of a multipart frame, so we try to find an + // existing slot with the previous parts of the frame in it. + for (uint8_t i = 0; i < wkbl->next_free_entry; i++) { + const struct RTPWorkBuffer *slot = &wkbl->work_buffer[i]; + + if ((slot->buf->header.sequnum == header->sequnum) && (slot->buf->header.timestamp == header->timestamp)) { + // Sequence number and timestamp match, so this slot belongs to + // the same frame. + // + // In reality, these will almost certainly either both match or + // both not match. Only if somehow there were 65535 frames + // between, the timestamp will matter. + return i; + } + } + } + + // The message may or may not be part of a multipart frame. + // + // If it is part of a multipart frame, then this is an entirely new frame + // for which we did not have a slot *or* the frame is so old that its slot + // has been evicted by now. + // + // |----------- time -----------> + // _________________ + // slot 0 | | + // ----------------- + // _________________ + // slot 1 | | + // ----------------- + // ____________ + // slot 2 | | -> frame too old, drop + // ------------ + // + // + // + // |----------- time -----------> + // _________________ + // slot 0 | | + // ----------------- + // _________________ + // slot 1 | | + // ----------------- + // ____________ + // slot 2 | | -> ok, start filling in a new slot + // ------------ + + // If there is a free slot: + if (wkbl->next_free_entry < USED_RTP_WORKBUFFER_COUNT) { + // If there is at least one filled slot: + if (wkbl->next_free_entry > 0) { + // Get the most recently filled slot. + const struct RTPWorkBuffer *slot = &wkbl->work_buffer[wkbl->next_free_entry - 1]; + + // If the incoming packet is older than our newest slot, drop it. + // This is the first situation in the above diagram. + if (slot->buf->header.timestamp > header->timestamp) { + LOGGER_DEBUG(log, "workbuffer:2:timestamp too old"); + return GET_SLOT_RESULT_DROP_INCOMING; + } + } + + // Not all slots are filled, and the packet is newer than our most + // recent slot, so it's a new frame we want to start assembling. This is + // the second situation in the above diagram. + return wkbl->next_free_entry; + } + + // If the incoming frame is a key frame, then stop assembling the oldest + // slot, regardless of whether there was a keyframe in that or not. + if (is_keyframe) { + return GET_SLOT_RESULT_DROP_OLDEST_SLOT; + } + + // The incoming slot is not a key frame, so we look at slot 0 to see what to + // do next. + const struct RTPWorkBuffer *slot = &wkbl->work_buffer[0]; + + // The incoming frame is not a key frame, but the existing slot 0 is also + // not a keyframe, so we stop assembling the existing frame and make space + // for the new one. + if (!slot->is_keyframe) { + return GET_SLOT_RESULT_DROP_OLDEST_SLOT; + } + + // If this key frame is fully received, we also stop assembling and clear + // slot 0. This also means sending the frame to the decoder. + if (slot->received_len == slot->buf->header.data_length_full) { + return GET_SLOT_RESULT_DROP_OLDEST_SLOT; + } + + // This is a key frame, not fully received yet, but it's already much older + // than the incoming frame, so we stop assembling it and send whatever part + // we did receive to the decoder. + if (slot->buf->header.timestamp + VIDEO_KEEP_KEYFRAME_IN_BUFFER_FOR_MS <= header->timestamp) { + return GET_SLOT_RESULT_DROP_OLDEST_SLOT; + } + + // This is a key frame, it's not too old yet, so we keep it in its slot for + // a little longer. + LOGGER_INFO(log, "keep KEYFRAME in workbuffer"); + return GET_SLOT_RESULT_DROP_INCOMING; +} + +/** + * Returns an assembled frame (as much data as we currently have for this frame, + * some pieces may be missing) + * + * If there are no frames ready, we return NULL. If this function returns + * non-NULL, it transfers ownership of the message to the caller, i.e. the + * caller is responsible for storing it elsewhere or calling free(). + */ +static struct RTPMessage *process_frame(Logger *log, struct RTPWorkBufferList *wkbl, uint8_t slot_id) +{ + assert(wkbl->next_free_entry >= 0); + + if (wkbl->next_free_entry == 0) { + // There are no frames in any slot. + return nullptr; + } + + // Slot 0 contains a key frame, slot_id points at an interframe that is + // relative to that key frame, so we don't use it yet. + if (wkbl->work_buffer[0].is_keyframe && slot_id != 0) { + LOGGER_DEBUG(log, "process_frame:KEYFRAME waiting in slot 0"); + return nullptr; + } + + // Either slot_id is 0 and slot 0 is a key frame, or there is no key frame + // in slot 0 (and slot_id is anything). + struct RTPWorkBuffer *const slot = &wkbl->work_buffer[slot_id]; + + // Move ownership of the frame out of the slot into m_new. + struct RTPMessage *const m_new = slot->buf; + slot->buf = nullptr; + + assert(wkbl->next_free_entry >= 1); + + if (slot_id != wkbl->next_free_entry - 1) { + // The slot is not the last slot, so we created a gap. We move all the + // entries after it one step up. + for (uint8_t i = slot_id; i < wkbl->next_free_entry - 1; i++) { + // Move entry (i+1) into entry (i). + wkbl->work_buffer[i] = wkbl->work_buffer[i + 1]; + } + } + + // We now have a free entry at the end of the array. + wkbl->next_free_entry--; + + // Clear the newly freed entry. + const struct RTPWorkBuffer empty = {0}; + wkbl->work_buffer[wkbl->next_free_entry] = empty; + + // Move ownership of the frame to the caller. + return m_new; +} + +/** + * @param log A logger. + * @param wkbl The list of in-progress frames, i.e. all the slots. + * @param slot_id The slot we want to fill the data into. + * @param is_keyframe Whether the data is part of a key frame. + * @param header The RTP header from the incoming packet. + * @param incoming_data The pure payload without header. + * @param incoming_data_length The length in bytes of the incoming data payload. + */ +static bool fill_data_into_slot(Logger *log, struct RTPWorkBufferList *wkbl, const uint8_t slot_id, bool is_keyframe, + const struct RTPHeader *header, const uint8_t *incoming_data, uint16_t incoming_data_length) +{ + // We're either filling the data into an existing slot, or in a new one that + // is the next free entry. + assert(slot_id <= wkbl->next_free_entry); + struct RTPWorkBuffer *const slot = &wkbl->work_buffer[slot_id]; + + assert(header != nullptr); + assert(is_keyframe == (bool)(header->flags & RTP_KEY_FRAME)); + + if (slot->received_len == 0) { + assert(slot->buf == nullptr); + + // No data for this slot has been received, yet, so we create a new + // message for it with enough memory for the entire frame. + struct RTPMessage *msg = (struct RTPMessage *)calloc(1, sizeof(struct RTPMessage) + header->data_length_full); + + if (msg == nullptr) { + LOGGER_ERROR(log, "Out of memory while trying to allocate for frame of size %u\n", + (unsigned)header->data_length_full); + // Out of memory: throw away the incoming data. + return false; + } + + // Unused in the new video receiving code, as it's 16 bit and can't hold + // the full length of large frames. Instead, we use slot->received_len. + msg->len = 0; + msg->header = *header; + + slot->buf = msg; + slot->is_keyframe = is_keyframe; + slot->received_len = 0; + + assert(wkbl->next_free_entry < USED_RTP_WORKBUFFER_COUNT); + wkbl->next_free_entry++; + } + + // We already checked this when we received the packet, but we rely on it + // here, so assert again. + assert(header->offset_full < header->data_length_full); + + // Copy the incoming chunk of data into the correct position in the full + // frame data array. + memcpy( + (slot->buf->data + header->offset_full), + incoming_data, + incoming_data_length + ); + + // Update the total received length of this slot. + slot->received_len += incoming_data_length; + + // Update received length also in the header of the message, for later use. + slot->buf->header.received_length_full = slot->received_len; + + return (slot->received_len == header->data_length_full); +} + +static void update_bwc_values(Logger *log, RTPSession *session, const struct RTPMessage *msg) +{ + if (session->first_packets_counter < DISMISS_FIRST_LOST_VIDEO_PACKET_COUNT) { + session->first_packets_counter++; + } else { + uint32_t data_length_full = msg->header.data_length_full; // without header + uint32_t received_length_full = msg->header.received_length_full; // without header + bwc_add_recv(session->bwc, data_length_full); + + if (received_length_full < data_length_full) { + LOGGER_DEBUG(log, "BWC: full length=%u received length=%d", data_length_full, received_length_full); + bwc_add_lost(session->bwc, (data_length_full - received_length_full)); + } + } +} + +/** + * Handle a single RTP video packet. + * + * The packet may or may not be part of a multipart frame. This function will + * find out and handle it appropriately. + * + * @param session The current RTP session with: + * session->mcb == vc_queue_message() // this function is called from here + * session->mp == struct RTPMessage * + * session->cs == call->video.second // == VCSession created by vc_new() call + * @param header The RTP header deserialised from the packet. + * @param incoming_data The packet data *not* header, i.e. this is the actual + * payload. + * @param incoming_data_length The packet length *not* including header, i.e. + * this is the actual payload length. + * @param log A logger. + * + * @return -1 on error, 0 on success. + */ +static int handle_video_packet(RTPSession *session, const struct RTPHeader *header, + const uint8_t *incoming_data, uint16_t incoming_data_length, Logger *log) +{ + // Full frame length in bytes. The frame may be split into multiple packets, + // but this value is the complete assembled frame size. + const uint32_t full_frame_length = header->data_length_full; + + // Current offset in the frame. If this is the first packet of a multipart + // frame or it's not a multipart frame, then this value is 0. + const uint32_t offset = header->offset_full; // without header + + // The sender tells us whether this is a key frame. + const bool is_keyframe = (header->flags & RTP_KEY_FRAME) != 0; + + LOGGER_DEBUG(log, "-- handle_video_packet -- full lens=%u len=%u offset=%u is_keyframe=%s", + (unsigned)incoming_data_length, (unsigned)full_frame_length, (unsigned)offset, is_keyframe ? "K" : "."); + LOGGER_DEBUG(log, "wkbl->next_free_entry:003=%d", session->work_buffer_list->next_free_entry); + + const bool is_multipart = (full_frame_length != incoming_data_length); + + /* The message was sent in single part */ + int8_t slot_id = get_slot(log, session->work_buffer_list, is_keyframe, header, is_multipart); + LOGGER_DEBUG(log, "slot num=%d", slot_id); + + // get_slot told us to drop the packet, so we ignore it. + if (slot_id == GET_SLOT_RESULT_DROP_INCOMING) { + return -1; + } + + // get_slot said there is no free slot. + if (slot_id == GET_SLOT_RESULT_DROP_OLDEST_SLOT) { + LOGGER_DEBUG(log, "there was no free slot, so we process the oldest frame"); + // We now own the frame. + struct RTPMessage *m_new = process_frame(log, session->work_buffer_list, 0); + + // The process_frame function returns NULL if there is no slot 0, i.e. + // the work buffer list is completely empty. It can't be empty, because + // get_slot just told us it's full, so process_frame must return non-null. + assert(m_new != nullptr); + + LOGGER_DEBUG(log, "-- handle_video_packet -- CALLBACK-001a b0=%d b1=%d", (int)m_new->data[0], (int)m_new->data[1]); + update_bwc_values(log, session, m_new); + // Pass ownership of m_new to the callback. + session->mcb(session->cs, m_new); + // Now we no longer own m_new. + m_new = nullptr; + + // Now we must have a free slot, so we either get that slot, i.e. >= 0, + // or get told to drop the incoming packet if it's too old. + slot_id = get_slot(log, session->work_buffer_list, is_keyframe, header, /* is_multipart */false); + + if (slot_id == GET_SLOT_RESULT_DROP_INCOMING) { + // The incoming frame is too old, so we drop it. + return -1; + } + } + + // We must have a valid slot here. + assert(slot_id >= 0); + + LOGGER_DEBUG(log, "fill_data_into_slot.1"); + + // fill in this part into the slot buffer at the correct offset + if (!fill_data_into_slot( + log, + session->work_buffer_list, + slot_id, + is_keyframe, + header, + incoming_data, + incoming_data_length)) { + // Memory allocation failed. Return error. + return -1; + } + + struct RTPMessage *m_new = process_frame(log, session->work_buffer_list, slot_id); + + if (m_new) { + LOGGER_DEBUG(log, "-- handle_video_packet -- CALLBACK-003a b0=%d b1=%d", (int)m_new->data[0], (int)m_new->data[1]); + update_bwc_values(log, session, m_new); + session->mcb(session->cs, m_new); + + m_new = nullptr; + } + + return 0; +} + +/** + * @return -1 on error, 0 on success. + */ +static int handle_rtp_packet(Messenger *m, uint32_t friendnumber, const uint8_t *data, uint16_t length, void *object) +{ + RTPSession *session = (RTPSession *)object; + + if (!session || length < RTP_HEADER_SIZE + 1) { + LOGGER_WARNING(m->log, "No session or invalid length of received buffer!"); + return -1; + } + + // Get the packet type. + const uint8_t packet_type = data[0]; + ++data; + --length; + + // Unpack the header. + struct RTPHeader header; + rtp_header_unpack(data, &header); + + if (header.pt != packet_type % 128) { + LOGGER_WARNING(m->log, "RTPHeader packet type and Tox protocol packet type did not agree: %d != %d", + header.pt, packet_type % 128); + return -1; + } + + if (header.pt != session->payload_type % 128) { + LOGGER_WARNING(m->log, "RTPHeader packet type does not match this session's payload type: %d != %d", + header.pt, session->payload_type % 128); + return -1; + } + + if (header.offset_full >= header.data_length_full) { + LOGGER_ERROR(m->log, "Invalid video packet: frame offset (%u) >= full frame length (%u)", + (unsigned)header.offset_full, (unsigned)header.data_length_full); + return -1; + } + + if (header.offset_lower >= header.data_length_lower) { + LOGGER_ERROR(m->log, "Invalid old protocol video packet: frame offset (%u) >= full frame length (%u)", + (unsigned)header.offset_lower, (unsigned)header.data_length_lower); + return -1; + } + + LOGGER_DEBUG(m->log, "header.pt %d, video %d", (uint8_t)header.pt, (rtp_TypeVideo % 128)); + + // The sender uses the new large-frame capable protocol and is sending a + // video packet. + if ((header.flags & RTP_LARGE_FRAME) && header.pt == (rtp_TypeVideo % 128)) { + return handle_video_packet(session, &header, data + RTP_HEADER_SIZE, length - RTP_HEADER_SIZE, m->log); + } + + // everything below here is for the old 16 bit protocol ------------------ + + if (header.data_length_lower == length - RTP_HEADER_SIZE) { + /* The message is sent in single part */ + + /* Message is not late; pick up the latest parameters */ + session->rsequnum = header.sequnum; + session->rtimestamp = header.timestamp; + bwc_add_recv(session->bwc, length); + + /* Invoke processing of active multiparted message */ + if (session->mp) { + session->mcb(session->cs, session->mp); + session->mp = nullptr; + } + + /* The message came in the allowed time; + */ + + return session->mcb(session->cs, new_message(&header, length - RTP_HEADER_SIZE, data + RTP_HEADER_SIZE, + length - RTP_HEADER_SIZE)); + } + + /* The message is sent in multiple parts */ + + if (session->mp) { + /* There are 2 possible situations in this case: + * 1) being that we got the part of already processing message. + * 2) being that we got the part of a new/old message. + * + * We handle them differently as we only allow a single multiparted + * processing message + */ + if (session->mp->header.sequnum == header.sequnum && + session->mp->header.timestamp == header.timestamp) { + /* First case */ + + /* Make sure we have enough allocated memory */ + if (session->mp->header.data_length_lower - session->mp->len < length - RTP_HEADER_SIZE || + session->mp->header.data_length_lower <= header.offset_lower) { + /* There happened to be some corruption on the stream; + * continue wihtout this part + */ + return 0; + } + + memcpy(session->mp->data + header.offset_lower, data + RTP_HEADER_SIZE, + length - RTP_HEADER_SIZE); + session->mp->len += length - RTP_HEADER_SIZE; + bwc_add_recv(session->bwc, length); + + if (session->mp->len == session->mp->header.data_length_lower) { + /* Received a full message; now push it for the further + * processing. + */ + session->mcb(session->cs, session->mp); + session->mp = nullptr; + } + } else { + /* Second case */ + if (session->mp->header.timestamp > header.timestamp) { + /* The received message part is from the old message; + * discard it. + */ + return 0; + } + + /* Push the previous message for processing */ + session->mcb(session->cs, session->mp); + + session->mp = nullptr; + goto NEW_MULTIPARTED; + } + } else { + /* In this case threat the message as if it was received in order + */ + /* This is also a point for new multiparted messages */ +NEW_MULTIPARTED: + + /* Message is not late; pick up the latest parameters */ + session->rsequnum = header.sequnum; + session->rtimestamp = header.timestamp; + bwc_add_recv(session->bwc, length); + + /* Store message. + */ + session->mp = new_message(&header, header.data_length_lower, data + RTP_HEADER_SIZE, length - RTP_HEADER_SIZE); + memmove(session->mp->data + header.offset_lower, session->mp->data, session->mp->len); + } + + return 0; +} size_t rtp_header_pack(uint8_t *const rdata, const struct RTPHeader *header) { uint8_t *p = rdata; - *p++ = (header->protocol_version & 3) << 6 + *p++ = (header->ve & 3) << 6 | (header->pe & 1) << 5 | (header->xe & 1) << 4 | (header->cc & 0xf); @@ -62,11 +608,10 @@ size_t rtp_header_pack(uint8_t *const rdata, const struct RTPHeader *header) return p - rdata; } - size_t rtp_header_unpack(const uint8_t *data, struct RTPHeader *header) { const uint8_t *p = data; - header->protocol_version = (*p >> 6) & 3; + header->ve = (*p >> 6) & 3; header->pe = (*p >> 5) & 1; header->xe = (*p >> 4) & 1; header->cc = *p & 0xf; @@ -93,44 +638,56 @@ size_t rtp_header_unpack(const uint8_t *data, struct RTPHeader *header) } -int handle_rtp_packet(Messenger *m, uint32_t friendnumber, const uint8_t *data, uint16_t length, void *object); - - RTPSession *rtp_new(int payload_type, Messenger *m, uint32_t friendnumber, BWController *bwc, void *cs, int (*mcb)(void *, struct RTPMessage *)) { - assert(mcb); - assert(cs); - assert(m); + assert(mcb != nullptr); + assert(cs != nullptr); + assert(m != nullptr); - RTPSession *retu = (RTPSession *)calloc(1, sizeof(RTPSession)); + RTPSession *session = (RTPSession *)calloc(1, sizeof(RTPSession)); - if (!retu) { + if (!session) { LOGGER_WARNING(m->log, "Alloc failed! Program might misbehave!"); return nullptr; } - retu->ssrc = random_u32(); - retu->payload_type = payload_type; + session->work_buffer_list = (struct RTPWorkBufferList *)calloc(1, sizeof(struct RTPWorkBufferList)); - retu->m = m; - retu->friend_number = friendnumber; + if (session->work_buffer_list == nullptr) { + LOGGER_ERROR(m->log, "out of memory while allocating work buffer list"); + free(session); + return nullptr; + } - /* Also set payload type as prefix */ + // First entry is free. + session->work_buffer_list->next_free_entry = 0; + + session->ssrc = payload_type == rtp_TypeVideo ? 0 : random_u32(); + session->payload_type = payload_type; + session->m = m; + session->friend_number = friendnumber; - retu->bwc = bwc; - retu->cs = cs; - retu->mcb = mcb; + // set NULL just in case + session->mp = nullptr; + session->first_packets_counter = 1; - if (-1 == rtp_allow_receiving(retu)) { + /* Also set payload type as prefix */ + session->bwc = bwc; + session->cs = cs; + session->mcb = mcb; + + if (-1 == rtp_allow_receiving(session)) { LOGGER_WARNING(m->log, "Failed to start rtp receiving mode"); - free(retu); + free(session->work_buffer_list); + free(session); return nullptr; } - return retu; + return session; } + void rtp_kill(RTPSession *session) { if (!session) { @@ -138,10 +695,15 @@ void rtp_kill(RTPSession *session) } LOGGER_DEBUG(session->m->log, "Terminated RTP session: %p", session); - rtp_stop_receiving(session); + + LOGGER_DEBUG(session->m->log, "Terminated RTP session V3 work_buffer_list->next_free_entry: %d", + (int)session->work_buffer_list->next_free_entry); + + free(session->work_buffer_list); free(session); } + int rtp_allow_receiving(RTPSession *session) { if (session == nullptr) { @@ -157,6 +719,7 @@ int rtp_allow_receiving(RTPSession *session) LOGGER_DEBUG(session->m->log, "Started receiving on session: %p", session); return 0; } + int rtp_stop_receiving(RTPSession *session) { if (session == nullptr) { @@ -168,42 +731,76 @@ int rtp_stop_receiving(RTPSession *session) LOGGER_DEBUG(session->m->log, "Stopped receiving on session: %p", session); return 0; } -int rtp_send_data(RTPSession *session, const uint8_t *data, uint16_t length, Logger *log) + +/** + * @param input is raw vpx data. + * @param length is the length of the raw data. + */ +int rtp_send_data(RTPSession *session, const uint8_t *data, uint32_t length, + bool is_keyframe, Logger *log) { if (!session) { LOGGER_ERROR(log, "No session!"); return -1; } - VLA(uint8_t, rdata, length + RTP_HEADER_SIZE + 1); - memset(rdata, 0, SIZEOF_VLA(rdata)); + uint8_t is_video_payload = 0; - rdata[0] = session->payload_type; + if (session->payload_type == rtp_TypeVideo) { + is_video_payload = 1; + } struct RTPHeader header = {0}; - header.protocol_version = 2; + header.ve = 2; // this is unused in toxav + header.pe = 0; + header.xe = 0; + header.cc = 0; header.ma = 0; + header.pt = session->payload_type % 128; header.sequnum = session->sequnum; + header.timestamp = current_time_monotonic(); + header.ssrc = session->ssrc; header.offset_lower = 0; + + // here the highest bits gets stripped anyway, no need to do keyframe bit magic here! header.data_length_lower = length; - if (MAX_CRYPTO_DATA_SIZE > length + RTP_HEADER_SIZE + 1) { + header.flags = RTP_LARGE_FRAME; + uint16_t length_safe = (uint16_t)length; + + if (length > UINT16_MAX) { + length_safe = UINT16_MAX; + } + + header.data_length_lower = length_safe; + header.data_length_full = length; // without header + header.offset_lower = 0; + header.offset_full = 0; + + if (is_keyframe) { + header.flags |= RTP_KEY_FRAME; + } + + VLA(uint8_t, rdata, length + RTP_HEADER_SIZE + 1); + memset(rdata, 0, SIZEOF_VLA(rdata)); + rdata[0] = session->payload_type; // packet id == payload_type + + if (MAX_CRYPTO_DATA_SIZE > (length + RTP_HEADER_SIZE + 1)) { /** * The length is lesser than the maximum allowed length (including header) * Send the packet in single piece. */ - rtp_header_pack(rdata + 1, &header); memcpy(rdata + 1 + RTP_HEADER_SIZE, data, length); @@ -211,13 +808,11 @@ int rtp_send_data(RTPSession *session, const uint8_t *data, uint16_t length, Log LOGGER_WARNING(session->m->log, "RTP send failed (len: %d)! std error: %s", SIZEOF_VLA(rdata), strerror(errno)); } } else { - /** * The length is greater than the maximum allowed length (including header) * Send the packet in multiple pieces. */ - - uint16_t sent = 0; + uint32_t sent = 0; uint16_t piece = MAX_CRYPTO_DATA_SIZE - (RTP_HEADER_SIZE + 1); while ((length - sent) + RTP_HEADER_SIZE + 1 > MAX_CRYPTO_DATA_SIZE) { @@ -232,6 +827,7 @@ int rtp_send_data(RTPSession *session, const uint8_t *data, uint16_t length, Log sent += piece; header.offset_lower = sent; + header.offset_full = sent; // raw data offset, without any header } /* Send remaining */ @@ -252,215 +848,3 @@ int rtp_send_data(RTPSession *session, const uint8_t *data, uint16_t length, Log session->sequnum ++; return 0; } - - -static bool chloss(const RTPSession *session, const struct RTPHeader *header) -{ - if (header->timestamp < session->rtimestamp) { - uint16_t hosq, lost = 0; - - hosq = header->sequnum; - - lost = (hosq > session->rsequnum) ? - (session->rsequnum + 65535) - hosq : - session->rsequnum - hosq; - - fprintf(stderr, "Lost packet\n"); - - while (lost --) { - bwc_add_lost(session->bwc, 0); - } - - return true; - } - - return false; -} -static struct RTPMessage *new_message(size_t allocate_len, const uint8_t *data, uint16_t data_length) -{ - assert(allocate_len >= data_length); - - struct RTPMessage *msg = (struct RTPMessage *)calloc(sizeof(struct RTPMessage) + - (allocate_len - RTP_HEADER_SIZE), 1); - - if (msg == nullptr) { - return nullptr; - } - - msg->len = data_length - RTP_HEADER_SIZE; - rtp_header_unpack(data, &msg->header); - memcpy(msg->data, data + RTP_HEADER_SIZE, allocate_len - RTP_HEADER_SIZE); - - return msg; -} - -int handle_rtp_packet(Messenger *m, uint32_t friendnumber, const uint8_t *data, uint16_t length, void *object) -{ - (void) m; - (void) friendnumber; - - RTPSession *session = (RTPSession *)object; - - data ++; - length--; - - if (!session || length < RTP_HEADER_SIZE) { - LOGGER_WARNING(m->log, "No session or invalid length of received buffer!"); - return -1; - } - - struct RTPHeader header; - - rtp_header_unpack(data, &header); - - if (header.pt != session->payload_type % 128) { - LOGGER_WARNING(m->log, "Invalid payload type with the session"); - return -1; - } - - if (header.offset_lower >= header.data_length_lower) { - /* Never allow this case to happen */ - return -1; - } - - bwc_feed_avg(session->bwc, length); - - if (header.data_length_lower == length - RTP_HEADER_SIZE) { - /* The message is sent in single part */ - - /* Only allow messages which have arrived in order; - * drop late messages - */ - if (chloss(session, &header)) { - return 0; - } - - /* Message is not late; pick up the latest parameters */ - session->rsequnum = header.sequnum; - session->rtimestamp = header.timestamp; - - bwc_add_recv(session->bwc, length); - - /* Invoke processing of active multiparted message */ - if (session->mp) { - if (session->mcb) { - session->mcb(session->cs, session->mp); - } else { - free(session->mp); - } - - session->mp = nullptr; - } - - /* The message came in the allowed time; - * process it only if handler for the session is present. - */ - - if (!session->mcb) { - return 0; - } - - return session->mcb(session->cs, new_message(length, data, length)); - } - - /* The message is sent in multiple parts */ - - if (session->mp) { - /* There are 2 possible situations in this case: - * 1) being that we got the part of already processing message. - * 2) being that we got the part of a new/old message. - * - * We handle them differently as we only allow a single multiparted - * processing message - */ - - if (session->mp->header.sequnum == header.sequnum && - session->mp->header.timestamp == header.timestamp) { - /* First case */ - - /* Make sure we have enough allocated memory */ - if (session->mp->header.data_length_lower - session->mp->len < length - RTP_HEADER_SIZE || - session->mp->header.data_length_lower <= header.offset_lower) { - /* There happened to be some corruption on the stream; - * continue wihtout this part - */ - return 0; - } - - memcpy(session->mp->data + header.offset_lower, data + RTP_HEADER_SIZE, - length - RTP_HEADER_SIZE); - - session->mp->len += length - RTP_HEADER_SIZE; - - bwc_add_recv(session->bwc, length); - - if (session->mp->len == session->mp->header.data_length_lower) { - /* Received a full message; now push it for the further - * processing. - */ - if (session->mcb) { - session->mcb(session->cs, session->mp); - } else { - free(session->mp); - } - - session->mp = nullptr; - } - } else { - /* Second case */ - - if (session->mp->header.timestamp > header.timestamp) { - /* The received message part is from the old message; - * discard it. - */ - return 0; - } - - /* Measure missing parts of the old message */ - bwc_add_lost(session->bwc, - (session->mp->header.data_length_lower - session->mp->len) + - - /* Must account sizes of rtp headers too */ - ((session->mp->header.data_length_lower - session->mp->len) / - MAX_CRYPTO_DATA_SIZE) * RTP_HEADER_SIZE); - - /* Push the previous message for processing */ - if (session->mcb) { - session->mcb(session->cs, session->mp); - } else { - free(session->mp); - } - - session->mp = nullptr; - goto NEW_MULTIPARTED; - } - } else { - /* In this case threat the message as if it was received in order - */ - - /* This is also a point for new multiparted messages */ -NEW_MULTIPARTED: - - /* Only allow messages which have arrived in order; - * drop late messages - */ - if (chloss(session, &header)) { - return 0; - } - - /* Message is not late; pick up the latest parameters */ - session->rsequnum = header.sequnum; - session->rtimestamp = header.timestamp; - - bwc_add_recv(session->bwc, length); - - /* Again, only store message if handler is present - */ - if (session->mcb) { - session->mp = new_message(header.data_length_lower + RTP_HEADER_SIZE, data, length); - memmove(session->mp->data + header.offset_lower, session->mp->data, session->mp->len); - } - } - - return 0; -} diff --git a/toxav/rtp.h b/toxav/rtp.h index c8af08d716..a310d58ae4 100644 --- a/toxav/rtp.h +++ b/toxav/rtp.h @@ -66,9 +66,10 @@ enum RTPFlags { RTP_KEY_FRAME = 1 << 1, }; + struct RTPHeader { /* Standard RTP header */ - unsigned protocol_version: 2; /* Version has only 2 bits! */ + unsigned ve: 2; /* Version has only 2 bits! */ unsigned pe: 1; /* Padding */ unsigned xe: 1; /* Extra header */ unsigned cc: 4; /* Contributing sources count */ @@ -113,33 +114,71 @@ struct RTPHeader { uint16_t data_length_lower; }; + struct RTPMessage { + /** + * This is used in the old code that doesn't deal with large frames, i.e. + * the audio code or receiving code for old 16 bit messages. We use it to + * record the number of bytes received so far in a multi-part message. The + * multi-part message in the old code is stored in \ref RTPSession::mp. + */ uint16_t len; struct RTPHeader header; uint8_t data[]; }; +#define USED_RTP_WORKBUFFER_COUNT 3 + +/** + * One slot in the work buffer list. Represents one frame that is currently + * being assembled. + */ +struct RTPWorkBuffer { + /** + * Whether this slot contains a key frame. This is true iff + * buf->header.flags & RTP_KEY_FRAME. + */ + bool is_keyframe; + /** + * The number of bytes received so far, regardless of which pieces. I.e. we + * could have received the first 1000 bytes and the last 1000 bytes with + * 4000 bytes in the middle still to come, and this number would be 2000. + */ + uint32_t received_len; + /** + * The message currently being assembled. + */ + struct RTPMessage *buf; +}; + +struct RTPWorkBufferList { + int8_t next_free_entry; + struct RTPWorkBuffer work_buffer[USED_RTP_WORKBUFFER_COUNT]; +}; + +#define DISMISS_FIRST_LOST_VIDEO_PACKET_COUNT 10 + /** * RTP control session. */ -typedef struct { +typedef struct RTPSession { uint8_t payload_type; uint16_t sequnum; /* Sending sequence number */ uint16_t rsequnum; /* Receiving sequence number */ uint32_t rtimestamp; - uint32_t ssrc; - + uint32_t ssrc; // this seems to be unused!? struct RTPMessage *mp; /* Expected parted message */ - + struct RTPWorkBufferList *work_buffer_list; + uint8_t first_packets_counter; /* dismiss first few lost video packets */ Messenger *m; uint32_t friend_number; - BWController *bwc; void *cs; int (*mcb)(void *, struct RTPMessage *msg); } RTPSession; + /** * Serialise an RTPHeader to bytes to be sent over the network. * @@ -164,7 +203,17 @@ RTPSession *rtp_new(int payload_type, Messenger *m, uint32_t friendnumber, void rtp_kill(RTPSession *session); int rtp_allow_receiving(RTPSession *session); int rtp_stop_receiving(RTPSession *session); -int rtp_send_data(RTPSession *session, const uint8_t *data, uint16_t length, Logger *log); +/** + * Send a frame of audio or video data, chunked in \ref RTPMessage instances. + * + * @param session The A/V session to send the data for. + * @param data A byte array of length \p length. + * @param length The number of bytes to send from @p data. + * @param is_keyframe Whether this video frame is a key frame. If it is an + * audio frame, this parameter is ignored. + */ +int rtp_send_data(RTPSession *session, const uint8_t *data, uint32_t length, + bool is_keyframe, Logger *log); #ifdef __cplusplus } // extern "C" diff --git a/toxav/rtp_test.cpp b/toxav/rtp_test.cpp index 60534b071e..d6717a28c4 100644 --- a/toxav/rtp_test.cpp +++ b/toxav/rtp_test.cpp @@ -38,7 +38,7 @@ TEST(Rtp, Deserialisation) RTPHeader unpacked = {0}; EXPECT_EQ(rtp_header_unpack(rdata, &unpacked), RTP_HEADER_SIZE); - EXPECT_EQ(header.protocol_version, unpacked.protocol_version); + EXPECT_EQ(header.ve, unpacked.ve); EXPECT_EQ(header.pe, unpacked.pe); EXPECT_EQ(header.xe, unpacked.xe); EXPECT_EQ(header.cc, unpacked.cc); diff --git a/toxav/toxav.c b/toxav/toxav.c index c48fc19206..4e45b7475d 100644 --- a/toxav/toxav.c +++ b/toxav/toxav.c @@ -36,7 +36,18 @@ #include #include -#define MAX_ENCODE_TIME_US ((1000 / 24) * 1000) +// TODO: don't hardcode this, let the application choose it +// VPX Info: Time to spend encoding, in microseconds (it's a *soft* deadline) +#define WANTED_MAX_ENCODER_FPS (40) +#define MAX_ENCODE_TIME_US (1000000 / WANTED_MAX_ENCODER_FPS) // to allow x fps + +#define VIDEO_SEND_X_KEYFRAMES_FIRST 7 // force the first n frames to be keyframes! + +/* +VPX_DL_REALTIME (1) deadline parameter analogous to VPx REALTIME mode. +VPX_DL_GOOD_QUALITY (1000000) deadline parameter analogous to VPx GOOD QUALITY mode. +VPX_DL_BEST_QUALITY (0) deadline parameter analogous to VPx BEST QUALITY mode. +*/ typedef struct ToxAVCall_s { ToxAV *av; @@ -752,13 +763,12 @@ bool toxav_audio_send_frame(ToxAV *av, uint32_t friend_number, const int16_t *pc goto END; } - if (rtp_send_data(call->audio.first, dest, vrc + sizeof(sampling_rate), av->m->log) != 0) { + if (rtp_send_data(call->audio.first, dest, vrc + sizeof(sampling_rate), false, av->m->log) != 0) { LOGGER_WARNING(av->m->log, "Failed to send audio packet"); rc = TOXAV_ERR_SEND_FRAME_RTP_FAILED; } } - pthread_mutex_unlock(call->mutex_audio); END: @@ -769,12 +779,15 @@ bool toxav_audio_send_frame(ToxAV *av, uint32_t friend_number, const int16_t *pc return rc == TOXAV_ERR_SEND_FRAME_OK; } + bool toxav_video_send_frame(ToxAV *av, uint32_t friend_number, uint16_t width, uint16_t height, const uint8_t *y, const uint8_t *u, const uint8_t *v, TOXAV_ERR_SEND_FRAME *error) { TOXAV_ERR_SEND_FRAME rc = TOXAV_ERR_SEND_FRAME_OK; ToxAVCall *call; + int vpx_encode_flags = 0; + if (m_friend_exists(av->m, friend_number) == 0) { rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_FOUND; goto END; @@ -810,12 +823,28 @@ bool toxav_video_send_frame(ToxAV *av, uint32_t friend_number, uint16_t width, u goto END; } - if (vc_reconfigure_encoder(call->video.second, call->video_bit_rate * 1000, width, height) != 0) { + if (vc_reconfigure_encoder(call->video.second, call->video_bit_rate * 1000, width, height, -1) != 0) { pthread_mutex_unlock(call->mutex_video); rc = TOXAV_ERR_SEND_FRAME_INVALID; goto END; } + if (call->video.first->ssrc < VIDEO_SEND_X_KEYFRAMES_FIRST) { + // Key frame flag for first frames + vpx_encode_flags = VPX_EFLAG_FORCE_KF; + LOGGER_INFO(av->m->log, "I_FRAME_FLAG:%d only-i-frame mode", call->video.first->ssrc); + + call->video.first->ssrc++; + } else if (call->video.first->ssrc == VIDEO_SEND_X_KEYFRAMES_FIRST) { + // normal keyframe placement + vpx_encode_flags = 0; + LOGGER_INFO(av->m->log, "I_FRAME_FLAG:%d normal mode", call->video.first->ssrc); + + call->video.first->ssrc++; + } + + // we start with I-frames (full frames) and then switch to normal mode later + { /* Encode */ vpx_image_t img; img.w = img.h = img.d_w = img.d_h = 0; @@ -829,7 +858,7 @@ bool toxav_video_send_frame(ToxAV *av, uint32_t friend_number, uint16_t width, u memcpy(img.planes[VPX_PLANE_V], v, (width / 2) * (height / 2)); vpx_codec_err_t vrc = vpx_codec_encode(call->video.second->encoder, &img, - call->video.second->frame_counter, 1, 0, MAX_ENCODE_TIME_US); + call->video.second->frame_counter, 1, vpx_encode_flags, MAX_ENCODE_TIME_US); vpx_img_free(&img); @@ -847,22 +876,31 @@ bool toxav_video_send_frame(ToxAV *av, uint32_t friend_number, uint16_t width, u vpx_codec_iter_t iter = nullptr; const vpx_codec_cx_pkt_t *pkt; - while ((pkt = vpx_codec_get_cx_data(call->video.second->encoder, &iter))) { + while ((pkt = vpx_codec_get_cx_data(call->video.second->encoder, &iter)) != nullptr) { if (pkt->kind == VPX_CODEC_CX_FRAME_PKT) { - const uint8_t *buf = (const uint8_t *)pkt->data.frame.buf; - const uint8_t *end = buf + pkt->data.frame.sz; - - while (buf < end) { - uint16_t size = MIN(UINT16_MAX, end - buf); - - if (rtp_send_data(call->video.first, buf, size, av->m->log) < 0) { - pthread_mutex_unlock(call->mutex_video); - LOGGER_WARNING(av->m->log, "Could not send video frame: %s\n", strerror(errno)); - rc = TOXAV_ERR_SEND_FRAME_RTP_FAILED; - goto END; - } - - buf += size; + const bool is_keyframe = (pkt->data.frame.flags & VPX_FRAME_IS_KEY) != 0; + + // https://www.webmproject.org/docs/webm-sdk/structvpx__codec__cx__pkt.html + // pkt->data.frame.sz -> size_t + const uint32_t frame_length_in_bytes = pkt->data.frame.sz; + + const int res = rtp_send_data( + call->video.first, + (const uint8_t *)pkt->data.frame.buf, + frame_length_in_bytes, + is_keyframe, + av->m->log); + + LOGGER_DEBUG(av->m->log, "+ _sending_FRAME_TYPE_==%s bytes=%d frame_len=%d", is_keyframe ? "K" : ".", + (int)pkt->data.frame.sz, (int)frame_length_in_bytes); + LOGGER_DEBUG(av->m->log, "+ _sending_FRAME_ b0=%d b1=%d", ((const uint8_t *)pkt->data.frame.buf)[0], + ((const uint8_t *)pkt->data.frame.buf)[1]); + + if (res < 0) { + pthread_mutex_unlock(call->mutex_video); + LOGGER_WARNING(av->m->log, "Could not send video frame: %s", strerror(errno)); + rc = TOXAV_ERR_SEND_FRAME_RTP_FAILED; + goto END; } } } @@ -878,6 +916,7 @@ bool toxav_video_send_frame(ToxAV *av, uint32_t friend_number, uint16_t width, u return rc == TOXAV_ERR_SEND_FRAME_OK; } + void toxav_callback_audio_receive_frame(ToxAV *av, toxav_audio_receive_frame_cb *callback, void *user_data) { pthread_mutex_lock(av->mutex); @@ -885,6 +924,7 @@ void toxav_callback_audio_receive_frame(ToxAV *av, toxav_audio_receive_frame_cb av->acb.second = user_data; pthread_mutex_unlock(av->mutex); } + void toxav_callback_video_receive_frame(ToxAV *av, toxav_video_receive_frame_cb *callback, void *user_data) { pthread_mutex_lock(av->mutex); @@ -893,7 +933,6 @@ void toxav_callback_video_receive_frame(ToxAV *av, toxav_video_receive_frame_cb pthread_mutex_unlock(av->mutex); } - /******************************************************************************* * * :: Internal @@ -913,7 +952,8 @@ void callback_bwc(BWController *bwc, uint32_t friend_number, float loss, void *u LOGGER_DEBUG(call->av->m->log, "Reported loss of %f%%", loss * 100); - if (loss < .01f) { + /* if less than 10% data loss we do nothing! */ + if (loss < 0.1f) { return; } @@ -1079,6 +1119,7 @@ bool invoke_call_state_callback(ToxAV *av, uint32_t friend_number, uint32_t stat return true; } + ToxAVCall *call_new(ToxAV *av, uint32_t friend_number, TOXAV_ERR_CALL *error) { /* Assumes mutex locked */ @@ -1100,7 +1141,6 @@ ToxAVCall *call_new(ToxAV *av, uint32_t friend_number, TOXAV_ERR_CALL *error) goto END; } - call = (ToxAVCall *)calloc(sizeof(ToxAVCall), 1); if (call == nullptr) { @@ -1161,6 +1201,7 @@ ToxAVCall *call_new(ToxAV *av, uint32_t friend_number, TOXAV_ERR_CALL *error) return call; } + ToxAVCall *call_get(ToxAV *av, uint32_t friend_number) { /* Assumes mutex locked */ @@ -1170,6 +1211,7 @@ ToxAVCall *call_get(ToxAV *av, uint32_t friend_number) return av->calls[friend_number]; } + ToxAVCall *call_remove(ToxAVCall *call) { if (call == nullptr) { @@ -1217,6 +1259,7 @@ ToxAVCall *call_remove(ToxAVCall *call) return nullptr; } + bool call_prepare_transmission(ToxAVCall *call) { /* Assumes mutex locked */ @@ -1305,6 +1348,7 @@ bool call_prepare_transmission(ToxAVCall *call) pthread_mutex_destroy(call->mutex_audio); return false; } + void call_kill_transmission(ToxAVCall *call) { if (call == nullptr || call->active == 0) { diff --git a/toxav/video.c b/toxav/video.c index eee542a2ce..0014dbb67e 100644 --- a/toxav/video.c +++ b/toxav/video.c @@ -33,8 +33,124 @@ #include #include -#define MAX_DECODE_TIME_US 0 /* Good quality encode. */ -#define VIDEO_DECODE_BUFFER_SIZE 20 +/** + * Soft deadline the decoder should attempt to meet, in "us" (microseconds). + * Set to zero for unlimited. + * + * By convention, the value 1 is used to mean "return as fast as possible." + */ +// TODO: don't hardcode this, let the application choose it +#define WANTED_MAX_DECODER_FPS 40 + +/** + * VPX_DL_REALTIME (1) + * deadline parameter analogous to VPx REALTIME mode. + * + * VPX_DL_GOOD_QUALITY (1000000) + * deadline parameter analogous to VPx GOOD QUALITY mode. + * + * VPX_DL_BEST_QUALITY (0) + * deadline parameter analogous to VPx BEST QUALITY mode. + */ +#define MAX_DECODE_TIME_US (1000000 / WANTED_MAX_DECODER_FPS) // to allow x fps + +/** + * Codec control function to set encoder internal speed settings. Changes in + * this value influences, among others, the encoder's selection of motion + * estimation methods. Values greater than 0 will increase encoder speed at the + * expense of quality. + * + * Note Valid range for VP8: -16..16 + */ +#define VP8E_SET_CPUUSED_VALUE 16 + +/** + * Initialize encoder with this value. Target bandwidth to use for this stream, in kilobits per second. + */ +#define VIDEO_BITRATE_INITIAL_VALUE 5000 +#define VIDEO_DECODE_BUFFER_SIZE 5 // this buffer has normally max. 1 entry + +#define VIDEO_CODEC_DECODER_INTERFACE (vpx_codec_vp8_dx()) +#define VIDEO_CODEC_ENCODER_INTERFACE (vpx_codec_vp8_cx()) + +#define VIDEO_CODEC_DECODER_MAX_WIDTH 800 // its a dummy value, because the struct needs a value there +#define VIDEO_CODEC_DECODER_MAX_HEIGHT 600 // its a dummy value, because the struct needs a value there + +#define VPX_MAX_DIST_NORMAL 40 +#define VPX_MAX_DIST_START 40 + +#define VPX_MAX_ENCODER_THREADS 4 +#define VPX_MAX_DECODER_THREADS 4 +#define VIDEO__VP8_DECODER_POST_PROCESSING_ENABLED 0 + +void vc_init_encoder_cfg(Logger *log, vpx_codec_enc_cfg_t *cfg, int16_t kf_max_dist) +{ + vpx_codec_err_t rc = vpx_codec_enc_config_default(VIDEO_CODEC_ENCODER_INTERFACE, cfg, 0); + + if (rc != VPX_CODEC_OK) { + LOGGER_ERROR(log, "vc_init_encoder_cfg:Failed to get config: %s", vpx_codec_err_to_string(rc)); + } + + /* Target bandwidth to use for this stream, in kilobits per second */ + cfg->rc_target_bitrate = VIDEO_BITRATE_INITIAL_VALUE; + cfg->g_w = VIDEO_CODEC_DECODER_MAX_WIDTH; + cfg->g_h = VIDEO_CODEC_DECODER_MAX_HEIGHT; + cfg->g_pass = VPX_RC_ONE_PASS; + cfg->g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT | VPX_ERROR_RESILIENT_PARTITIONS; + cfg->g_lag_in_frames = 0; + + /* Allow lagged encoding + * + * If set, this value allows the encoder to consume a number of input + * frames before producing output frames. This allows the encoder to + * base decisions for the current frame on future frames. This does + * increase the latency of the encoding pipeline, so it is not appropriate + * in all situations (ex: realtime encoding). + * + * Note that this is a maximum value -- the encoder may produce frames + * sooner than the given limit. Set this value to 0 to disable this + * feature. + */ + cfg->kf_min_dist = 0; + cfg->kf_mode = VPX_KF_AUTO; // Encoder determines optimal placement automatically + cfg->rc_end_usage = VPX_VBR; // what quality mode? + + /* + * VPX_VBR Variable Bit Rate (VBR) mode + * VPX_CBR Constant Bit Rate (CBR) mode + * VPX_CQ Constrained Quality (CQ) mode -> give codec a hint that we may be on low bandwidth connection + * VPX_Q Constant Quality (Q) mode + */ + if (kf_max_dist > 1) { + cfg->kf_max_dist = kf_max_dist; // a full frame every x frames minimum (can be more often, codec decides automatically) + LOGGER_DEBUG(log, "kf_max_dist=%d (1)", cfg->kf_max_dist); + } else { + cfg->kf_max_dist = VPX_MAX_DIST_START; + LOGGER_DEBUG(log, "kf_max_dist=%d (2)", cfg->kf_max_dist); + } + + cfg->g_threads = VPX_MAX_ENCODER_THREADS; // Maximum number of threads to use + /* TODO: set these to something reasonable */ + // cfg->g_timebase.num = 1; + // cfg->g_timebase.den = 60; // 60 fps + cfg->rc_resize_allowed = 1; // allow encoder to resize to smaller resolution + cfg->rc_resize_up_thresh = 40; + cfg->rc_resize_down_thresh = 5; + + /* TODO: make quality setting an API call, but start with normal quality */ +#if 0 + /* Highest-resolution encoder settings */ + cfg->rc_dropframe_thresh = 0; + cfg->rc_resize_allowed = 0; + cfg->rc_min_quantizer = 2; + cfg->rc_max_quantizer = 56; + cfg->rc_undershoot_pct = 100; + cfg->rc_overshoot_pct = 15; + cfg->rc_buf_initial_sz = 500; + cfg->rc_buf_optimal_sz = 600; + cfg->rc_buf_sz = 1000; +#endif +} VCSession *vc_new(Logger *log, ToxAV *av, uint32_t friend_number, toxav_video_receive_frame_cb *cb, void *cb_data) { @@ -52,49 +168,72 @@ VCSession *vc_new(Logger *log, ToxAV *av, uint32_t friend_number, toxav_video_re return nullptr; } + int cpu_used_value = VP8E_SET_CPUUSED_VALUE; + if (!(vc->vbuf_raw = rb_new(VIDEO_DECODE_BUFFER_SIZE))) { goto BASE_CLEANUP; } - rc = vpx_codec_dec_init(vc->decoder, VIDEO_CODEC_DECODER_INTERFACE, nullptr, 0); + /* + * VPX_CODEC_USE_FRAME_THREADING + * Enable frame-based multi-threading + * + * VPX_CODEC_USE_ERROR_CONCEALMENT + * Conceal errors in decoded frames + */ + vpx_codec_dec_cfg_t dec_cfg; + dec_cfg.threads = VPX_MAX_DECODER_THREADS; // Maximum number of threads to use + dec_cfg.w = VIDEO_CODEC_DECODER_MAX_WIDTH; + dec_cfg.h = VIDEO_CODEC_DECODER_MAX_HEIGHT; + + LOGGER_DEBUG(log, "Using VP8 codec for decoder (0)"); + rc = vpx_codec_dec_init(vc->decoder, VIDEO_CODEC_DECODER_INTERFACE, &dec_cfg, + VPX_CODEC_USE_FRAME_THREADING | VPX_CODEC_USE_POSTPROC); + + if (rc == VPX_CODEC_INCAPABLE) { + LOGGER_WARNING(log, "Postproc not supported by this decoder (0)"); + rc = vpx_codec_dec_init(vc->decoder, VIDEO_CODEC_DECODER_INTERFACE, &dec_cfg, VPX_CODEC_USE_FRAME_THREADING); + } if (rc != VPX_CODEC_OK) { LOGGER_ERROR(log, "Init video_decoder failed: %s", vpx_codec_err_to_string(rc)); goto BASE_CLEANUP; } - /* Set encoder to some initial values - */ - vpx_codec_enc_cfg_t cfg; - rc = vpx_codec_enc_config_default(VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0); + if (VIDEO__VP8_DECODER_POST_PROCESSING_ENABLED == 1) { + vp8_postproc_cfg_t pp = {VP8_DEBLOCK, 1, 0}; + vpx_codec_err_t cc_res = vpx_codec_control(vc->decoder, VP8_SET_POSTPROC, &pp); - if (rc != VPX_CODEC_OK) { - LOGGER_ERROR(log, "Failed to get config: %s", vpx_codec_err_to_string(rc)); - goto BASE_CLEANUP_1; + if (cc_res != VPX_CODEC_OK) { + LOGGER_WARNING(log, "Failed to turn on postproc"); + } else { + LOGGER_DEBUG(log, "turn on postproc: OK"); + } + } else { + vp8_postproc_cfg_t pp = {0, 0, 0}; + vpx_codec_err_t cc_res = vpx_codec_control(vc->decoder, VP8_SET_POSTPROC, &pp); + + if (cc_res != VPX_CODEC_OK) { + LOGGER_WARNING(log, "Failed to turn OFF postproc"); + } else { + LOGGER_DEBUG(log, "Disable postproc: OK"); + } } - cfg.rc_target_bitrate = 500000; - cfg.g_w = 800; - cfg.g_h = 600; - cfg.g_pass = VPX_RC_ONE_PASS; - /* TODO(mannol): If we set error resilience the app will crash due to bug in vp8. - Perhaps vp9 has solved it?*/ -#if 0 - cfg.g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT | VPX_ERROR_RESILIENT_PARTITIONS; -#endif - cfg.g_lag_in_frames = 0; - cfg.kf_min_dist = 0; - cfg.kf_max_dist = 48; - cfg.kf_mode = VPX_KF_AUTO; + /* Set encoder to some initial values + */ + vpx_codec_enc_cfg_t cfg; + vc_init_encoder_cfg(log, &cfg, 1); - rc = vpx_codec_enc_init(vc->encoder, VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0); + LOGGER_DEBUG(log, "Using VP8 codec for encoder (0.1)"); + rc = vpx_codec_enc_init(vc->encoder, VIDEO_CODEC_ENCODER_INTERFACE, &cfg, VPX_CODEC_USE_FRAME_THREADING); if (rc != VPX_CODEC_OK) { LOGGER_ERROR(log, "Failed to initialize encoder: %s", vpx_codec_err_to_string(rc)); goto BASE_CLEANUP_1; } - rc = vpx_codec_control(vc->encoder, VP8E_SET_CPUUSED, 8); + rc = vpx_codec_control(vc->encoder, VP8E_SET_CPUUSED, cpu_used_value); if (rc != VPX_CODEC_OK) { LOGGER_ERROR(log, "Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); @@ -102,6 +241,20 @@ VCSession *vc_new(Logger *log, ToxAV *av, uint32_t friend_number, toxav_video_re goto BASE_CLEANUP_1; } + /* + VPX_CTRL_USE_TYPE(VP8E_SET_NOISE_SENSITIVITY, unsigned int) + control function to set noise sensitivity + 0: off, 1: OnYOnly, 2: OnYUV, 3: OnYUVAggressive, 4: Adaptive + */ + /* + rc = vpx_codec_control(vc->encoder, VP8E_SET_NOISE_SENSITIVITY, 2); + + if (rc != VPX_CODEC_OK) { + LOGGER_ERROR(log, "Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); + vpx_codec_destroy(vc->encoder); + goto BASE_CLEANUP_1; + } + */ vc->linfts = current_time_monotonic(); vc->lcfd = 60; vc->vcb.first = cb; @@ -109,9 +262,7 @@ VCSession *vc_new(Logger *log, ToxAV *av, uint32_t friend_number, toxav_video_re vc->friend_number = friend_number; vc->av = av; vc->log = log; - return vc; - BASE_CLEANUP_1: vpx_codec_destroy(vc->decoder); BASE_CLEANUP: @@ -120,6 +271,7 @@ VCSession *vc_new(Logger *log, ToxAV *av, uint32_t friend_number, toxav_video_re free(vc); return nullptr; } + void vc_kill(VCSession *vc) { if (!vc) { @@ -128,7 +280,6 @@ void vc_kill(VCSession *vc) vpx_codec_destroy(vc->encoder); vpx_codec_destroy(vc->decoder); - void *p; while (rb_read((RingBuffer *)vc->vbuf_raw, &p)) { @@ -136,12 +287,11 @@ void vc_kill(VCSession *vc) } rb_kill((RingBuffer *)vc->vbuf_raw); - pthread_mutex_destroy(vc->queue_mutex); - LOGGER_DEBUG(vc->log, "Terminated video handler: %p", vc); free(vc); } + void vc_iterate(VCSession *vc) { if (!vc) { @@ -154,45 +304,63 @@ void vc_iterate(VCSession *vc) pthread_mutex_lock(vc->queue_mutex); + uint32_t full_data_len; + if (rb_read((RingBuffer *)vc->vbuf_raw, (void **)&p)) { pthread_mutex_unlock(vc->queue_mutex); + const struct RTPHeader *const header = &p->header; + + if (header->flags & RTP_LARGE_FRAME) { + full_data_len = header->data_length_full; + LOGGER_DEBUG(vc->log, "vc_iterate:001:full_data_len=%d", (int)full_data_len); + } else { + full_data_len = p->len; + LOGGER_DEBUG(vc->log, "vc_iterate:002"); + } - rc = vpx_codec_decode(vc->decoder, p->data, p->len, nullptr, MAX_DECODE_TIME_US); + LOGGER_DEBUG(vc->log, "vc_iterate: rb_read p->len=%d p->header.xe=%d", (int)full_data_len, p->header.xe); + LOGGER_DEBUG(vc->log, "vc_iterate: rb_read rb size=%d", (int)rb_size((RingBuffer *)vc->vbuf_raw)); + rc = vpx_codec_decode(vc->decoder, p->data, full_data_len, nullptr, MAX_DECODE_TIME_US); free(p); if (rc != VPX_CODEC_OK) { - LOGGER_ERROR(vc->log, "Error decoding video: %s", vpx_codec_err_to_string(rc)); + LOGGER_ERROR(vc->log, "Error decoding video: %d %s", (int)rc, vpx_codec_err_to_string(rc)); } else { + /* Play decoded images */ vpx_codec_iter_t iter = nullptr; - vpx_image_t *dest = vpx_codec_get_frame(vc->decoder, &iter); + vpx_image_t *dest = nullptr; - /* Play decoded images */ - for (; dest; dest = vpx_codec_get_frame(vc->decoder, &iter)) { + while ((dest = vpx_codec_get_frame(vc->decoder, &iter)) != nullptr) { if (vc->vcb.first) { vc->vcb.first(vc->av, vc->friend_number, dest->d_w, dest->d_h, (const uint8_t *)dest->planes[0], (const uint8_t *)dest->planes[1], (const uint8_t *)dest->planes[2], dest->stride[0], dest->stride[1], dest->stride[2], vc->vcb.second); } - vpx_img_free(dest); + vpx_img_free(dest); // is this needed? none of the VPx examples show that } } return; + } else { + LOGGER_TRACE(vc->log, "no Video frame data available"); } pthread_mutex_unlock(vc->queue_mutex); } + int vc_queue_message(void *vcp, struct RTPMessage *msg) { - /* This function does the reconstruction of video packets. - * See more info about video splitting in docs + /* This function is called with complete messages + * they have already been assembled. + * this function gets called from handle_rtp_packet() and handle_rtp_packet_v3() */ if (!vcp || !msg) { return -1; } VCSession *vc = (VCSession *)vcp; + const struct RTPHeader *const header = &msg->header; if (msg->header.pt == (rtp_TypeVideo + 2) % 128) { LOGGER_WARNING(vc->log, "Got dummy!"); @@ -201,41 +369,45 @@ int vc_queue_message(void *vcp, struct RTPMessage *msg) } if (msg->header.pt != rtp_TypeVideo % 128) { - LOGGER_WARNING(vc->log, "Invalid payload type!"); + LOGGER_WARNING(vc->log, "Invalid payload type! pt=%d", (int)msg->header.pt); free(msg); return -1; } pthread_mutex_lock(vc->queue_mutex); - free(rb_write((RingBuffer *)vc->vbuf_raw, msg)); - { - /* Calculate time took for peer to send us this frame */ - uint32_t t_lcfd = current_time_monotonic() - vc->linfts; - vc->lcfd = t_lcfd > 100 ? vc->lcfd : t_lcfd; - vc->linfts = current_time_monotonic(); + + if ((header->flags & RTP_LARGE_FRAME) && header->pt == rtp_TypeVideo % 128) { + LOGGER_DEBUG(vc->log, "rb_write msg->len=%d b0=%d b1=%d", (int)msg->len, (int)msg->data[0], (int)msg->data[1]); } - pthread_mutex_unlock(vc->queue_mutex); + free(rb_write((RingBuffer *)vc->vbuf_raw, msg)); + + /* Calculate time it took for peer to send us this frame */ + uint32_t t_lcfd = current_time_monotonic() - vc->linfts; + vc->lcfd = t_lcfd > 100 ? vc->lcfd : t_lcfd; + vc->linfts = current_time_monotonic(); + pthread_mutex_unlock(vc->queue_mutex); return 0; } -int vc_reconfigure_encoder(VCSession *vc, uint32_t bit_rate, uint16_t width, uint16_t height) + +int vc_reconfigure_encoder(VCSession *vc, uint32_t bit_rate, uint16_t width, uint16_t height, int16_t kf_max_dist) { if (!vc) { return -1; } - vpx_codec_enc_cfg_t cfg = *vc->encoder->config.enc; + vpx_codec_enc_cfg_t cfg2 = *vc->encoder->config.enc; vpx_codec_err_t rc; - if (cfg.rc_target_bitrate == bit_rate && cfg.g_w == width && cfg.g_h == height) { + if (cfg2.rc_target_bitrate == bit_rate && cfg2.g_w == width && cfg2.g_h == height && kf_max_dist == -1) { return 0; /* Nothing changed */ } - if (cfg.g_w == width && cfg.g_h == height) { + if (cfg2.g_w == width && cfg2.g_h == height && kf_max_dist == -1) { /* Only bit rate changed */ - cfg.rc_target_bitrate = bit_rate; - - rc = vpx_codec_enc_config_set(vc->encoder, &cfg); + LOGGER_INFO(vc->log, "bitrate change from: %u to: %u", (uint32_t)cfg2.rc_target_bitrate, (uint32_t)bit_rate); + cfg2.rc_target_bitrate = bit_rate; + rc = vpx_codec_enc_config_set(vc->encoder, &cfg2); if (rc != VPX_CODEC_OK) { LOGGER_ERROR(vc->log, "Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); @@ -245,23 +417,25 @@ int vc_reconfigure_encoder(VCSession *vc, uint32_t bit_rate, uint16_t width, uin /* Resolution is changed, must reinitialize encoder since libvpx v1.4 doesn't support * reconfiguring encoder to use resolutions greater than initially set. */ - LOGGER_DEBUG(vc->log, "Have to reinitialize vpx encoder on session %p", vc); - + vpx_codec_ctx_t new_c; + vpx_codec_enc_cfg_t cfg; + vc_init_encoder_cfg(vc->log, &cfg, kf_max_dist); cfg.rc_target_bitrate = bit_rate; cfg.g_w = width; cfg.g_h = height; - vpx_codec_ctx_t new_c; - - rc = vpx_codec_enc_init(&new_c, VIDEO_CODEC_ENCODER_INTERFACE, &cfg, 0); + LOGGER_DEBUG(vc->log, "Using VP8 codec for encoder"); + rc = vpx_codec_enc_init(&new_c, VIDEO_CODEC_ENCODER_INTERFACE, &cfg, VPX_CODEC_USE_FRAME_THREADING); if (rc != VPX_CODEC_OK) { LOGGER_ERROR(vc->log, "Failed to initialize encoder: %s", vpx_codec_err_to_string(rc)); return -1; } - rc = vpx_codec_control(&new_c, VP8E_SET_CPUUSED, 8); + int cpu_used_value = VP8E_SET_CPUUSED_VALUE; + + rc = vpx_codec_control(&new_c, VP8E_SET_CPUUSED, cpu_used_value); if (rc != VPX_CODEC_OK) { LOGGER_ERROR(vc->log, "Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); diff --git a/toxav/video.h b/toxav/video.h index 02670e0a96..85ad312936 100644 --- a/toxav/video.h +++ b/toxav/video.h @@ -31,8 +31,7 @@ #include #include -#define VIDEO_CODEC_DECODER_INTERFACE (vpx_codec_vp8_dx()) -#define VIDEO_CODEC_ENCODER_INTERFACE (vpx_codec_vp8_cx()) + #include @@ -64,6 +63,6 @@ VCSession *vc_new(Logger *log, ToxAV *av, uint32_t friend_number, toxav_video_re void vc_kill(VCSession *vc); void vc_iterate(VCSession *vc); int vc_queue_message(void *vcp, struct RTPMessage *msg); -int vc_reconfigure_encoder(VCSession *vc, uint32_t bit_rate, uint16_t width, uint16_t height); +int vc_reconfigure_encoder(VCSession *vc, uint32_t bit_rate, uint16_t width, uint16_t height, int16_t kf_max_dist); #endif /* VIDEO_H */