Skip to content

Commit

Permalink
Made AV1/SVC finally work
Browse files Browse the repository at this point in the history
  • Loading branch information
@lminiero
lminiero committed May 12, 2023
1 parent f282022 commit 8de2c78
Show file tree
Hide file tree
Showing 4 changed files with 159 additions and 54 deletions.
7 changes: 2 additions & 5 deletions src/plugins/janus_echotest.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -639,7 +639,7 @@ void janus_echotest_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp
av1ctx = &session->av1_context[session->sim_context.substream];
uint8_t template = 0;
if(janus_av1_svc_context_process_dd(av1ctx, packet->extensions.dd_content,
packet->extensions.dd_len, &template)) {
packet->extensions.dd_len, &template, NULL)) {
janus_av1_svc_template *t = g_hash_table_lookup(av1ctx->templates, GUINT_TO_POINTER(template));
if(t) {
int temporal_layer = session->sim_context.templayer;
Expand Down Expand Up @@ -670,7 +670,7 @@ void janus_echotest_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp
} else {
/* Process this SVC packet: don't relay if it's not the layer we wanted to handle */
relay = janus_rtp_svc_context_process_rtp(&session->svc_context,
buf, len, session->vcodec, NULL, &session->context);
buf, len, packet->extensions.dd_content, packet->extensions.dd_len, session->vcodec, NULL, &session->context);
}
if(session->sim_context.need_pli || session->svc_context.need_pli) {
/* Send a PLI */
Expand Down Expand Up @@ -1035,9 +1035,6 @@ static void *janus_echotest_handler(void *data) {
janus_rtp_svc_context_reset(&session->svc_context);
session->svc_context.spatial_target = 2; /* FIXME Actually depends on the scalabilityMode */
session->svc_context.temporal_target = 2; /* FIXME Actually depends on the scalabilityMode */
janus_av1_svc_context_reset(&session->av1_context[0]);
janus_av1_svc_context_reset(&session->av1_context[1]);
janus_av1_svc_context_reset(&session->av1_context[2]);
session->svc = TRUE;
}
/* FIXME We're stopping at the first item, there may be more */
Expand Down
2 changes: 1 addition & 1 deletion src/plugins/janus_videoroom.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12149,7 +12149,7 @@ static void janus_videoroom_relay_rtp_packet(gpointer data, gpointer user_data)
/* Process this packet: don't relay if it's not the layer we wanted to handle */
janus_rtp_header rtp = *(packet->data);
gboolean relay = janus_rtp_svc_context_process_rtp(&stream->svc_context,
(char *)packet->data, packet->length, ps->vcodec, &packet->svc_info, &stream->context);
(char *)packet->data, packet->length, NULL, NULL, ps->vcodec, &packet->svc_info, &stream->context);
if(stream->svc_context.need_pli) {
/* Send a PLI */
JANUS_LOG(LOG_VERB, "We need a PLI for the SVC context\n");
Expand Down
111 changes: 107 additions & 4 deletions src/rtp.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1277,14 +1277,16 @@ void janus_rtp_svc_context_reset(janus_rtp_svc_context *context) {
if(context == NULL)
return;
/* Reset the context values */
janus_av1_svc_context_reset(&context->dd_context);
memset(context, 0, sizeof(*context));
context->spatial = -1;
context->temporal = -1;
}

gboolean janus_rtp_svc_context_process_rtp(janus_rtp_svc_context *context, char *buf, int len,
gboolean janus_rtp_svc_context_process_rtp(janus_rtp_svc_context *context,
char *buf, int len, uint8_t *dd_content, int dd_len,
janus_videocodec vcodec, janus_vp9_svc_info *info, janus_rtp_switching_context *sc) {
if(!context || !buf || len < 1 || vcodec != JANUS_VIDEOCODEC_VP9)
if(!context || !buf || len < 1 || (vcodec != JANUS_VIDEOCODEC_VP9 && vcodec != JANUS_VIDEOCODEC_AV1))
return FALSE;
janus_rtp_header *header = (janus_rtp_header *)buf;
/* Reset the flags */
Expand All @@ -1297,7 +1299,106 @@ gboolean janus_rtp_svc_context_process_rtp(janus_rtp_svc_context *context, char
char *payload = janus_rtp_payload(buf, len, &plen);
if(payload == NULL)
return FALSE;
/* If we don't have any info parsed from the VP9 payload header, get it now */
/* Check if we should use the Dependency Descriptor */
if(vcodec == JANUS_VIDEOCODEC_AV1) {
/* We do, make sure the data is there */
if(dd_content == NULL || dd_len < 1)
return FALSE;
uint8_t template = 0, ebit = 0;
if(!janus_av1_svc_context_process_dd(&context->dd_context, dd_content, dd_len, &template, &ebit)) {
/* We couldn't parse the Dependency Descriptor, relay as it is */
return TRUE;
}
janus_av1_svc_template *t = g_hash_table_lookup(context->dd_context.templates, GUINT_TO_POINTER(template));
if(t == NULL) {
/* We couldn't find the template, relay as it is */
return TRUE;
}
/* Now let's check if we should let the packet through or not */
gboolean keyframe = janus_av1_is_keyframe((const char *)payload, plen);
gboolean override_mark_bit = FALSE, has_marker_bit = header->markerbit;
int spatial_layer = context->spatial;
if(t->spatial >= 0 && t->spatial <= 2)
context->last_spatial_layer[t->spatial] = now;
if(context->spatial_target > context->spatial) {
JANUS_LOG(LOG_HUGE, "We need to upscale spatially: (%d < %d)\n",
context->spatial, context->spatial_target);
/* We need to upscale: wait for a keyframe */
if(keyframe) {
int new_spatial_layer = context->spatial_target;
while(new_spatial_layer > context->spatial && new_spatial_layer > 0) {
if(now - context->last_spatial_layer[new_spatial_layer] >= (context->drop_trigger ? context->drop_trigger : 250000)) {
/* We haven't received packets from this layer for a while, try a lower layer */
JANUS_LOG(LOG_HUGE, "Haven't received packets from layer %d for a while, trying %d instead...\n",
new_spatial_layer, new_spatial_layer-1);
new_spatial_layer--;
} else {
break;
}
}
if(new_spatial_layer > context->spatial) {
JANUS_LOG(LOG_HUGE, " -- Upscaling spatial layer: %d --> %d (need %d)\n",
context->spatial, new_spatial_layer, context->spatial_target);
context->spatial = new_spatial_layer;
spatial_layer = context->spatial;
context->changed_spatial = TRUE;
}
}
} else if(context->spatial_target < context->spatial) {
/* We need to scale: wait for a keyframe */
JANUS_LOG(LOG_HUGE, "We need to downscale spatially: (%d > %d)\n",
context->spatial, context->spatial_target);
/* Check the E bit to see if this is an end-of-frame */
if(ebit) {
JANUS_LOG(LOG_HUGE, " -- Downscaling spatial layer: %d --> %d\n",
context->spatial, context->spatial_target);
context->spatial = context->spatial_target;
context->changed_spatial = TRUE;
}
}
if(spatial_layer < t->spatial) {
/* Drop the packet: update the context to make sure sequence number is increased normally later */
JANUS_LOG(LOG_HUGE, "Dropping packet (spatial layer %d < %d)\n", spatial_layer, t->spatial);
if(sc)
sc->base_seq++;
return FALSE;
} else if(ebit && spatial_layer == t->spatial) {
/* If we stop at layer 0, we need a marker bit now, as the one from layer 1 will not be received */
override_mark_bit = TRUE;
}
int temporal = context->temporal;
if(context->temporal_target > context->temporal) {
/* We need to upscale */
if(t->temporal > context->temporal && t->temporal <= context->temporal_target) {
context->temporal = t->temporal;
temporal = context->temporal;
context->changed_temporal = TRUE;
}
} else if(context->temporal_target < context->temporal) {
/* We need to downscale */
if(t->temporal == context->temporal_target) {
context->temporal = context->temporal_target;
context->changed_temporal = TRUE;
}
}
if(temporal < t->temporal) {
JANUS_LOG(LOG_HUGE, "Dropping packet (it's temporal layer %d, but we're capping at %d)\n",
t->temporal, context->temporal);
/* We increase the base sequence number, or there will be gaps when delivering later */
if(sc)
sc->base_seq++;
return FALSE;
}
/* If we got here, we can send the frame: this doesn't necessarily mean it's
* one of the layers the user wants, as there may be dependencies involved */
JANUS_LOG(LOG_HUGE, "Sending packet (spatial=%d, temporal=%d)\n",
t->spatial, t->temporal);
if(override_mark_bit && !has_marker_bit)
header->markerbit = 1;
return TRUE;
}
/* If we got here, it's VP9, for which we parse the payload manually:
* if we don't have any info parsed from the VP9 payload header, get it now */
janus_vp9_svc_info svc_info = { 0 };
if(!info) {
gboolean found = FALSE;
Expand Down Expand Up @@ -1425,7 +1526,7 @@ void janus_av1_svc_context_reset(janus_av1_svc_context *context) {
}

gboolean janus_av1_svc_context_process_dd(janus_av1_svc_context *context,
uint8_t *dd, int dd_len, uint8_t *template_id) {
uint8_t *dd, int dd_len, uint8_t *template_id, uint8_t *ebit) {
if(!context || !dd || dd_len < 3)
return FALSE;

Expand All @@ -1435,6 +1536,8 @@ gboolean janus_av1_svc_context_process_dd(janus_av1_svc_context *context,
/* mandatory_descriptor_fields() */
uint8_t start = janus_bitstream_getbit(dd, offset++);
uint8_t end = janus_bitstream_getbit(dd, offset++);
if(ebit)
*ebit = end;
uint8_t template = janus_bitstream_getbits(dd, 6, &offset);
uint16_t frame = janus_bitstream_getbits(dd, 16, &offset);
JANUS_LOG(LOG_HUGE, " -- s=%u, e=%u, t=%u, f=%u\n",
Expand Down
93 changes: 49 additions & 44 deletions src/rtp.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -359,49 +359,6 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte
janus_videocodec vcodec, janus_rtp_switching_context *sc, janus_mutex *rid_mutex);
///@}

/** @name Janus SVC processing methods
*/
///@{
/*! \brief Helper struct for processing and tracking VP9-SVC streams */
typedef struct janus_rtp_svc_context {
/*! \brief Which SVC spatial layer we should forward back */
int spatial;
/*! \brief As above, but to handle transitions (e.g., wait for keyframe, or get this if available) */
int spatial_target;
/*! \brief Which SVC temporal layer we should forward back */
int temporal;
/*! \brief As above, but to handle transitions (e.g., wait for keyframe) */
int temporal_target;
/*! \brief How much time (in us, default 250000) without receiving packets will make us drop to the substream below */
guint32 drop_trigger;
/*! \brief When we relayed the last packet (used to detect when layers become unavailable) */
gint64 last_spatial_layer[3];
/*! \brief Whether the spatial layer has changed after processing a packet */
gboolean changed_spatial;
/*! \brief Whether the temporal layer has changed after processing a packet */
gboolean changed_temporal;
/*! \brief Whether we need to send the user a keyframe request (PLI) */
gboolean need_pli;
} janus_rtp_svc_context;

/*! \brief Set (or reset) the context fields to their default values
* @param[in] context The context to (re)set */
void janus_rtp_svc_context_reset(janus_rtp_svc_context *context);

/*! \brief Process an RTP packet, and decide whether this should be relayed or not, updating the context accordingly
* \note Calling this method resets the \c changed_spatial , \c changed_temporal and \c need_pli
* properties, and updates them according to the decisions made after processing the packet
* @param[in] context The VP9 SVC context to use
* @param[in] buf The RTP packet to process
* @param[in] len The length of the RTP packet (header, extension and payload)
* @param[in] vcodec Video codec of the RTP payload
* @param[in] info Parsed info on VP9-SVC, if any
* @param[in] sc RTP switching context to refer to, if any
* @returns TRUE if the packet should be relayed, FALSE if it should be dropped instead */
gboolean janus_rtp_svc_context_process_rtp(janus_rtp_svc_context *context, char *buf, int len,
janus_videocodec vcodec, janus_vp9_svc_info *info, janus_rtp_switching_context *sc);
///@}

/** @name Janus AV1-SVC processing methods (still WIP)
*/
///@{
Expand Down Expand Up @@ -444,10 +401,58 @@ void janus_av1_svc_context_reset(janus_av1_svc_context *context);
* @param[in] dd Pointer to the Dependency Descriptor data
* @param[in] dd_len The length of the Dependendy Descriptor data
* @param[out] template_id Pointer to the ID of the template referenced in this packet
* @param[out] ebit Whether this packet is an end of frame or not
* @returns TRUE if the packet is valid, FALSE if it should be dropped instead */
gboolean janus_av1_svc_context_process_dd(janus_av1_svc_context *context,
uint8_t *dd, int dd_len, uint8_t *template_id);
uint8_t *dd, int dd_len, uint8_t *template_id, uint8_t *ebit);
///@}

/** @name Janus SVC processing methods
*/
///@{
/*! \brief Helper struct for processing and tracking VP9-SVC streams */
typedef struct janus_rtp_svc_context {
/*! \brief Dependency Descriptor context, in case it's needed */
struct janus_av1_svc_context dd_context;
/*! \brief Which SVC spatial layer we should forward back */
int spatial;
/*! \brief As above, but to handle transitions (e.g., wait for keyframe, or get this if available) */
int spatial_target;
/*! \brief Which SVC temporal layer we should forward back */
int temporal;
/*! \brief As above, but to handle transitions (e.g., wait for keyframe) */
int temporal_target;
/*! \brief How much time (in us, default 250000) without receiving packets will make us drop to the substream below */
guint32 drop_trigger;
/*! \brief When we relayed the last packet (used to detect when layers become unavailable) */
gint64 last_spatial_layer[3];
/*! \brief Whether the spatial layer has changed after processing a packet */
gboolean changed_spatial;
/*! \brief Whether the temporal layer has changed after processing a packet */
gboolean changed_temporal;
/*! \brief Whether we need to send the user a keyframe request (PLI) */
gboolean need_pli;
} janus_rtp_svc_context;

/*! \brief Set (or reset) the context fields to their default values
* @param[in] context The context to (re)set */
void janus_rtp_svc_context_reset(janus_rtp_svc_context *context);

/*! \brief Process an RTP packet, and decide whether this should be relayed or not, updating the context accordingly
* \note Calling this method resets the \c changed_spatial , \c changed_temporal and \c need_pli
* properties, and updates them according to the decisions made after processing the packet
* @param[in] context The VP9 SVC context to use
* @param[in] buf The RTP packet to process
* @param[in] len The length of the RTP packet (header, extension and payload)
* @param[in] dd_content The Dependency Descriptor RTP extension data, if available
* @param[in] dd_len Length of the Dependency Descriptor data, if available
* @param[in] vcodec Video codec of the RTP payload
* @param[in] info Parsed info on VP9-SVC, if any
* @param[in] sc RTP switching context to refer to, if any
* @returns TRUE if the packet should be relayed, FALSE if it should be dropped instead */
gboolean janus_rtp_svc_context_process_rtp(janus_rtp_svc_context *context,
char *buf, int len, uint8_t *dd_content, int dd_len,
janus_videocodec vcodec, janus_vp9_svc_info *info, janus_rtp_switching_context *sc);
///@}

#endif

0 comments on commit 8de2c78

Please sign in to comment.