diff --git a/libavformat/rtpenc.c b/libavformat/rtpenc.c
index 360fa2cc934ab..1eb01911a8c72 100644
--- a/libavformat/rtpenc.c
+++ b/libavformat/rtpenc.c
@@ -152,33 +152,6 @@ static int rtp_write_header(AVFormatContext *s1)
     }
     s->max_payload_size = s1->packet_size - 12;
 
-    s->max_frames_per_packet = 0;
-    if (s1->max_delay > 0) {
-        if (st->codec->codec_type == AVMEDIA_TYPE_AUDIO) {
-            int frame_size = av_get_audio_frame_duration(st->codec, 0);
-            if (!frame_size)
-                frame_size = st->codec->frame_size;
-            if (frame_size == 0) {
-                av_log(s1, AV_LOG_ERROR, "Cannot respect max delay: frame size = 0\n");
-            } else {
-                s->max_frames_per_packet =
-                        av_rescale_q_rnd(s1->max_delay,
-                                         AV_TIME_BASE_Q,
-                                         (AVRational){ frame_size, st->codec->sample_rate },
-                                         AV_ROUND_DOWN);
-            }
-        }
-        if (st->codec->codec_type == AVMEDIA_TYPE_VIDEO) {
-            /* FIXME: We should round down here... */
-            if (st->avg_frame_rate.num > 0 && st->avg_frame_rate.den > 0) {
-                s->max_frames_per_packet = av_rescale_q(s1->max_delay,
-                                                        (AVRational){1, 1000000},
-                                                        av_inv_q(st->avg_frame_rate));
-            } else
-                s->max_frames_per_packet = 1;
-        }
-    }
-
     if (st->codec->codec_type == AVMEDIA_TYPE_AUDIO) {
         avpriv_set_pts_info(st, 32, 1, st->codec->sample_rate);
     } else {
@@ -228,9 +201,7 @@ static int rtp_write_header(AVFormatContext *s1)
         break;
     case AV_CODEC_ID_VORBIS:
     case AV_CODEC_ID_THEORA:
-        if (!s->max_frames_per_packet)
-            s->max_frames_per_packet = 15;
-        s->max_frames_per_packet = av_clip(s->max_frames_per_packet, 1, 15);
+        s->max_frames_per_packet = 15;
         break;
     case AV_CODEC_ID_ADPCM_G722:
         /* Due to a historical error, the clock rate for G722 in RTP is
@@ -252,15 +223,11 @@ static int rtp_write_header(AVFormatContext *s1)
             av_log(s1, AV_LOG_ERROR, "Incorrect iLBC block size specified\n");
             goto fail;
         }
-        if (!s->max_frames_per_packet)
-            s->max_frames_per_packet = 1;
-        s->max_frames_per_packet = FFMIN(s->max_frames_per_packet,
-                                         s->max_payload_size / st->codec->block_align);
+        s->max_frames_per_packet = s->max_payload_size / st->codec->block_align;
         break;
     case AV_CODEC_ID_AMR_NB:
     case AV_CODEC_ID_AMR_WB:
-        if (!s->max_frames_per_packet)
-            s->max_frames_per_packet = 12;
+        s->max_frames_per_packet = 50;
         if (st->codec->codec_id == AV_CODEC_ID_AMR_NB)
             n = 31;
         else
@@ -276,8 +243,7 @@ static int rtp_write_header(AVFormatContext *s1)
         }
         break;
     case AV_CODEC_ID_AAC:
-        if (!s->max_frames_per_packet)
-            s->max_frames_per_packet = 5;
+        s->max_frames_per_packet = 50;
         break;
     default:
         break;
@@ -496,18 +462,23 @@ static int rtp_send_ilbc(AVFormatContext *s1, const uint8_t *buf, int size)
     int frames = size / frame_size;
 
     while (frames > 0) {
-        int n = FFMIN(s->max_frames_per_packet - s->num_frames, frames);
+        if (s->num_frames > 0 &&
+            av_compare_ts(s->cur_timestamp - s->timestamp, st->time_base,
+                          s1->max_delay, AV_TIME_BASE_Q) >= 0) {
+            ff_rtp_send_data(s1, s->buf, s->buf_ptr - s->buf, 1);
+            s->num_frames = 0;
+        }
 
         if (!s->num_frames) {
             s->buf_ptr = s->buf;
             s->timestamp = s->cur_timestamp;
         }
-        memcpy(s->buf_ptr, buf, n * frame_size);
-        frames           -= n;
-        s->num_frames    += n;
-        s->buf_ptr       += n * frame_size;
-        buf              += n * frame_size;
-        s->cur_timestamp += n * frame_duration;
+        memcpy(s->buf_ptr, buf, frame_size);
+        frames--;
+        s->num_frames++;
+        s->buf_ptr       += frame_size;
+        buf              += frame_size;
+        s->cur_timestamp += frame_duration;
 
         if (s->num_frames == s->max_frames_per_packet) {
             ff_rtp_send_data(s1, s->buf, s->buf_ptr - s->buf, 1);
diff --git a/libavformat/rtpenc_aac.c b/libavformat/rtpenc_aac.c
index c20bbebc8fe59..f835183ac3f89 100644
--- a/libavformat/rtpenc_aac.c
+++ b/libavformat/rtpenc_aac.c
@@ -27,6 +27,7 @@
 void ff_rtp_send_aac(AVFormatContext *s1, const uint8_t *buff, int size)
 {
     RTPMuxContext *s = s1->priv_data;
+    AVStream *st = s1->streams[0];
     const int max_au_headers_size = 2 + 2 * s->max_frames_per_packet;
     int len, max_packet_size = s->max_payload_size - max_au_headers_size;
     uint8_t *p;
@@ -41,7 +42,9 @@ void ff_rtp_send_aac(AVFormatContext *s1, const uint8_t *buff, int size)
     len = (s->buf_ptr - s->buf);
     if (s->num_frames &&
         (s->num_frames == s->max_frames_per_packet ||
-         (len + size) > s->max_payload_size)) {
+         (len + size) > s->max_payload_size ||
+         av_compare_ts(s->cur_timestamp - s->timestamp, st->time_base,
+                       s1->max_delay, AV_TIME_BASE_Q) >= 0)) {
         int au_size = s->num_frames * 2;
 
         p = s->buf + max_au_headers_size - au_size - 2;
diff --git a/libavformat/rtpenc_amr.c b/libavformat/rtpenc_amr.c
index 08b614dd4d16b..8df5a3cf3e999 100644
--- a/libavformat/rtpenc_amr.c
+++ b/libavformat/rtpenc_amr.c
@@ -30,6 +30,7 @@
 void ff_rtp_send_amr(AVFormatContext *s1, const uint8_t *buff, int size)
 {
     RTPMuxContext *s          = s1->priv_data;
+    AVStream *st              = s1->streams[0];
     int max_header_toc_size   = 1 + s->max_frames_per_packet;
     uint8_t *p;
     int len;
@@ -38,7 +39,9 @@ void ff_rtp_send_amr(AVFormatContext *s1, const uint8_t *buff, int size)
     len = s->buf_ptr - s->buf;
     if (s->num_frames &&
         (s->num_frames == s->max_frames_per_packet ||
-         len + size - 1 > s->max_payload_size)) {
+         len + size - 1 > s->max_payload_size ||
+         av_compare_ts(s->cur_timestamp - s->timestamp, st->time_base,
+                       s1->max_delay, AV_TIME_BASE_Q) >= 0)) {
         int header_size = s->num_frames + 1;
         p = s->buf + max_header_toc_size - header_size;
         if (p != s->buf)
diff --git a/libavformat/rtpenc_xiph.c b/libavformat/rtpenc_xiph.c
index 7572b176beef6..b4d950b91d76f 100644
--- a/libavformat/rtpenc_xiph.c
+++ b/libavformat/rtpenc_xiph.c
@@ -33,6 +33,7 @@
 void ff_rtp_send_xiph(AVFormatContext *s1, const uint8_t *buff, int size)
 {
     RTPMuxContext *s = s1->priv_data;
+    AVStream *st = s1->streams[0];
     int max_pkt_size, xdt, frag;
     uint8_t *q;
 
@@ -78,8 +79,10 @@ void ff_rtp_send_xiph(AVFormatContext *s1, const uint8_t *buff, int size)
         av_assert1(s->num_frames <= s->max_frames_per_packet);
         if (s->num_frames > 0 &&
             (remaining < 0 ||
-             s->num_frames == s->max_frames_per_packet)) {
-            // send previous packets now; no room for new data
+             s->num_frames == s->max_frames_per_packet ||
+             av_compare_ts(s->cur_timestamp - s->timestamp, st->time_base,
+                           s1->max_delay, AV_TIME_BASE_Q) >= 0)) {
+            // send previous packets now; no room for new data, or too much delay
             ff_rtp_send_data(s1, s->buf, s->buf_ptr - s->buf, 0);
             s->num_frames = 0;
         }