Fix remaining libfishsound issues in Mozilla bug 480014

vorbis.c: Remove unnecessary alloca
speex.c: Check that frame_size is not so large that the buffer size
calculations would wrap. In reality, frame_size is set by libspeex
according to the mode index specified in the file header, and is
usually equal to 320. Requires uintptr_t, checked by configure.

git-svn-id: http://svn.annodex.net/libfishsound/trunk@3900 8158c8cd-e7e1-0310-9fa4-c5954c97daef

config.h.in

@@ -60,6 +60,9 @@
 /* Define to 1 if you have the <sys/types.h> header file. */
 #undef HAVE_SYS_TYPES_H
 
+/* Define to 1 if the system has the type `uintptr_t'. */
+#undef HAVE_UINTPTR_T
+
 /* Define to 1 if you have the <unistd.h> header file. */
 #undef HAVE_UNISTD_H
 

configure.ac

@@ -20,6 +20,8 @@ AC_PROG_MAKE_SET
 
 AC_C_CONST
 AC_C_BIGENDIAN
+AC_CHECK_HEADERS([stdint.h])
+AC_CHECK_TYPES([uintptr_t])
 
 dnl Add parameters for aclocal
 AC_SUBST(ACLOCAL_AMFLAGS, "-I m4")

src/libfishsound/speex.c

@@ -36,6 +36,10 @@
 #include <stdlib.h>
 #include <string.h>
 
+#if HAVE_STDINT_H
+#include <stdint.h>
+#endif
+
 #include <ctype.h>
 
 #include "private.h"
@@ -290,6 +294,21 @@ fs_speex_decode (FishSound * fsound, unsigned char * buf, long bytes)
     fsound->info.samplerate = rate;
     fsound->info.channels = channels;
 
+    /* Sanity check the channels value, as we will use it to determine buffer
+       sizes below.
+     */
+    if (channels < 1 || channels > 2)
+      return FISH_SOUND_ERR_GENERIC;
+
+#if HAVE_UINTPTR_T
+    /* Sanity check: frame_size is not so large that the buffer size calculations
+     * would wrap. In reality, frame_size is set by libspeex according to the
+     * mode index specified in the file header, and is usually equal to 320.
+     */
+    if (fss->frame_size > UINTPTR_MAX / (sizeof(float) * channels))
+      return FISH_SOUND_ERR_GENERIC;
+#endif
+
     fss->ipcm = fs_malloc (sizeof (float) * fss->frame_size * channels);
     if (fss->ipcm == NULL) {
       return FISH_SOUND_ERR_OUT_OF_MEMORY;
@@ -646,6 +665,15 @@ fs_speex_update (FishSound * fsound, int interleave)
     if (fsound->info.channels == 1) {
       fss->pcm[0] = (float *) fss->ipcm;
     } else if (fsound->info.channels == 2) {
+#if HAVE_UINTPTR_T
+      /* Sanity check: frame_size is not so large that the buffer size calculations
+       * would wrap. In reality, frame_size is set by libspeex according to the
+       * mode index specified in the file header, and is usually equal to 320.
+       */
+      if (fss->frame_size > UINTPTR_MAX / pcm_size)
+        return FISH_SOUND_ERR_GENERIC;
+#endif
+
       pcm0 = fs_realloc (fss->pcm[0], pcm_size * fss->frame_size);
       if (pcm0 == NULL) {
         return FISH_SOUND_ERR_OUT_OF_MEMORY;

src/libfishsound/vorbis.c

@@ -113,7 +113,7 @@ fs_vorbis_decode (FishSound * fsound, unsigned char * buf, long bytes)
   FishSoundVorbisInfo * fsv = (FishSoundVorbisInfo *)fsound->codec_data;
   ogg_packet op;
   long samples;
-  float ** pcm_new;
+  float * pcm_new;
   int ret;
 
   /* Make an ogg_packet structure to pass the data to libvorbis */
@@ -343,7 +343,6 @@ fs_vorbis_encode_f (FishSound * fsound, float * pcm[], long frames)
   float ** vpcm;
   long len, remaining = frames;
   int i;
-  float ** ppcm = alloca (sizeof (float *) * fsound->info.channels);
 
   if (fsv->packetno == 0) {
     fs_vorbis_enc_headers (fsound);
@@ -354,10 +353,6 @@ fs_vorbis_encode_f (FishSound * fsound, float * pcm[], long frames)
     return 0;
   }
 
-  for (i = 0; i < fsound->info.channels; i++) {
-    ppcm[i] = pcm[i];
-  }
-
   while (remaining > 0) {
     len = MIN (1024, remaining);
 
@@ -369,7 +364,7 @@ fs_vorbis_encode_f (FishSound * fsound, float * pcm[], long frames)
     vpcm = vorbis_analysis_buffer (&fsv->vd, 1024);
 
     for (i = 0; i < fsound->info.channels; i++) {
-      memcpy (vpcm[i], ppcm[i], sizeof (float) * len);
+      memcpy (vpcm[i], pcm[i], sizeof (float) * len);
     }
 
     fs_vorbis_encode_write (fsound, len);