opusenc: Validate AIFF/WAV channel count

Interpret channel count properly for input format (signed for AIFF,
unsigned for WAV) and reject invalid values.

Fixes https://trac.xiph.org/ticket/2136 (CVE-2014-9639),
https://trac.xiph.org/ticket/2137 (CVE-2014-9638).

src/audio-in.c

@@ -321,11 +321,17 @@ int aiff_open(FILE *in, oe_enc_opt *opt, unsigned char *buf, int buflen)
         return 0;
     }
 
-    format.channels = READ_U16_BE(buffer);
+    format.channels = (short)READ_U16_BE(buffer);
     format.totalframes = READ_U32_BE(buffer+2);
     format.samplesize = READ_U16_BE(buffer+6);
     format.rate = (int)read_IEEE80(buffer+8);
 
+    if(format.channels <= 0)
+    {
+        fprintf(stderr, _("ERROR: Invalid channel count in AIFF header\n"));
+        return 0;
+    }
+
     if(aifc)
     {
         if(len < 22)
@@ -487,6 +493,12 @@ int wav_open(FILE *in, oe_enc_opt *opt, unsigned char *oldbuf, int buflen)
     format.align =       READ_U16_LE(buf+12);
     format.samplesize =  READ_U16_LE(buf+14);
 
+    if(format.channels == 0)
+    {
+        fprintf(stderr, _("ERROR: Zero channels in WAV header\n"));
+        return 0;
+    }
+
     if(format.format == -2) /* WAVE_FORMAT_EXTENSIBLE */
     {
       if(len<40)

src/opusenc.h

@@ -61,7 +61,7 @@ typedef struct
 
 typedef struct {
     short format;
-    short channels;
+    unsigned short channels;
     int samplerate;
     int bytespersec;
     short align;
@@ -70,7 +70,7 @@ typedef struct {
 } wav_fmt;
 
 typedef struct {
-    short channels;
+    unsigned short channels;
     short samplesize;
     opus_int64 totalsamples;
     opus_int64 samplesread;