Skip to content
This repository
Browse code

Fixed bug related to the unpacking of mp4 file samples. Turns out eac…

…h sample can have one or more NAL units inside of it, this sometimes caused the first I-Frame to not be decoded properly.
  • Loading branch information...
commit b2f16267ecd3c6762d18f05ba4f02302bd01b2a5 1 parent b74caba
Michael Bebenita authored

Showing 1 changed file with 98 additions and 95 deletions. Show diff stats Hide diff stats

  1. +98 95 Player/mp4.js
193 Player/mp4.js
@@ -3,7 +3,7 @@
3 3 var Bytestream = (function BytestreamClosure() {
4 4 function constructor(arrayBuffer, start, length) {
5 5 this.bytes = new Uint8Array(arrayBuffer);
6   - this.start = start || 0;
  6 + this.start = start || 0;
7 7 this.pos = this.start;
8 8 this.end = (start + length) || this.bytes.length;
9 9 }
@@ -41,7 +41,7 @@ var Bytestream = (function BytestreamClosure() {
41 41 if (names) {
42 42 row = {};
43 43 for (var j = 0; j < cols; j++) {
44   - row[names[j]] = this.readU32();
  44 + row[names[j]] = this.readU32();
45 45 }
46 46 } else {
47 47 row = new Uint32Array(cols);
@@ -105,7 +105,7 @@ var Bytestream = (function BytestreamClosure() {
105 105 var pos = this.pos;
106 106 if (pos > this.end - 4)
107 107 return null;
108   - var res = "";
  108 + var res = "";
109 109 for (var i = 0; i < 4; i++) {
110 110 res += String.fromCharCode(this.bytes[pos + i]);
111 111 }
@@ -167,33 +167,33 @@ var PARANOID = true; // Heavy-weight assertions.
167 167
168 168 /**
169 169 * Reads an mp4 file and constructs a object graph that corresponds to the box/atom
170   - * structure of the file. Mp4 files are based on the ISO Base Media format, which in
  170 + * structure of the file. Mp4 files are based on the ISO Base Media format, which in
171 171 * turn is based on the Apple Quicktime format. The Quicktime spec is available at:
172   - * http://developer.apple.com/library/mac/#documentation/QuickTime/QTFF. An mp4 spec
173   - * also exists, but I cannot find it freely available.
174   - *
  172 + * http://developer.apple.com/library/mac/#documentation/QuickTime/QTFF. An mp4 spec
  173 + * also exists, but I cannot find it freely available.
  174 + *
175 175 * Mp4 files contain a tree of boxes (or atoms in Quicktime). The general structure
176 176 * is as follows (in a pseudo regex syntax):
177   - *
  177 + *
178 178 * Box / Atom Structure:
179   - *
  179 + *
180 180 * [size type [version flags] field* box*]
181 181 * <32> <4C> <--8--> <24-> <-?-> <?>
182 182 * <------------- box size ------------>
183   - *
  183 + *
184 184 * The box size indicates the entire size of the box and its children, we can use it
185   - * to skip over boxes that are of no interest. Each box has a type indicated by a
  185 + * to skip over boxes that are of no interest. Each box has a type indicated by a
186 186 * four character code (4C), this describes how the box should be parsed and is also
187   - * used as an object key name in the resulting box tree. For example, the expression:
  187 + * used as an object key name in the resulting box tree. For example, the expression:
188 188 * "moov.trak[0].mdia.minf" can be used to access individual boxes in the tree based
189 189 * on their 4C name. If two or more boxes with the same 4C name exist in a box, then
190   - * an array is built with that name.
191   - *
  190 + * an array is built with that name.
  191 + *
192 192 */
193 193 var MP4Reader = (function reader() {
194 194 var BOX_HEADER_SIZE = 8;
195 195 var FULL_BOX_HEADER_SIZE = BOX_HEADER_SIZE + 4;
196   -
  196 +
197 197 function constructor(stream) {
198 198 this.stream = stream;
199 199 this.tracks = {};
@@ -216,33 +216,33 @@ var MP4Reader = (function reader() {
216 216 },
217 217 readBox: function readBox(stream) {
218 218 var box = { offset: stream.position };
219   -
  219 +
220 220 function readHeader() {
221 221 box.size = stream.readU32();
222 222 box.type = stream.read4CC();
223 223 }
224   -
  224 +
225 225 function readFullHeader() {
226 226 box.version = stream.readU8();
227 227 box.flags = stream.readU24();
228 228 }
229   -
  229 +
230 230 function remainingBytes() {
231 231 return box.size - (stream.position - box.offset);
232 232 }
233   -
  233 +
234 234 function skipRemainingBytes () {
235 235 stream.skip(remainingBytes());
236 236 }
237   -
  237 +
238 238 var readRemainingBoxes = function () {
239 239 var subStream = stream.subStream(stream.position, remainingBytes());
240 240 this.readBoxes(subStream, box);
241   - stream.skip(subStream.length);
  241 + stream.skip(subStream.length);
242 242 }.bind(this);
243   -
  243 +
244 244 readHeader();
245   -
  245 +
246 246 switch (box.type) {
247 247 case 'ftyp':
248 248 box.name = "File Type Box";
@@ -366,7 +366,7 @@ var MP4Reader = (function reader() {
366 366 box.compressionId = stream.readU16();
367 367 box.packetSize = stream.readU16();
368 368 box.sampleRate = stream.readU32() >>> 16;
369   -
  369 +
370 370 // TODO: Parse other version levels.
371 371 assert (box.version == 0);
372 372 readRemainingBoxes();
@@ -416,7 +416,7 @@ var MP4Reader = (function reader() {
416 416 case 'stsc':
417 417 box.name = "Sample to Chunk Box";
418 418 readFullHeader();
419   - box.table = stream.readU32Array(stream.readU32(), 3,
  419 + box.table = stream.readU32Array(stream.readU32(), 3,
420 420 ["firstChunk", "samplesPerChunk", "sampleDescriptionId"]);
421 421 break;
422 422 case 'stsz':
@@ -459,20 +459,20 @@ var MP4Reader = (function reader() {
459 459 traceSamples: function () {
460 460 var video = this.tracks[1];
461 461 var audio = this.tracks[2];
462   -
  462 +
463 463 console.info("Video Samples: " + video.getSampleCount());
464 464 console.info("Audio Samples: " + audio.getSampleCount());
465   -
  465 +
466 466 var vi = 0;
467 467 var ai = 0;
468   -
  468 +
469 469 for (var i = 0; i < 100; i++) {
470 470 var vo = video.sampleToOffset(vi);
471 471 var ao = audio.sampleToOffset(ai);
472   -
  472 +
473 473 var vs = video.sampleToSize(vi, 1);
474 474 var as = audio.sampleToSize(ai, 1);
475   -
  475 +
476 476 if (vo < ao) {
477 477 console.info("V Sample " + vi + " Offset : " + vo + ", Size : " + vs);
478 478 vi ++;
@@ -491,7 +491,7 @@ var Track = (function track () {
491 491 this.file = file;
492 492 this.trak = trak;
493 493 }
494   -
  494 +
495 495 constructor.prototype = {
496 496 getSampleSizeTable: function () {
497 497 return this.trak.mdia.minf.stbl.stsz.table;
@@ -512,15 +512,15 @@ var Track = (function track () {
512 512 },
513 513 /**
514 514 * Computes the chunk that contains the specified sample, as well as the offset of
515   - * the sample in the computed chunk.
  515 + * the sample in the computed chunk.
516 516 */
517 517 sampleToChunk: function (sample) {
518   -
  518 +
519 519 /* Samples are grouped in chunks which may contain a variable number of samples.
520 520 * The sample-to-chunk table in the stsc box describes how samples are arranged
521   - * in chunks. Each table row corresponds to a set of consecutive chunks with the
  521 + * in chunks. Each table row corresponds to a set of consecutive chunks with the
522 522 * same number of samples and description ids. For example, the following table:
523   - *
  523 + *
524 524 * +-------------+-------------------+----------------------+
525 525 * | firstChunk | samplesPerChunk | sampleDescriptionId |
526 526 * +-------------+-------------------+----------------------+
@@ -528,29 +528,29 @@ var Track = (function track () {
528 528 * | 3 | 1 | 23 |
529 529 * | 5 | 1 | 24 |
530 530 * +-------------+-------------------+----------------------+
531   - *
  531 + *
532 532 * describes 5 chunks with a total of (2 * 3) + (2 * 1) + (1 * 1) = 9 samples,
533   - * each chunk containing samples 3, 3, 1, 1, 1 in chunk order, or
  533 + * each chunk containing samples 3, 3, 1, 1, 1 in chunk order, or
534 534 * chunks 1, 1, 1, 2, 2, 2, 3, 4, 5 in sample order.
535   - *
  535 + *
536 536 * This function determines the chunk that contains a specified sample by iterating
537 537 * over every entry in the table. It also returns the position of the sample in the
538 538 * chunk which can be used to compute the sample's exact position in the file.
539   - *
  539 + *
540 540 * TODO: Determine if we should memoize this function.
541 541 */
542   -
  542 +
543 543 var table = this.trak.mdia.minf.stbl.stsc.table;
544   -
  544 +
545 545 if (table.length === 1) {
546 546 var row = table[0];
547 547 assert (row.firstChunk === 1);
548 548 return {
549 549 index: sample / row.samplesPerChunk,
550 550 offset: sample % row.samplesPerChunk
551   - }
  551 + };
552 552 }
553   -
  553 +
554 554 var totalChunkCount = 0;
555 555 for (var i = 0; i < table.length; i++) {
556 556 var row = table[i];
@@ -572,7 +572,7 @@ var Track = (function track () {
572 572 offset: sample % previousRow.samplesPerChunk
573 573 };
574 574 }
575   - totalChunkCount += previousChunkCount;
  575 + totalChunkCount += previousChunkCount;
576 576 }
577 577 }
578 578 assert(false);
@@ -590,10 +590,10 @@ var Track = (function track () {
590 590 * Computes the sample at the specified time.
591 591 */
592 592 timeToSample: function (time) {
593   - /* In the time-to-sample table samples are grouped by their duration. The count field
  593 + /* In the time-to-sample table samples are grouped by their duration. The count field
594 594 * indicates the number of consecutive samples that have the same duration. For example,
595 595 * the following table:
596   - *
  596 + *
597 597 * +-------+-------+
598 598 * | count | delta |
599 599 * +-------+-------+
@@ -601,12 +601,12 @@ var Track = (function track () {
601 601 * | 2 | 1 |
602 602 * | 3 | 2 |
603 603 * +-------+-------+
604   - *
  604 + *
605 605 * describes 9 samples with a total time of (4 * 3) + (2 * 1) + (3 * 2) = 20.
606   - *
  606 + *
607 607 * This function determines the sample at the specified time by iterating over every
608 608 * entry in the table.
609   - *
  609 + *
610 610 * TODO: Determine if we should memoize this function.
611 611 */
612 612 var table = this.trak.mdia.minf.stbl.stts.table;
@@ -642,7 +642,7 @@ var Track = (function track () {
642 642 return this.trak.mdia.mdhd.timeScale;
643 643 },
644 644 /**
645   - * Converts time units to real time (seconds).
  645 + * Converts time units to real time (seconds).
646 646 */
647 647 timeToSeconds: function (time) {
648 648 return time / this.getTimeScale();
@@ -657,37 +657,34 @@ var Track = (function track () {
657 657 /*
658 658 for (var i = 0; i < this.getSampleCount(); i++) {
659 659 var res = this.sampleToChunk(i);
660   - console.info("Sample " + i + " -> " + res.index + " % " + res.offset +
661   - " @ " + this.chunkToOffset(res.index) +
  660 + console.info("Sample " + i + " -> " + res.index + " % " + res.offset +
  661 + " @ " + this.chunkToOffset(res.index) +
662 662 " @@ " + this.sampleToOffset(i));
663 663 }
664 664 console.info("Total Time: " + this.timeToSeconds(this.getTotalTime()));
665 665 var total = this.getTotalTimeInSeconds();
666 666 for (var i = 50; i < total; i += 0.1) {
667   - // console.info("Time: " + i.toFixed(2) + " " + this.secondsToTime(i));
668   -
  667 + // console.info("Time: " + i.toFixed(2) + " " + this.secondsToTime(i));
  668 +
669 669 console.info("Time: " + i.toFixed(2) + " " + this.timeToSample(this.secondsToTime(i)));
670 670 }
671 671 */
672 672 },
673 673 /**
674   - * Video samples in AVC file format are framed with a start prefix that
675   - * indicates the length of the sample. This function only returns the contained
676   - * NAL unit without the length prefix.
  674 + * AVC samples contain one or more NAL units each of which have a length prefix.
  675 + * This function returns an array of NAL units without their length prefixes.
677 676 */
678   - getSampleBytes: function (sample, withoutLengthPrefix) {
  677 + getSampleNALUnits: function (sample) {
679 678 var bytes = this.file.stream.bytes;
680 679 var offset = this.sampleToOffset(sample);
681   - if (withoutLengthPrefix) {
682   - var size = this.sampleToSize(sample, 1);
683   - if (PARANOID) {
684   - var prefix = (new Bytestream(bytes.buffer, offset)).readU32();
685   - assert (size >= prefix + 4);
686   -// return bytes.subarray(offset + 4, offset + prefix + 4);
687   - }
688   - return bytes.subarray(offset + 4, offset + size);
  680 + var end = offset + this.sampleToSize(sample, 1);
  681 + var nalUnits = [];
  682 + while(end - offset > 0) {
  683 + var length = (new Bytestream(bytes.buffer, offset)).readU32();
  684 + nalUnits.push(bytes.subarray(offset + 4, offset + length + 4));
  685 + offset = offset + length + 4;
689 686 }
690   - return bytes.subarray(offset, offset + this.sampleToSize(sample, 1));
  687 + return nalUnits;
691 688 }
692 689 };
693 690 return constructor;
@@ -731,7 +728,7 @@ var MP4Player = (function reader() {
731 728 filterVerLumaEdge: "optimized",
732 729 getBoundaryStrengthsA: "optimized"
733 730 };
734   -
  731 +
735 732 function constructor(stream, canvas, useWorkers, render) {
736 733 this.canvas = canvas;
737 734 this.webGLCanvas = null;
@@ -749,11 +746,11 @@ var MP4Player = (function reader() {
749 746 fpsMax: -1000,
750 747 webGLTextureUploadTime: 0
751 748 };
752   -
  749 +
753 750 this.onStatisticsUpdated = function () {};
754   -
  751 +
755 752 if (this.useWorkers) {
756   - this.avcWorker = new WorkerSocket("avc-worker.js");
  753 + this.avcWorker = new WorkerSocket("avc-worker.js");
757 754 this.avcWorker.onReceiveMessage("console.info", function (message) {
758 755 console.info("AVC Worker Says: " + message.payload);
759 756 });
@@ -787,7 +784,7 @@ var MP4Player = (function reader() {
787 784 if (videoElapsedTime < 1000) {
788 785 return;
789 786 }
790   -
  787 +
791 788 if (!s.windowStartTime) {
792 789 s.windowStartTime = now;
793 790 return;
@@ -796,33 +793,33 @@ var MP4Player = (function reader() {
796 793 var fps = (s.windowPictureCounter / windowElapsedTime) * 1000;
797 794 s.windowStartTime = now;
798 795 s.windowPictureCounter = 0;
799   -
  796 +
800 797 if (fps < s.fpsMin) s.fpsMin = fps;
801 798 if (fps > s.fpsMax) s.fpsMax = fps;
802 799 s.fps = fps;
803 800 }
804   -
  801 +
805 802 var fps = (s.videoPictureCounter / videoElapsedTime) * 1000;
806 803 s.fpsSinceStart = fps;
807 804 this.onStatisticsUpdated(this.statistics);
808 805 return ;
809 806 }
810   -
  807 +
811 808 function onPictureDecoded(buffer, width, height) {
812 809 updateStatistics.call(this);
813   -
  810 +
814 811 if (!buffer || !this.render) {
815 812 return;
816 813 }
817 814 var lumaSize = width * height;
818 815 var chromaSize = lumaSize >> 2;
819   -
  816 +
820 817 this.webGLCanvas.YTexture.fill(buffer.subarray(0, lumaSize));
821 818 this.webGLCanvas.UTexture.fill(buffer.subarray(lumaSize, lumaSize + chromaSize));
822 819 this.webGLCanvas.VTexture.fill(buffer.subarray(lumaSize + chromaSize, lumaSize + 2 * chromaSize));
823 820 this.webGLCanvas.drawScene();
824 821 }
825   -
  822 +
826 823 constructor.prototype = {
827 824 readAll: function(callback) {
828 825 console.info("MP4Player::readAll()");
@@ -837,23 +834,23 @@ var MP4Player = (function reader() {
837 834 },
838 835 play: function() {
839 836 var reader = this.reader;
840   -
  837 +
841 838 if (!reader) {
842 839 this.readAll(this.play.bind(this));
843 840 return;
844 841 } else {
845 842 this.canvas.width = this.size.w;
846 843 this.canvas.height = this.size.h;
847   - this.webGLCanvas = new YUVWebGLCanvas(this.canvas, this.size);
  844 + this.webGLCanvas = new YUVWebGLCanvas(this.canvas, this.size);
848 845 }
849   -
  846 +
850 847 var video = reader.tracks[1];
851 848 var audio = reader.tracks[2];
852   -
  849 +
853 850 var avc = reader.tracks[1].trak.mdia.minf.stbl.stsd.avc1.avcC;
854 851 var sps = avc.sps[0];
855 852 var pps = avc.pps[0];
856   -
  853 +
857 854 /* Decode Sequence & Picture Parameter Sets */
858 855 if (this.useWorkers) {
859 856 this.avcWorker.sendMessage("decode-sample", sps);
@@ -867,18 +864,24 @@ var MP4Player = (function reader() {
867 864 var pic = 0;
868 865 setTimeout(function foo() {
869 866 if (this.useWorkers) {
870   - this.avcWorker.sendMessage("decode-sample", video.getSampleBytes(pic, true));
  867 + var avcWorker = this.avcWorker;
  868 + video.getSampleNALUnits(pic).forEach(function (nal) {
  869 + avcWorker.sendMessage("decode-sample", nal);
  870 + });
871 871 } else {
872   - this.avc.decode(video.getSampleBytes(pic, true));
  872 + var avc = this.avc;
  873 + video.getSampleNALUnits(pic).forEach(function (nal) {
  874 + avc.decode(nal);
  875 + });
873 876 }
874 877 pic ++;
875 878 if (pic < 3000) {
876 879 setTimeout(foo.bind(this), 1);
877   - }
  880 + }
878 881 }.bind(this), 1);
879 882 }
880   - }
881   -
  883 + };
  884 +
882 885 return constructor;
883 886 })();
884 887
@@ -894,7 +897,7 @@ var Broadway = (function broadway() {
894 897 this.canvas.onclick = function () {
895 898 this.play();
896 899 }.bind(this);
897   -
  900 +
898 901 div.appendChild(this.canvas);
899 902 var controls = document.createElement('div');
900 903 controls.setAttribute('style', "z-index: 100; position: absolute; bottom: 0px; background-color: rgba(0,0,0,0.8); height: 30px; width: 100%; text-align: left;");
@@ -902,12 +905,12 @@ var Broadway = (function broadway() {
902 905 this.info.setAttribute('style', "font-size: 14px; font-weight: bold; padding: 6px; color: lime;");
903 906 controls.appendChild(this.info);
904 907 div.appendChild(controls);
905   -
  908 +
906 909 var useWorkers = div.attributes.workers ? div.attributes.workers.value == "true" : true;
907 910 var render = div.attributes.render ? div.attributes.render.value == "true" : true;
908   -
  911 +
909 912 this.player = new MP4Player(new Stream(src), this.canvas, useWorkers, render);
910   -
  913 +
911 914 this.score = null;
912 915 this.player.onStatisticsUpdated = function (statistics) {
913 916 if (statistics.videoPictureCounter % 10 != 0) {
@@ -922,12 +925,12 @@ var Broadway = (function broadway() {
922 925 }
923 926 var scoreCutoff = 1200;
924 927 if (statistics.videoPictureCounter < scoreCutoff) {
925   - this.score = scoreCutoff - statistics.videoPictureCounter;
  928 + this.score = scoreCutoff - statistics.videoPictureCounter;
926 929 } else if (statistics.videoPictureCounter == scoreCutoff) {
927 930 this.score = statistics.fpsSinceStart.toFixed(2);
928 931 }
929 932 // info += " score: " + this.score;
930   -
  933 +
931 934 this.info.innerHTML = info;
932 935 }.bind(this);
933 936 }
@@ -937,4 +940,4 @@ var Broadway = (function broadway() {
937 940 }
938 941 };
939 942 return constructor;
940   -})();
  943 +})();

0 comments on commit b2f1626

Please sign in to comment.
Something went wrong with that request. Please try again.