Bug 1035170 - Implement lazy SWF parsing

This is a monster-patch as it's the result of squashing together 81 individual patches. It contains a full refactoring of how SWF files are parsed and the results managed.

In the new system, SWFs aren't deeply parsed and all symbols and other tags extracted. Instead, they're scanned through, detecting all symbols, frames, and code blocks. The player is notified of these tags, but they're only deeply parsed upon first usage. Even then, the `SWFFile` class that represents the loaded SWF doesn't hold on to the parsed tag. It is up to the playback system to decide whether that's required or not. For now, it does happen in most cases, but we might decide to hold most parsing results temporarily only.

In the following, I tried to include the important commit messages for the original patches squashed down into this one. The original patches themselves are available in [a branch](https://github.com/tschneidereit/shumway/tree/parser-unsquashed) in @tschneidereit's fork.

Bug fix in SoundStreamHead: StreamSoundSampleCount field is a UI16, not UI32

Disable usage of DOM Workers in fakeSyncWorker.ts for now
This creates lots and lots of IO events, overwhelming browsers in some cases. Also, it most certainly imposes substantial overhead.

Move initialization of loaded frames from Loader into MovieClip

Null out the SWFFile's decompressor after loading is completed, to free its memory.

Don't treat BMPs as eagerly-decoded images

Increases height of profiler output panel to accomodate parser timeline

Remove some somewhatImplemented warnings
These got in the way of trace tests after some changes to how logging of warnings works.

Add timeouts to the GNU parallel command. (Original patch by @mbebenita)

Add simple DOM stubs for Image, URL and Blob to the shell testing harness. (Original patch by @mbebenita)

Adds profiling option -o to the shell driver. This let's you dumps out timeline events to stdout when verbose mode is also on. (Original patch by @mbebenita)

Optimize DataBuffer access by providing methods for writing multiple values at the same time. Also, lazify the way views are allocated since not all views are always needed. (Original patch by @mbebenita)

Switches AVM1Utils.addEventListenerProxy to a native implementation

Optimize parsing of AVM1 ClipEvents
And moved their application to avm1lib. Before, it was partly in FrameDelta#_initialize, partly in the late Sprite#_initAVM1Bindings.
This also fixes the mapping of some of the ClipEvents to equivalent AVM2 events. Some aren't trivially mapped, so more work has to be done there. Those will show a warning at least when they're used.

Add new test for mouse events on nested AVM1 objects and reactivate some existing tests
The existing tests where disabled before the nat branch merge and then forgotten about.

Give PromiseWrapper a `then` method

Delay committing Loader updates until the right point in the event loop

Support initialization from Uint8Array in source instead of special-casing BinarySymbol in ByteArray initializer

Change how resolved symbols get stored on ASClasses
Instead of having the Symbol ctor do that, the symbolResolver now does it. Not only is that cleaner, it's also required to make embedded images work: for those, the symbol is associated with a subclass of `Bitmap`, but we create a `BitmapSymbol` for the actual data. This was then stored on `flash.display.BitmapData`. Not ideal.

Fix a few assert guards to let the assert execute in non-release mode instead of release mode

Don't take 3 screenshots of an unchanging reftest

Add parser tracing output for eagerly parsed images and fonts

Gruntfile.js

@@ -126,20 +126,20 @@ module.exports = function(grunt) {
       },
       smoke_parse_database: {
         maxBuffer: Infinity,
-        cmd: 'find -L test/swf -name "*.swf" | parallel -X -N100 utils/jsshell/js build/ts/shell.js -p -r -po {} >> data.json.txt'
+        cmd: 'find -L test/swf -name "*.swf" | parallel --no-notice -X -N50 --timeout 200% utils/jsshell/js build/ts/shell.js -p -r -po {} >> data.json.txt'
       },
       smoke_play: {
         maxBuffer: Infinity,
-        cmd: 'find -L test/swf -name "*.swf" | parallel -X -N100 utils/jsshell/js build/ts/shell.js -x -md 1000 -v -tp 60 -v {}'
+        cmd: 'find -L test/swf -name "*.swf" | parallel --no-notice -X -N50 --timeout 200% utils/jsshell/js build/ts/shell.js -x -md 1000 -v -tp 60 -v {}'
       },
       smoke_parse: {
         maxBuffer: Infinity,
-        cmd: 'find -L test/swf -name "*.swf" | parallel -X -N100 utils/jsshell/js build/ts/shell.js -p -r ' +
+        cmd: 'find -L test/swf -name "*.swf" | parallel --no-notice -X -N50 --timeout 200% utils/jsshell/js build/ts/shell.js -p -r ' +
              (grunt.option('verbose') ? '-v ' : '') + ' {}'
       },
       smoke_parse_images: {
         maxBuffer: Infinity,
-        cmd: 'find -L test/swf -name "*.swf" | parallel -X -N100 utils/jsshell/js build/ts/shell.js -p -r -f "CODE_DEFINE_BITS,CODE_DEFINE_BITS_JPEG2,CODE_DEFINE_BITS_JPEG3,CODE_DEFINE_BITS_JPEG4,CODE_JPEG_TABLES,CODE_DEFINE_BITS_LOSSLESS,CODE_DEFINE_BITS_LOSSLESS2" {}'
+        cmd: 'find -L test/swf -name "*.swf" | parallel --no-notice -X -N50 --timeout 200% utils/jsshell/js build/ts/shell.js -p -r -f "CODE_DEFINE_BITS,CODE_DEFINE_BITS_JPEG2,CODE_DEFINE_BITS_JPEG3,CODE_DEFINE_BITS_JPEG4,CODE_JPEG_TABLES,CODE_DEFINE_BITS_LOSSLESS,CODE_DEFINE_BITS_LOSSLESS2" {}'
       },
       closure: {
         // This needs a special build of closure that has SHUMWAY_OPTIMIZATIONS.

examples/inspector/style/style.css

@@ -36,7 +36,7 @@ body.simple #easelContainer {
   right: 0;
   left: 0;
   width: auto;
-  height: 332px;
+  height: 380px;
   overflow: hidden;
   background-color: #000;
 }

src/TextContent.ts

@@ -411,10 +411,11 @@ module Shumway {
       textRunData.writeInt(size);
 
       var font = flash.text.Font.getByName(textFormat.font) || flash.text.Font.getDefaultFont();
+      // TODO: ensure that font serialization really isn't required and clean this up.
+      textRunData.writeInt(0);
       if (font.fontType === flash.text.FontType.EMBEDDED) {
-        textRunData.writeInt(font._id);
+        textRunData.writeUTF('swffont' + font._id);
       } else {
-        textRunData.writeInt(0);
         textRunData.writeUTF(font._fontFamily);
       }
       textRunData.writeInt(font.ascent * size);

src/avm1/context.ts

@@ -36,15 +36,14 @@ module Shumway.AVM1 {
   export class AVM1Context {
     public static instance: AVM1Context = null;
     public root: AVM1MovieClip;
-    public swfVersion: number;
+    public loaderInfo: Shumway.AVM2.AS.flash.display.LoaderInfo;
     public globals: AVM1Globals;
     constructor() {
       this.root = null;
-      this.swfVersion = 0;
       this.globals = null;
     }
 
-    public static create: (swfVersion: number) => AVM1Context;
+    public static create: (loaderInfo: Shumway.AVM2.AS.flash.display.LoaderInfo) => AVM1Context;
 
     public flushPendingScripts() {}
     public addAsset(className: string, symbolId: number, symbolProps) {}

src/avm1/interpreter.ts

@@ -132,7 +132,6 @@ module Shumway.AVM1 {
   }
 
   class AVM1ContextImpl extends AVM1Context {
-    swfVersion: number;
     initialScope: AVM1ScopeListItem;
     isActive: boolean;
     executionProhibited: boolean;
@@ -149,16 +148,17 @@ module Shumway.AVM1 {
     private assetsSymbols: Array<any>;
     private assetsClasses: Array<any>;
 
-    constructor(swfVersion: number) {
+    constructor(loaderInfo: Shumway.AVM2.AS.flash.display.LoaderInfo) {
       super();
-      this.swfVersion = swfVersion;
+      this.loaderInfo = loaderInfo;
       this.globals = new Shumway.AVM2.AS.avm1lib.AVM1Globals();
-      if (swfVersion >= 8) {
+      if (loaderInfo.swfVersion >= 8) {
         this.globals.asSetPublicProperty("flash",
           Shumway.AVM2.AS.avm1lib.createFlashObject());
       }
       this.initialScope = new AVM1ScopeListItem(this.globals, null);
       this.assets = {};
+      // TODO: remove this list and always retrieve symbols from LoaderInfo.
       this.assetsSymbols = [];
       this.assetsClasses = [];
       this.isActive = false;
@@ -172,9 +172,6 @@ module Shumway.AVM1 {
     }
     addAsset(className: string, symbolId: number, symbolProps) {
       this.assets[className] = symbolId;
-      if (this.assetsSymbols[symbolId]) {
-        Debug.warning('Symbol ' + symbolId + ' was exported already under different name');
-      }
       this.assetsSymbols[symbolId] = symbolProps;
 
     }
@@ -191,9 +188,18 @@ module Shumway.AVM1 {
       if (symbolId === undefined) {
         return undefined;
       }
+      var symbol = this.assetsSymbols[symbolId];
+      if (!symbol) {
+        symbol = this.loaderInfo.getSymbolById(symbolId);
+        if (!symbol) {
+          Debug.warning("Symbol " + symbolId + " is not defined.");
+          return undefined;
+        }
+        this.assetsSymbols[symbolId] = symbol;
+      }
       return {
         symbolId: symbolId,
-        symbolProps: this.assetsSymbols[symbolId],
+        symbolProps: symbol,
         theClass: this.assetsClasses[symbolId]
       };
     }
@@ -241,8 +247,8 @@ module Shumway.AVM1 {
     }
   }
 
-  AVM1Context.create = function (swfVersion: number): AVM1Context {
-    return new AVM1ContextImpl(swfVersion);
+  AVM1Context.create = function(loaderInfo: Shumway.AVM2.AS.flash.display.LoaderInfo): AVM1Context {
+    return new AVM1ContextImpl(loaderInfo);
   };
 
   class AVM1Error {
@@ -280,7 +286,7 @@ module Shumway.AVM1 {
   }
 
   function as2GetCurrentSwfVersion() : number {
-    return (<AVM1ContextImpl> AVM1Context.instance).swfVersion;
+    return AVM1Context.instance.loaderInfo.swfVersion;
   }
 
   function as2ToAddPrimitive(value) {
@@ -2490,7 +2496,7 @@ module Shumway.AVM1 {
       var currentContext = <AVM1ContextImpl> AVM1Context.instance;
 
       if (!actionsData.ir) {
-        var stream = new ActionsDataStream(actionsData.bytes, currentContext.swfVersion);
+        var stream = new ActionsDataStream(actionsData.bytes, currentContext.loaderInfo.swfVersion);
         var parser = new ActionsDataParser(stream);
         parser.dataId = actionsData.id;
         var analyzer = new ActionsDataAnalyzer();
@@ -2509,7 +2515,7 @@ module Shumway.AVM1 {
       var compiled: Function = (<any> ir).compiled;
 
       var stack = [];
-      var isSwfVersion5 = currentContext.swfVersion >= 5;
+      var isSwfVersion5 = currentContext.loaderInfo.swfVersion >= 5;
       var actionTracer = ActionTracerFactory.get();
       var scope = scopeContainer.scope;
 

src/avm1lib/AVM1Button.ts

@@ -70,13 +70,13 @@ module Shumway.AVM2.AS.avm1lib {
     _init(nativeButton: flash.display.SimpleButton): any {
       this._nativeAS3Object = nativeButton;
       initDefaultListeners(this);
-      if (!nativeButton._symbol || !nativeButton._symbol.buttonActions) {
+      if (!nativeButton._symbol || !nativeButton._symbol.data.buttonActions) {
         return;
       }
       this._nativeAS3Object.addEventListener('addedToStage', this._addListeners.bind(this));
       this._nativeAS3Object.addEventListener('removedFromStage', this._removeListeners.bind(this));
       var requiredListeners = this._requiredListeners = Object.create(null);
-      var actions = this._actions = nativeButton._symbol.buttonActions;
+      var actions = this._actions = nativeButton._symbol.data.buttonActions;
       for (var i = 0; i < actions.length; i++) {
         var action = actions[i];
         if (!action.actionsBlock) {

src/avm1lib/AVM1Globals.ts

@@ -159,7 +159,7 @@ module Shumway.AVM2.AS.avm1lib {
      */
     escape(str: string): string {
       var result = encodeURIComponent(str);
-      return result.replace(/\!|'|\(|\)|\*|-|\.|_|~/g, function(char: string): string {
+      return result.replace(/!|'|\(|\)|\*|-|\.|_|~/g, function(char: string): string {
         switch (char) {
           case '*':
             return '%2A';

src/avm1lib/AVM1Utils.as

@@ -67,10 +67,7 @@ package avm1lib {
       return path;
     }
 
-    public static function addEventHandlerProxy(obj: Object, propertyName: String, eventName: String, argsConverter: Function = null)
-    {
-      _addEventHandlerProxy(obj, propertyName, eventName, argsConverter);
-    }
+    public static native function addEventHandlerProxy(obj: Object, propertyName: String, eventName: String, argsConverter: Function = null);
 
     private static native function _installObjectMethods();
 
@@ -79,54 +76,3 @@ package avm1lib {
     }
   }
 }
-
-import avm1lib.AVM1Utils;
-
-AVM1Utils;
-
-function _addEventHandlerProxy(obj: Object, propertyName: String, eventName: String, argsConverter: Function)
-{
-  var currentHandler: Function = null;
-  var handlerRunner: Function = null;
-  AVM1Utils.addProperty(obj, propertyName,
-    function(): Function {
-      return currentHandler;
-    },
-    function setter(newHandler: Function) {
-      if (!this._as3Object) { // prototype/class ?
-        var defaultListeners = this._as2DefaultListeners ||
-          (this._as2DefaultListeners = []);
-        defaultListeners.push({setter: setter, value: newHandler});
-        // see also initDefaultListeners()
-        return;
-      }
-      // AVM1 MovieClips don't receive roll/release events by default until they set one of the following properties.
-      // This behaviour gets triggered whenever those properties are set, despite of the actual value they are set to.
-      if (propertyName === 'onRelease' ||
-          propertyName === 'onReleaseOutside' ||
-          propertyName === 'onRollOut' ||
-          propertyName === 'onRollOver') {
-        this._as3Object.mouseEnabled = true;
-        this._as3Object.buttonMode = true;
-      }
-      if (currentHandler === newHandler) {
-        return;
-      }
-      if (currentHandler != null) {
-        this._as3Object.removeEventListener(eventName, handlerRunner);
-      }
-      currentHandler = newHandler;
-      if (currentHandler != null) {
-        handlerRunner = (function (obj: Object, handler: Function) {
-          return function handlerRunner() {
-            var args = argsConverter != null ? argsConverter(arguments) : null;
-            return handler.apply(obj, args);
-          };
-        })(this, currentHandler);
-        this._as3Object.addEventListener(eventName, handlerRunner);
-      } else {
-        handlerRunner = null;
-      }
-    },
-    false);
-}