Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

compress all data files into one big file

  • Loading branch information...
commit 825407da4f8374155749852255a71505be9550ee 1 parent c0911ab
@kripken kripken authored
Showing with 75 additions and 30 deletions.
  1. +62 −21 emcc
  2. +13 −9 tests/runner.py
View
83 emcc
@@ -826,6 +826,9 @@ try:
if DEBUG: print >> sys.stderr, 'emcc: setting up files'
code = ''
+ code += 'var BlobBuilder = typeof MozBlobBuilder != "undefined" ? MozBlobBuilder : (typeof WebKitBlobBuilder != "undefined" ? WebKitBlobBuilder : console.log("warning: cannot build blobs"));\n'
+ code += 'var URLObject = typeof window != "undefined" ? (window.URL ? window.URL : window.webkitURL) : console.log("warning: cannot create object URLs");\n'
+
# Expand directories into individual files
def add(mode, dirname, names):
for name in names:
@@ -840,19 +843,8 @@ try:
for file_ in data_files:
file_['net_name'] = file_['name']
- file_['compressed'] = False
- if Compression.on:
- # Compress each file, if it is worth it
- for file_ in data_files:
- Compression.compress(file_['name'])
- if Compression.worth_it(os.stat(file_['name']).st_size,
- os.stat(Compression.compressed_name(file_['name'])).st_size):
- file_['net_name'] = Compression.compressed_name(file_['name'])
- file_['compressed'] = True
- else:
- if DEBUG: print >> sys.stderr, 'emcc: not compressing %s since not worth it' % file_['name']
- os.remove(Compression.compressed_name(file_['name']))
+ data_target = unsuffixed(target) + '.data'
# Set up folders
partial_dirs = []
@@ -866,8 +858,30 @@ try:
code += '''FS.createFolder('/%s', '%s', true, false);\n''' % (os.path.sep.join(parts[:i]), parts[i])
partial_dirs.append(partial)
- code += 'var BlobBuilder = typeof MozBlobBuilder != "undefined" ? MozBlobBuilder : (typeof WebKitBlobBuilder != "undefined" ? WebKitBlobBuilder : console.log("warning: cannot build blobs"));\n'
- code += 'var URLObject = typeof window != "undefined" ? (window.URL ? window.URL : window.webkitURL) : console.log("warning: cannot create object URLs");\n'
+ if Compression.on:
+ # Compress all datafiles into one archive
+ data = open(data_target, 'wb')
+ start = 0
+ for file_ in data_files:
+ file_['compression_start'] = start
+ curr = open(file_['name']).read()
+ file_['compression_end'] = start + len(curr)
+ start += len(curr)
+ data.write(curr)
+ data.close()
+ Compression.compress(data_target)
+
+ # Decompression requests have a similar API to XHRs
+ code += '''
+ function DecompressionRequest() {}
+ DecompressionRequest.prototype = {
+ requests: {},
+ open: function(mode, name) {
+ this.requests[name] = this;
+ },
+ send: function() {}
+ };
+ '''
counter = 0
for file_ in data_files:
@@ -881,28 +895,25 @@ try:
counter += 1
image = filename.endswith(IMAGE_SUFFIXES)
code += '''
- var %(varname)s = new XMLHttpRequest();
+ var %(varname)s = new %(request)s();
%(varname)s.open('GET', '%(netname)s', true);
%(varname)s.responseType = 'arraybuffer';
%(varname)s.onload = function() {
- var arrayBuffer = %(varname)s.response; // Note: not X.responseText
+ var arrayBuffer = %(varname)s.response;
assert(arrayBuffer, 'Loading file %(filename)s failed.');
- var byteArray = new Uint8Array(arrayBuffer);
- %(decompress_start)s
+ var byteArray = arrayBuffer.byteLength ? new Uint8Array(arrayBuffer) : arrayBuffer;
FS.createDataFile('/%(dirname)s', '%(basename)s', byteArray, true, true);
%(finish)s
- %(decompress_end)s
};
addRunDependency();
%(varname)s.send(null);
''' % {
+ 'request': 'XMLHttpRequest' if not Compression.on else 'DecompressionRequest',
'varname': varname,
'filename': filename,
'netname': file_['net_name'],
'dirname': os.path.dirname(filename),
'basename': os.path.basename(filename),
- 'decompress_start': '' if not file_['compressed'] else 'Module["decompress"](byteArray, function(decompressed) { byteArray = new Uint8Array(decompressed);',
- 'decompress_end': '' if not file_['compressed'] else '});',
'finish': 'removeRunDependency();' if not image else '''var bb = new BlobBuilder();
bb.append(byteArray.buffer);
var b = bb.getBlob();
@@ -927,6 +938,36 @@ try:
}
else:
assert 0
+
+ if Compression.on:
+ use_decompressed = ''
+ for file_ in data_files:
+ if file_['mode'] == 'preload':
+ use_decompressed += '''
+ curr = DecompressionRequest.prototype.requests['%s'];
+ curr.response = byteArray.subarray(%d,%d);
+ curr.onload();
+ ''' % (file_['name'], file_['compression_start'], file_['compression_end'])
+
+ code += '''
+ var dataFile = new XMLHttpRequest();
+ dataFile.open('GET', '%s', true);
+ dataFile.responseType = 'arraybuffer';
+ dataFile.onload = function() {
+ var arrayBuffer = dataFile.response;
+ assert(arrayBuffer, 'Loading data file failed.');
+ var byteArray = new Uint8Array(arrayBuffer);
+ Module["decompress"](byteArray, function(decompressed) {
+ byteArray = new Uint8Array(decompressed);
+ var curr;
+ %s
+ removeRunDependency();
+ });
+ };
+ addRunDependency();
+ dataFile.send(null);
+ ''' % (Compression.compressed_name(data_target), use_decompressed)
+
src = open(final).read().replace('// {{PRE_RUN_ADDITIONS}}', code)
final += '.files.js'
open(final, 'w').write(src)
View
22 tests/runner.py
@@ -6320,6 +6320,7 @@ def test_multifile(self):
def test_compressed_file(self):
open(os.path.join(self.get_dir(), 'datafile.txt'), 'w').write('compress this please' + (2000*'.'))
+ open(os.path.join(self.get_dir(), 'datafile2.txt'), 'w').write('moar' + (100*'!'))
open(os.path.join(self.get_dir(), 'main.cpp'), 'w').write(self.with_report_result(r'''
#include <stdio.h>
#include <string.h>
@@ -6332,17 +6333,23 @@ def test_compressed_file(self):
fclose(f);
printf("file says: |%s|\n", buf);
int result = !strcmp("compress this please", buf);
+ FILE *f2 = fopen("datafile2.txt", "r");
+ fread(buf, 1, 5, f2);
+ buf[5] = 0;
+ fclose(f2);
+ result = result && !strcmp("moar!", buf);
+ printf("file 2 says: |%s|\n", buf);
REPORT_RESULT();
return 0;
}
'''))
- Popen(['python', EMCC, os.path.join(self.get_dir(), 'main.cpp'), '-o', 'page.html', '--preload-file', 'datafile.txt',
+ Popen(['python', EMCC, os.path.join(self.get_dir(), 'main.cpp'), '-o', 'page.html', '--preload-file', 'datafile.txt', '--preload-file', 'datafile2.txt',
'--compression', '%s,%s,%s' % (path_from_root('third_party', 'lzma.js', 'lzma-native'),
path_from_root('third_party', 'lzma.js', 'lzma-decoder.js'),
'LZMA.decompress')]).communicate()
assert os.path.exists(os.path.join(self.get_dir(), 'datafile.txt')), 'must be data file'
- assert os.path.exists(os.path.join(self.get_dir(), 'datafile.txt.compress')), 'must be data file in compressed form'
+ assert os.path.exists(os.path.join(self.get_dir(), 'page.data.compress')), 'must be data file in compressed form'
assert os.stat(os.path.join(self.get_dir(), 'page.js')).st_size != os.stat(os.path.join(self.get_dir(), 'page.js.compress')).st_size, 'compressed file must be different'
shutil.move(os.path.join(self.get_dir(), 'datafile.txt'), 'datafile.txt.renamedsoitcannotbefound');
self.run_browser('page.html', '', '/report_result?1')
@@ -6356,10 +6363,10 @@ def test_sdl_image(self):
self.run_browser('page.html', '', '/report_result?600')
def test_sdl_image_compressed(self):
- for image, worth_compressing, width in [(path_from_root('tests', 'screenshot2.png'), True, 300),
- (path_from_root('tests', 'screenshot.jpg'), False, 600)]:
+ for image, width in [(path_from_root('tests', 'screenshot2.png'), 300),
+ (path_from_root('tests', 'screenshot.jpg'), 600)]:
self.clear()
- print image, worth_compressing
+ print image
basename = os.path.basename(image)
shutil.copyfile(image, os.path.join(self.get_dir(), basename))
@@ -6369,10 +6376,7 @@ def test_sdl_image_compressed(self):
'--compression', '%s,%s,%s' % (path_from_root('third_party', 'lzma.js', 'lzma-native'),
path_from_root('third_party', 'lzma.js', 'lzma-decoder.js'),
'LZMA.decompress')]).communicate()
- assert ('.compress' in open('page.js').read()) == worth_compressing, 'do not compress image if not worth it'
- assert os.path.exists(basename + '.compress') == worth_compressing, 'remove .compress if not compressing'
- if worth_compressing:
- shutil.move(os.path.join(self.get_dir(), basename), basename + '.renamedsoitcannotbefound');
+ shutil.move(os.path.join(self.get_dir(), basename), basename + '.renamedsoitcannotbefound');
self.run_browser('page.html', '', '/report_result?' + str(width))
def test_sdl_canvas(self):
Please sign in to comment.
Something went wrong with that request. Please try again.