Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

full nodejs support in file_packager.py v2 #183

Closed
wants to merge 4 commits into from
Closed

full nodejs support in file_packager.py v2 #183

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a4f6f2b
full nodejs support in file_packager.py
gabrielfreire Sep 13, 2018
e06aa82
apply PR changes
gabrielfreire Sep 14, 2018
7ddb26d
Fix es6 string formatting
gabrielfreire Sep 20, 2018
0ec3030
Add global variable to Module for better environment control
gabrielfreire Sep 27, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/hiwire.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ EM_JS(void, hiwire_push_object_pair, (int idobj, int idkey, int idval), {

EM_JS(int, hiwire_get_global, (int idname), {
var jsname = UTF8ToString(idname);
return Module.hiwire_new_value(window[jsname]);
return Module.hiwire_new_value(Module.global[jsname]);
});

EM_JS(int, hiwire_get_member_string, (int idobj, int idkey), {
Expand Down
2 changes: 1 addition & 1 deletion src/pyodide.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,7 +253,7 @@ var languagePluginLoader = new Promise((resolve, reject) => {
.then(instance => receiveInstance(instance));
return {};
};

Module.global = window;
Module.locateFile = (path) => baseURL + path;
var postRunPromise = new Promise((resolve, reject) => {
Module.postRun = () => {
Expand Down
179 changes: 128 additions & 51 deletions tools/file_packager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,14 @@
# emcc.py will add this to the output itself, so it is only needed for standalone calls
if not from_emcc:
ret = '''
var Module = typeof %(EXPORT_NAME)s !== 'undefined' ? %(EXPORT_NAME)s : {};
var Module;
if (typeof %(EXPORT_NAME)s !== 'undefined') {
Module = %(EXPORT_NAME)s; /* browser */
} else if (typeof process['%(EXPORT_NAME)s'] !== 'undefined') {
Module = process['%(EXPORT_NAME)s']; /* node */
} else {
Module = {};
}
''' % {"EXPORT_NAME": export_name}

ret += '''
Expand Down Expand Up @@ -479,12 +486,16 @@ def was_seen(name):
ret += r'''
var PACKAGE_PATH;
if (typeof window === 'object') {
// browser
PACKAGE_PATH = window['encodeURIComponent'](window.location.pathname.toString().substring(0, window.location.pathname.toString().lastIndexOf('/')) + '/');
} else if (typeof location !== 'undefined') {
// worker
PACKAGE_PATH = encodeURIComponent(location.pathname.toString().substring(0, location.pathname.toString().lastIndexOf('/')) + '/');
} else if (typeof process === "object" && typeof require === "function") {
// node
PACKAGE_PATH = encodeURIComponent(require('path').join(__dirname, '/'));
} else {
throw 'using preloaded data can only be done on a web page or in a web worker';
throw 'using preloaded data can only be done on a web page, web worker or in nodejs';
}
var PACKAGE_NAME = '%s';
var REMOTE_PACKAGE_BASE = '%s';
Expand Down Expand Up @@ -595,51 +606,109 @@ def was_seen(name):

ret += r'''
function fetchRemotePackage(packageName, packageSize, callback, errback) {
var xhr = new XMLHttpRequest();
xhr.open('GET', packageName, true);
xhr.responseType = 'arraybuffer';
xhr.onprogress = function(event) {
var url = packageName;
var size = packageSize;
if (event.total) size = event.total;
if (event.loaded) {
if (!xhr.addedTotal) {
xhr.addedTotal = true;
if (!Module.dataFileDownloads) Module.dataFileDownloads = {};
Module.dataFileDownloads[url] = {
loaded: event.loaded,
total: size
};
} else {
Module.dataFileDownloads[url].loaded = event.loaded;
if (typeof XMLHttpRequest !== 'undefined') { /* browser */
var xhr = new XMLHttpRequest();
xhr.open('GET', packageName, true);
xhr.responseType = 'arraybuffer';
xhr.onprogress = function(event) {
var url = packageName;
var size = packageSize;
if (event.total) size = event.total;
if (event.loaded) {
if (!xhr.addedTotal) {
xhr.addedTotal = true;
if (!Module.dataFileDownloads) Module.dataFileDownloads = {};
Module.dataFileDownloads[url] = {
loaded: event.loaded,
total: size
};
} else {
Module.dataFileDownloads[url].loaded = event.loaded;
}
var total = 0;
var loaded = 0;
var num = 0;
for (var download in Module.dataFileDownloads) {
var data = Module.dataFileDownloads[download];
total += data.total;
loaded += data.loaded;
num++;
}
total = Math.ceil(total * Module.expectedDataFileDownloads/num);
if (Module['setStatus']) Module['setStatus']('Downloading data... (' + loaded + '/' + total + ')');
} else if (!Module.dataFileDownloads) {
if (Module['setStatus']) Module['setStatus']('Downloading data...');
}
var total = 0;
var loaded = 0;
var num = 0;
for (var download in Module.dataFileDownloads) {
var data = Module.dataFileDownloads[download];
total += data.total;
loaded += data.loaded;
num++;
};
xhr.onerror = function(event) {
throw new Error("NetworkError for: " + packageName);
}
xhr.onload = function(event) {
if (xhr.status == 200 || xhr.status == 304 || xhr.status == 206 || (xhr.status == 0 && xhr.response)) { // file URLs can return 0
var packageData = xhr.response;
callback(packageData);
} else {
throw new Error(xhr.statusText + " : " + xhr.responseURL);
}
total = Math.ceil(total * Module.expectedDataFileDownloads/num);
if (Module['setStatus']) Module['setStatus']('Downloading data... (' + loaded + '/' + total + ')');
} else if (!Module.dataFileDownloads) {
if (Module['setStatus']) Module['setStatus']('Downloading data...');
};
xhr.send(null);
} else { /* node */
function fetch_node(file) {
var fs = require('fs');
var fetch = require('isomorphic-fetch');
return new Promise(function(resolve, reject) {
if(file.indexOf('http') == -1) { // local
fs.readFile(file, function (err, data) {
if (err) {
reject(err);
} else {
resolve({
buffer: function() {
return data;
}
});
}
});
} else { // remote
fetch(file).then(function(buff) {
resolve({
buffer: function() {
return buff.buffer()
}
})
});
}
});
}
};
xhr.onerror = function(event) {
throw new Error("NetworkError for: " + packageName);
}
xhr.onload = function(event) {
if (xhr.status == 200 || xhr.status == 304 || xhr.status == 206 || (xhr.status == 0 && xhr.response)) { // file URLs can return 0
var packageData = xhr.response;
fetch_node(packageName).then(function(buffer) { return buffer.buffer() }).then(function (packageData) {
if (!Module.dataFileDownloads) {
if (Module['setStatus']) Module['setStatus']('Downloading data...');
} else {
Module.dataFileDownloads[packageName] = {
loaded: packageSize,
total: packageSize
}
var total = 0;
var loaded = 0;
var num = 0;
for (var download in Module.dataFileDownloads) {
var data = Module.dataFileDownloads[download];
total += data.total;
loaded += data.loaded;
num++
}
total = Math.ceil(total * Module.expectedDataFileDownloads / num);
if (Module['setStatus']) {
Module['setStatus']('Downloading data... (' + loaded + '/' + total + ')');
console.log('Downloaded ' + packageName + ' data... (' + loaded + '/' + total + ')');
}
}
callback(packageData);
} else {
throw new Error(xhr.statusText + " : " + xhr.responseURL);
}
};
xhr.send(null);
}).catch(function(err) {
console.error('Something wrong happened ' + err);
throw new Error('Something wrong happened ' + err);
});
}
};

function handleError(error) {
Expand All @@ -650,6 +719,7 @@ def was_seen(name):
code += r'''
function processPackageData(arrayBuffer) {
Module.finishedDataFileDownloads++;
arrayBuffer = arrayBuffer instanceof ArrayBuffer ? arrayBuffer : arrayBuffer.buffer;
assert(arrayBuffer, 'Loading data file failed.');
assert(arrayBuffer instanceof ArrayBuffer, 'bad input to processPackageData');
var byteArray = new Uint8Array(arrayBuffer);
Expand Down Expand Up @@ -748,15 +818,22 @@ def was_seen(name):
function runMetaWithFS() {
Module['addRunDependency']('%(metadata_file)s');
var REMOTE_METADATA_NAME = Module['locateFile'] ? Module['locateFile']('%(metadata_file)s', '') : '%(metadata_file)s';
var xhr = new XMLHttpRequest();
xhr.onreadystatechange = function() {
if (xhr.readyState === 4 && xhr.status === 200) {
loadPackage(JSON.parse(xhr.responseText));
}
if (typeof XMLHttpRequest !== 'undefined') { /* browser */
var xhr = new XMLHttpRequest();
xhr.onreadystatechange = function() {
if (xhr.readyState === 4 && xhr.status === 200) {
loadPackage(JSON.parse(xhr.responseText));
}
}
xhr.open('GET', REMOTE_METADATA_NAME, true);
xhr.overrideMimeType('application/json');
xhr.send(null);
} else { /* node */
var fetch = require('isomorphic-fetch');
fetch(REMOTE_METADATA_NAME).then(function(buffer) { return buffer.json(); }).then(function(packageData) {
loadPackage(packageData);
});
}
xhr.open('GET', REMOTE_METADATA_NAME, true);
xhr.overrideMimeType('application/json');
xhr.send(null);
}

if (Module['calledRun']) {
Expand Down