Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
branch: master
Fetching contributors…

Cannot retrieve contributors at this time

executable file 253 lines (213 sloc) 17.106 kb
#!/bin/bash
################################################################################
# Config #
################################################################################
# Whether to produce optimized, name-mangled, debugging-unfriendly JS.
OPTIMIZED=1
# Commands to use for various compilation steps.
LLVM=/home/max/emscripten-workspace/llvm-build/Release/bin/
PYTHON=python2
# Which modules to build dynamically.
# Maps each module name to the C files implementing it.
declare -A MODULES
MODULES[array]="arraymodule.c"
MODULES[cmath]="cmathmodule.c _math.c"
MODULES[math]="mathmodule.c _math.c"
MODULES[strop]="stropmodule.c"
MODULES[time]="timemodule.c"
MODULES[datetime]="datetimemodule.c timemodule.c"
MODULES[itertools]="itertoolsmodule.c"
MODULES[future_builtins]="future_builtins.c"
MODULES[_random]="_randommodule.c"
MODULES[_collections]="_collectionsmodule.c"
MODULES[_bisect]="_bisectmodule.c"
MODULES[_heapq]="_heapqmodule.c"
MODULES[operator]="operator.c"
MODULES[_functools]="_functoolsmodule.c"
MODULES[_io]="_io/bufferedio.c _io/bytesio.c _io/fileio.c _io/iobase.c _io/_iomodule.c _io/stringio.c _io/textio.c"
MODULES[_json]="_json.c"
MODULES[_testcapi]="_testcapimodule.c"
MODULES[_hotshot]="_hotshot.c"
MODULES[_lsprof]="_lsprof.c rotatingtree.c"
MODULES[unicodedata]="unicodedata.c"
MODULES[_locale]="_localemodule.c"
MODULES[fcntl]="fcntlmodule.c"
MODULES[grp]="grpmodule.c"
MODULES[spwd]="spwdmodule.c"
MODULES[cStringIO]="cStringIO.c"
MODULES[cPickle]="cPickle.c"
MODULES[mmap]="mmapmodule.c"
MODULES[syslog]="syslogmodule.c"
MODULES[audioop]="audioop.c"
MODULES[imageop]="imageop.c"
MODULES[crypt]="cryptmodule.c"
MODULES[_csv]="_csv.c"
MODULES[_socket]="socketmodule.c"
MODULES[_sha]="shamodule.c"
MODULES[_md5]="md5module.c md5.c"
MODULES[_sha256]="sha256module.c"
MODULES[_sha512]="sha512module.c"
MODULES[resource]="resource.c"
MODULES[nis]="nismodule.c"
MODULES[zlib]="zlibmodule.c"
MODULES[binascii]="binascii.c"
MODULES[bz2]="bz2module.c"
MODULES[_elementtree]="_elementtree.c"
MODULES[_multibytecodec]="cjkcodecs/multibytecodec.c"
MODULES[_codecs_kr]="cjkcodecs/_codecs_kr.c"
MODULES[_codecs_jp]="cjkcodecs/_codecs_jp.c"
MODULES[_codecs_cn]="cjkcodecs/_codecs_cn.c"
MODULES[_codecs_tw]="cjkcodecs/_codecs_tw.c"
MODULES[_codecs_hk]="cjkcodecs/_codecs_hk.c"
MODULES[_codecs_iso2022]="cjkcodecs/_codecs_iso2022.c"
MODULES[dl]="dlmodule.c"
MODULES[pyexpat]="pyexpat.c expat/xmlparse.c expat/xmlrole.c expat/xmltok.c"
MODULES[parser]="parsermodule.c"
MODULES[_struct]="_struct.c"
# Disabled because it's useless in our environment and will fail anyway.
# MODULES[termios]="termios.c"
# Requires external libraries which are too much work to compile.
# MODULES[gdbm]="gdbmmodule.c"
# This is disabled due to problems with libffi headers.
# MODULES[_ctypes]="_ctypes/_ctypes.c _ctypes/callbacks.c _ctypes/callproc.c _ctypes/stgdict.c _ctypes/cfield.c"
# These are baked into the main executable but can also be built separately.
# MODULES[_symtable]="symtablemodule.c"
# MODULES[zipimport]="zipimport.c"
# These are disabled because they contain inline assembly.
# MODULES[fpectl]="fpectlmodule.c"
# MODULES[select]="selectmodule.c"
# MODULES[linuxaudiodev]="linuxaudiodev.c"
# MODULES[ossaudiodev]="ossaudiodev.c"
################################################################################
# Patches #
################################################################################
CORRECT_OVERFLOWS_LINES='["stringobject.c:1271","intobject.c:800","longobject.c:1367","dlmalloc.c:4647","peephole.c:624","367189","tupleobject.c:352","10456","binascii.c:1159","unicodeobject.c:6552","typeobject.c:2474","longobject.c:839","intobject.c:754","407211","_randommodule.c:154","unicodedata.c:965","longobject.c:2403","peephole.c:556","longobject.c:113","typeobject.c:2507","418798","215089","intobject.c:767","10542","intobject.c:522","unicodedata.c:957","dictobject.c:443","469034","215967","longobject.c:2431","_randommodule.c:189","10618","_randommodule.c:181","object.c:1069","418680","dictobject.c:365","mystrtoul.c:220"]'
CORRECT_SIGNS_LINES='["compile.c:3542","sha256module.c:180","sha256module.c:183","sha256module.c:182","dlmalloc.c:4235","sha256module.c:184","sha256module.c:187","sha256module.c:186","sha256module.c:189","sha256module.c:188","sha256module.c:347","sha256module.c:346","sha256module.c:340","sha256module.c:185","sha256module.c:348","typeobject.c:2474","sha512module.c:76","sha512module.c:75","sha512module.c:74","sha512module.c:73","sha512module.c:79","sha512module.c:78","sha512module.c:421","sha512module.c:420","sha512module.c:423","sha512module.c:422","sha512module.c:425","sha512module.c:424","sha512module.c:228","sha512module.c:426","sha256module.c:338","sha256module.c:339","shamodule.c:327","shamodule.c:326","shamodule.c:324","shamodule.c:323","shamodule.c:322","shamodule.c:320","shamodule.c:328","_randommodule.c:140","dlmalloc.c:4626","stringobject.c:1271","sha512module.c:168","sha512module.c:169","sha512module.c:164","sha512module.c:165","sha512module.c:166","sha512module.c:167","sha512module.c:160","sha512module.c:161","sha512module.c:162","sha512module.c:163","md5.c:248","md5.c:249","md5.c:240","md5.c:241","md5.c:242","md5.c:243","md5.c:244","md5.c:245","md5.c:246","md5.c:247","stringobject.c:72","sha512module.c:77","sha512module.c:195","md5.c:292","md5.c:291","peephole.c:615","sha512module.c:196","sha512module.c:191","sha512module.c:190","sha512module.c:193","md5.c:294","compile.c:3461","longobject.c:2283","binascii.c:316","binascii.c:1365","tupleobject.c:352","binascii.c:1044","compile.c:3934","sha256module.c:156","sha256module.c:157","sha256module.c:154","sha256module.c:155","sha256module.c:152","sha256module.c:153","sha256module.c:150","sha256module.c:151","sha256module.c:158","sha256module.c:159","sha256module.c:72","sha256module.c:70","symtable.c:702","fastsearch.h:65","intobject.c:522","_randommodule.c:189","peephole.c:460","_randommodule.c:181","setobject.c:190","sha256module.c:202","sha256module.c:203","sha256module.c:200","sha256module.c:201","sha256module.c:206","sha256module.c:207","sha256module.c:204","sha256module.c:205","sha256module.c:208","sha256module.c:209","dlmalloc.c:4199","_randommodule.c:112","_randommodule.c:115","md5.c:208","md5.c:209","sha512module.c:227","sha512module.c:226","sha512module.c:225","sha512module.c:224","sha512module.c:388","sha512module.c:389","sha512module.c:221","sha512module.c:220","sha512module.c:384","sha512module.c:385","sha512module.c:386","sha512module.c:380","sha512module.c:381","sha512module.c:382","sha512module.c:383","longobject.c:163","longobject.c:2431","md5.c:271","md5.c:270","md5.c:273","md5.c:272","md5.c:275","md5.c:274","listobject.c:51","shamodule.c:70","longobject.c:2159","longobject.c:2150","intobject.c:754","setobject.c:259","peephole.c:581","peephole.c:429","sha512module.c:414","sha512module.c:415","sha512module.c:416","sha512module.c:417","sha512module.c:412","sha512module.c:413","sha512module.c:418","dlmalloc.c:4205","dlmalloc.c:4647","marshal.c:114","binascii.c:424","_randommodule.c:154","dlmalloc.c:4227","symtable.c:488","symtable.c:484","sha512module.c:173","sha512module.c:172","sha512module.c:171","sha512module.c:170","sha512module.c:177","sha512module.c:176","sha512module.c:175","sha512module.c:174","sha512module.c:179","sha512module.c:178","md5.c:239","md5.c:238","md5.c:235","md5.c:234","md5.c:237","md5.c:236","longobject.c:2484","sha512module.c:223","sha512module.c:222","sha512module.c:218","sha512module.c:219","sha512module.c:216","sha512module.c:217","sha512module.c:214","sha512module.c:215","sha512module.c:212","sha512module.c:213","sha512module.c:210","sha512module.c:211","peephole.c:602","compile.c:3670","sha512module.c:229","sha512module.c:188","sha512module.c:189","sha512module.c:186","sha512module.c:187","sha512module.c:184","sha512module.c:185","sha512module.c:182","sha512module.c:183","sha512module.c:180","sha512module.c:181","stringobject.c:1174","_sre.c:1541","compile.c:3692","binascii.c:398","dictobject.c:442","dictobject.c:443","sha256module.c:149","binascii.c:660","marshal.c:107","sha256module.c:364","sha256module.c:360","sha256module.c:363","sha256module.c:362","peephole.c:556","setobject.c:189","compile.c:3451","sha256module.c:211","sha256module.c:210","shamodule.c:189","sha256module.c:212","shamodule.c:183","shamodule.c:182","shamodule.c:187","shamodule.c:186","shamodule.c:185","shamodule.c:184","_randommodule.c:124","_randommodule.c:121","ceval.c:2487","md5.c:393","longobject.c:881","longobject.c:2490","longobject.c:155","md5.c:309","md5.c:308","dlmalloc.c:4191","unicodedata.c:965","md5.c:307","md5.c:301","md5.c:300","longobject.c:2400","md5.c:262","md5.c:263","md5.c:260","md5.c:261","md5.c:266","md5.c:267","md5.c:264","md5.c:265","md5.c:268","md5.c:269","longobject.c:2395","sha512module.c:244","dlmalloc.c:4816","intobject.c:767","ceval.c:988","sha256module.c:136","longobject.c:2140","sha512module.c:393","sha512module.c:392","sha512module.c:391","sha512module.c:390","sha512module.c:394","compile.c:1531","_randommodule.c:108","cPickle.c:4686","sha256module.c:181","shamodule.c:68","binascii.c:876","mystrtoul.c:224","mystrtoul.c:220","longobject.c:299","longobject.c:836","md5.c:222","md5.c:223","md5.c:220","md5.c:221","sha512module.c:209","sha512module.c:208","sha512module.c:205","sha512module.c:204","sha512module.c:207","sha512module.c:206","sha512module.c:201","sha512module.c:200","sha512module.c:203","sha512module.c:202","unicodedata.c:957","md5.c:299","md5.c:298","sha512module.c:199","sha512module.c:198","md5.c:293","sha512module.c:194","sha512module.c:197","md5.c:290","md5.c:297","md5.c:296","md5.c:295","object.c:1069","binascii.c:722","compile.c:3728","longobject.c:2135","longobject.c:2139","typeobject.c:2507","sha256module.c:178","sha256module.c:179","sha256module.c:170","sha256module.c:171","sha256module.c:172","sha256module.c:173","sha256module.c:174","sha256module.c:175","sha256module.c:176","sha256module.c:177","binascii.c:653","stringobject.c:108","sha256module.c:192","sha256module.c:193","sha256module.c:190","sha256module.c:191","sha256module.c:196","sha256module.c:197","sha256module.c:194","sha256module.c:195","sha256module.c:198","sha256module.c:199","compile.c:3433","binascii.c:833","sha256module.c:350","sha256module.c:351","sha256module.c:352","sha512module.c:192","sha256module.c:354","sha256module.c:355","sha256module.c:356","sha256module.c:358","sha256module.c:359","peephole.c:446","compile.c:3440","shamodule.c:190","shamodule.c:191","shamodule.c:192","shamodule.c:193","shamodule.c:194","binascii.c:1159","unicodeobject.c:6552","sha512module.c:159","stringobject.c:1106","md5.c:310","shamodule.c:188","complexobject.c:474","compile.c:3792","compile.c:3796","compile.c:3794","peephole.c:624","binascii.c:510","sha256module.c:218","longobject.c:2162","shamodule.c:318","shamodule.c:319","shamodule.c:316","shamodule.c:314","shamodule.c:315","shamodule.c:312","shamodule.c:310","shamodule.c:311","setobject.c:118","dlmalloc.c:4246","dictobject.c:569","md5.c:213","md5.c:212","md5.c:211","md5.c:210","md5.c:217","md5.c:216","md5.c:215","md5.c:214","md5.c:219","md5.c:218","peephole.c:377","dictobject.c:365","dictobject.c:364","sha512module.c:230","sha512module.c:231","sha512module.c:232","sha512module.c:233","sha512module.c:234","sha512module.c:235","sha512module.c:236","sha512module.c:237","sha512module.c:238","shamodule.c:173","_sre.c:1142","md5.c:288","md5.c:289","md5.c:286","md5.c:287","longobject.c:889","compile.c:3730","sha256module.c:169","sha256module.c:168","sha256module.c:167","sha256module.c:166","sha256module.c:165","sha256module.c:164","sha256module.c:163","sha256module.c:162","sha256module.c:161","sha256module.c:160"]'
################################################################################
# Functions #
################################################################################
function build_module {
echo " Building $1..."
# Compile.
FLAGS="-c -emit-llvm -fPIC -fno-strict-aliasing -I. -IInclude -I../cpython/Include -I../cpython/Modules/expat -Wstrict-prototypes"
FILES=""
for ((i=2; i<=$#; i++)); do
../ccproxy.py $FLAGS ../cpython/Modules/${!i} -o `basename ${!i}`.o
FILES="$FILES `basename ${!i}`.o"
done
# Link.
../ccproxy.py $FILES -o $1.so.bc
# Emscript.
$PYTHON ../emscripten/emscripten.py $1.so.bc \
-o $1.so.js \
-s BUILD_AS_SHARED_LIB=1 \
-s EXPORTED_FUNCTIONS="[\"_init$1\"]" \
-s CORRECT_SIGNS=2 -s CORRECT_SIGNS_LINES=$CORRECT_SIGNS_LINES \
-s CORRECT_OVERFLOWS=2 -s CORRECT_OVERFLOWS_LINES=$CORRECT_OVERFLOWS_LINES \
$OPTIMIZATION_ARGS
# Run eliminator.
../emscripten/tools/eliminator/node_modules/coffee-script/bin/coffee ../emscripten/tools/eliminator/eliminator.coffee < $1.so.js > $1.so.elim.js 2>/dev/null
# Copy the new module to dist.
cp $1.so.elim.js ../dist/lib/python2.7/$1.so.js
}
################################################################################
# Script #
################################################################################
# TODO: Make it possible to build only the main python.js, only modules or only
# a specific module.
# Exit on first error.
set -e
# Setup optimization flags for all future emscripting.
if [ $OPTIMIZED -eq 1 ]; then
# TODO: Figure out if reloop is possible and useful.
OPTIMIZATION_ARGS="-O -s OPTIMIZE=1 -s RELOOP=1 -s ASSERTIONS=0"
else
OPTIMIZATION_ARGS=" -s OPTIMIZE=0 -s RELOOP=0 -s ASSERTIONS=1"
fi
# Create two folders for intermediate and final files, respectively.
mkdir -p obj
mkdir -p dist
# Remove old build.
rm -rf obj/*
rm -rf dist/{*.js,lib,include}
# Start building in the obj folder.
cd obj
# Create the Makefile and configurations using a filtering proxy for llvm-gcc.
CC=../ccproxy.py ../cpython/configure --without-threads --without-pymalloc
# Adjust configuration.
# Remove the closing endif so we can insert new options.
sed -i 's~#endif /\*Py_PYCONFIG_H\*/~~' pyconfig.h
# Emscripten doesn't support CPU-specific assembly code.
echo '#undef HAVE_GCC_ASM_FOR_X87' >> pyconfig.h
# Emscripten does not support interrupt signals.
echo '#undef HAVE_SIGACTION' >> pyconfig.h
echo '#undef HAVE_SIGINTERRUPT' >> pyconfig.h
echo '#undef HAVE_SIGRELSE' >> pyconfig.h
# Put the closing endif back.
echo '#endif /*Py_PYCONFIG_H*/' >> pyconfig.h
# Compile CPython.
echo 'Compiling...'
make
# Link the resulting libraries. Discard the main function.
echo 'Relinking...'
mkdir -p relinked
cd relinked
ar x ../libpython2.7.a
rm main.o
${LLVM}/llvm-link -o=python.bc *.o
cp python.bc ..
cd ..
# Compile the bytecode into JS using Emscripten.
echo 'Emscripting...'
$PYTHON ../emscripten/emscripten.py python.bc \
-m \
-o python.js \
-s INVOKE_RUN=0 \
-s INCLUDE_FULL_LIBRARY=1 \
-s CORRECT_SIGNS=2 -s CORRECT_SIGNS_LINES=$CORRECT_SIGNS_LINES \
-s CORRECT_OVERFLOWS=2 -s CORRECT_OVERFLOWS_LINES=$CORRECT_OVERFLOWS_LINES \
$OPTIMIZATION_ARGS
# Copy the main Python JS executable to dist.
cp python.js ../dist/python.js
# Copy the pure Python modules to dist.
mkdir -p ../dist/lib
mkdir -p ../dist/lib/python2.7
cp -r ../cpython/Lib/* ../dist/lib/python2.7
rm -rf ../dist/lib/python2.7/{idlelib,lib-tk,multiprocessing,curses,bsddb}
rm -rf ../dist/lib/python2.7/plat-{aix3,aix4,atheos,beos5,darwin,freebsd4,freebsd5,freebsd6,freebsd7,freebsd8,generic,irix5,irix6,mac,netbsd1,next3,os2emx,riscos,sunos5,unixware7}
if [ $OPTIMIZED -eq 1 ]; then
rm -rf ../dist/lib/python2.7/test
rm -rf ../dist/lib/python2.7/*/test{,s}
fi
find ../dist -name '*.pyc' | xargs rm
# Make sure everything is readable and folders are executable (browseable).
chmod -R ugo+rX ../dist
# Copy the configs.
mkdir -p ../dist/lib/python2.7/config
cp Makefile Modules/{Setup*,config.c} ../dist/lib/python2.7/config
cp ../cpython/{Modules/{config.c.in,makesetup},install-sh} ../dist/lib/python2.7/config
mkdir -p ../dist/include
mkdir -p ../dist/include/python2.7
cp pyconfig.h ../dist/include/python2.7/
# Build dynamically loaded modules.
echo 'Building dynamic modules...'
for i in "${!MODULES[@]}"; do
build_module $i ${MODULES[$i]}
done
# Create virtual file system entries.
echo 'Mapping virtual filesystem.'
python2 ../map_filesystem.py ../dist >> ../dist/python.js
# Add a tiny handcoded JavaScript API.
echo 'Adding entry point.'
cat ../entry_point.js >> ../dist/python.js
# Run optimizations.
if [ $OPTIMIZED -eq 1 ]; then
echo 'Running eliminator...'
../emscripten/tools/eliminator/node_modules/coffee-script/bin/coffee ../emscripten/tools/eliminator/eliminator.coffee < ../dist/python.js > ../dist/python.elim.js 2>/dev/null
echo 'Minifying...'
cd ..
./minify.py
else
cp ../dist/python.js ../dist/python.elim.js
cp ../dist/python.js ../dist/python.opt.js
fi
Jump to Line
Something went wrong with that request. Please try again.