Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

ENH: add Dag Sverre's cythonize command

./tools/cythonize checks for modifications in pyx files relative to the
corresponding c file, and updates the c file if it detects a mismatch.
  • Loading branch information...
commit f801c62520fa30aa76a90edac8d3a2924056046b 1 parent 1eac7bf
Matthew Brett matthew-brett authored rgommers committed
Showing with 202 additions and 0 deletions.
  1. +1 −0  .gitignore
  2. +201 −0 tools/cythonize
1  .gitignore
View
@@ -11,6 +11,7 @@
.\#*
.deps
.libs
+cythonize.dat
/build
/dist
/scipy.egg-info
201 tools/cythonize
View
@@ -0,0 +1,201 @@
+#!/usr/bin/env python
+""" cythonize
+
+Cythonize pyx files into C files as needed.
+
+Usage: cythonize [root_dir]
+
+Default [root_dir] is 'scipy'.
+
+Checks pyx files to see if they have been changed relative to their
+corresponding C files. If they have, then runs cython on these files to
+recreate the C files.
+
+The script thinks that the pyx files have changed relative to the C files if:
+
+* The pyx file is modified compared to the version checked into git, and the C
+ file is not
+* The pyx file was committed to the repository more recently than the C file.
+
+Simple script to invoke Cython (and Tempita) on all .pyx (.pyx.in)
+files; while waiting for a proper build system. Uses file hashes to
+figure out if rebuild is needed (using dates seem to fail with
+frequent change of git branch).
+
+For now, this script should be run by developers when changing Cython files
+only, and the resulting C files checked in, so that end-users (and Python-only
+developers) do not get the Cython/Tempita dependencies.
+
+Originally written by Dag Sverre Seljebotn, and copied here from:
+
+https://raw.github.com/dagss/private-scipy-refactor/cythonize/cythonize.py
+
+Note: this script does not check any of the dependent C libraries; it only
+operates on the Cython .pyx files.
+"""
+
+import os
+import sys
+import hashlib
+import cPickle
+from subprocess import Popen, PIPE
+from distutils.version import LooseVersion
+
+try:
+ import Cython
+except ImportError:
+ raise OSError('We need cython for this script')
+from Cython.Compiler.Version import version as cython_version
+
+HAVE_CYTHON_0p14 = LooseVersion(cython_version) >= LooseVersion('0.14')
+
+HASH_FILE = 'cythonize.dat'
+DEFAULT_ROOT = 'scipy'
+
+#
+# Rules
+#
+def process_pyx(fromfile, tofile):
+ if HAVE_CYTHON_0p14:
+ opt_str = '--fast-fail'
+ else:
+ opt_str = ''
+ if os.system('cython %s -o "%s" "%s"' % (opt_str, tofile, fromfile)) != 0:
+ raise Exception('Cython failed')
+
+def process_tempita_pyx(fromfile, tofile):
+ import tempita
+ with file(fromfile) as f:
+ tmpl = f.read()
+ pyxcontent = tempita.sub(tmpl)
+ assert fromfile.endswith('.pyx.in')
+ pyxfile = fromfile[:-len('.pyx.in')] + '.pyx'
+ with file(pyxfile, 'w') as f:
+ f.write(pyxcontent)
+ process_pyx(pyxfile, tofile)
+
+rules = {
+ # fromext : (toext, function)
+ '.pyx' : ('.c', process_pyx),
+ '.pyx.in' : ('.c', process_tempita_pyx)
+ }
+#
+# Hash db
+#
+def load_hashes(filename):
+ # Return { filename : (sha1 of input, sha1 of output) }
+ if os.path.isfile(filename):
+ with file(filename) as f:
+ hashes = cPickle.load(f)
+ else:
+ hashes = {}
+ return hashes
+
+def save_hashes(hash_db, filename):
+ with file(filename, 'w') as f:
+ cPickle.dump(hash_db, f)
+
+def sha1_of_file(filename):
+ h = hashlib.sha1()
+ with file(filename) as f:
+ h.update(f.read())
+ return h.hexdigest()
+
+#
+# interface with git
+#
+
+def execproc(cmd):
+ assert isinstance(cmd, (list, tuple))
+ pp = Popen(cmd, stdout=PIPE, stderr=PIPE)
+ result = pp.stdout.read().strip()
+ err = pp.stderr.read()
+ retcode = pp.wait()
+ if retcode != 0:
+ return None
+ else:
+ return result
+
+def git_last_commit_to(filename):
+ out = execproc(['git', 'log', '-1', '--format=format:%H', filename])
+ if out == '':
+ out = None
+ return out
+
+def git_is_dirty(filename):
+ out = execproc(['git', 'status', '--porcelain', filename])
+ assert out is not None
+ return (out != '')
+
+def git_is_child(parent_sha, child_sha):
+ out = execproc(['git', 'rev-list', child_sha, '^%s^' % parent_sha])
+ assert out is not None
+ for line in out.split('\n'):
+ if line == parent_sha:
+ return True
+ return False
+
+#
+# Main program
+#
+
+def get_hash(frompath, topath):
+ from_hash = sha1_of_file(frompath)
+ to_hash = sha1_of_file(topath) if os.path.exists(topath) else None
+ return (from_hash, to_hash)
+
+def process(path, fromfile, tofile, processor_function, hash_db):
+ fullfrompath = os.path.join(path, fromfile)
+ fulltopath = os.path.join(path, tofile)
+ current_hash = get_hash(fullfrompath, fulltopath)
+ if current_hash == hash_db.get(fullfrompath, None):
+ print '%s has not changed' % fullfrompath
+ return
+
+ from_sha = git_last_commit_to(fullfrompath)
+ to_sha = git_last_commit_to(fulltopath)
+ if (from_sha is not None and to_sha is not None and
+ not git_is_dirty(fullfrompath)):
+ # Both source and target is under revision control;
+ # check with revision control system whether we need to
+ # update
+ if git_is_child(from_sha, to_sha):
+ hash_db[fullfrompath] = current_hash
+ print '%s is up to date (according to git)' % fullfrompath
+ return
+
+ orig_cwd = os.getcwd()
+ try:
+ os.chdir(path)
+ print 'Processing %s' % fullfrompath
+ processor_function(fromfile, tofile)
+ finally:
+ os.chdir(orig_cwd)
+ # changed target file, recompute hash
+ current_hash = get_hash(fullfrompath, fulltopath)
+ # store hash in db
+ hash_db[fullfrompath] = current_hash
+
+
+def find_process_files(root_dir):
+ hash_db = load_hashes(HASH_FILE)
+ for cur_dir, dirs, files in os.walk(root_dir):
+ for filename in files:
+ for fromext, rule in rules.iteritems():
+ if filename.endswith(fromext):
+ toext, function = rule
+ fromfile = filename
+ tofile = filename[:-len(fromext)] + toext
+ process(cur_dir, fromfile, tofile, function, hash_db)
+ save_hashes(hash_db, HASH_FILE)
+
+def main():
+ try:
+ root_dir = sys.argv[1]
+ except IndexError:
+ root_dir = DEFAULT_ROOT
+ find_process_files(root_dir)
+
+
+if __name__ == '__main__':
+ main()
Please sign in to comment.
Something went wrong with that request. Please try again.