Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Import speedup - Ticket #2181 #334

Closed
wants to merge 9 commits into from
Closed

Import speedup - Ticket #2181 #334

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
2e52dcc
Remove the 5 'exec' calls definining polynomial classes.
adalke Jul 6, 2012
d59b142
STY: Ran reindent tool
scottza Jul 10, 2012
08dadee
Revert "STY: Ran reindent tool"
scottza Jul 11, 2012
1739d60
REF: Rename files in preparation for build time module generation
scottza Jul 11, 2012
b53c462
ENH: Generate polynomial class modules at build time
scottza Jul 11, 2012
ea6b365
REF: Simplify class generation from template
scottza Jul 12, 2012
461d5be
DOC: Add notes explaining that polynomial classes are auto-generated
scottza Jul 12, 2012
76b700c
BUG: Explicit imports to avoid confusing 2to3 tool for Py3 builds
scottza Jul 16, 2012
ddaadac
BLD: Put generated files in build directory for distutils build
scottza Jul 17, 2012
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
12 changes: 6 additions & 6 deletions numpy/polynomial/__init__.py
View file
Open in desktop
Expand Up @@ -15,12 +15,12 @@
"""
import warnings

from polynomial import Polynomial
from chebyshev import Chebyshev
from legendre import Legendre
from hermite import Hermite
from hermite_e import HermiteE
from laguerre import Laguerre
from numpy.polynomial.polynomial import Polynomial
from numpy.polynomial.chebyshev import Chebyshev
from numpy.polynomial.legendre import Legendre
from numpy.polynomial.hermite import Hermite
from numpy.polynomial.hermite_e import HermiteE
from numpy.polynomial.laguerre import Laguerre

from numpy.testing import Tester
test = Tester().test
Expand Down
8 changes: 5 additions & 3 deletions numpy/polynomial/chebyshev.py → numpy/polynomial/_chebyshev.py
View file
Open in desktop
Expand Up @@ -91,7 +91,6 @@
import numpy.linalg as la
import polyutils as pu
import warnings
from polytemplate import polytemplate

__all__ = ['chebzero', 'chebone', 'chebx', 'chebdomain', 'chebline',
'chebadd', 'chebsub', 'chebmulx', 'chebmul', 'chebdiv', 'chebpow',
Expand Down Expand Up @@ -416,7 +415,7 @@ def cheb2poly(c) :
array([ -2., -8., 4., 12.])

"""
from polynomial import polyadd, polysub, polymulx
from numpy.polynomial.polynomial import polyadd, polysub, polymulx

[c] = pu.as_series([c])
n = len(c)
Expand Down Expand Up @@ -2005,5 +2004,8 @@ def chebpts2(npts):
#
# Chebyshev series class
#
# Code below this point is generated at build time from the template
# in polytemplate.py and combined with _chebyshev.py (see setup.py in
# this directory).
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A note explaining that code for the Chebyshev class will be adding during the install process would be helpful. Likewise for the other classes.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good idea.

#

exec polytemplate.substitute(name='Chebyshev', nick='cheb', domain='[-1,1]')
8 changes: 5 additions & 3 deletions numpy/polynomial/hermite.py → numpy/polynomial/_hermite.py
View file
Open in desktop
Expand Up @@ -63,7 +63,6 @@
import numpy.linalg as la
import polyutils as pu
import warnings
from polytemplate import polytemplate

__all__ = ['hermzero', 'hermone', 'hermx', 'hermdomain', 'hermline',
'hermadd', 'hermsub', 'hermmulx', 'hermmul', 'hermdiv', 'hermpow',
Expand Down Expand Up @@ -160,7 +159,7 @@ def herm2poly(c) :
array([ 0., 1., 2., 3.])

"""
from polynomial import polyadd, polysub, polymulx
from numpy.polynomial.polynomial import polyadd, polysub, polymulx

[c] = pu.as_series([c])
n = len(c)
Expand Down Expand Up @@ -1740,5 +1739,8 @@ def hermweight(x):
#
# Hermite series class
#
# Code below this point is generated at build time from the template
# in polytemplate.py and combined with _hermite.py (see setup.py in
# this directory).
#

exec polytemplate.substitute(name='Hermite', nick='herm', domain='[-1,1]')
8 changes: 5 additions & 3 deletions numpy/polynomial/hermite_e.py → numpy/polynomial/_hermite_e.py
View file
Open in desktop
Expand Up @@ -63,7 +63,6 @@
import numpy.linalg as la
import polyutils as pu
import warnings
from polytemplate import polytemplate

__all__ = ['hermezero', 'hermeone', 'hermex', 'hermedomain', 'hermeline',
'hermeadd', 'hermesub', 'hermemulx', 'hermemul', 'hermediv', 'hermpow',
Expand Down Expand Up @@ -160,7 +159,7 @@ def herme2poly(c) :
array([ 0., 1., 2., 3.])

"""
from polynomial import polyadd, polysub, polymulx
from numpy.polynomial.polynomial import polyadd, polysub, polymulx

[c] = pu.as_series([c])
n = len(c)
Expand Down Expand Up @@ -1736,5 +1735,8 @@ def hermeweight(x):
#
# HermiteE series class
#
# Code below this point is generated at build time from the template
# in polytemplate.py and combined with _hermite_e.py (see setup.py in
# this directory).
#

exec polytemplate.substitute(name='HermiteE', nick='herme', domain='[-1,1]')
8 changes: 5 additions & 3 deletions numpy/polynomial/laguerre.py → numpy/polynomial/_laguerre.py
View file
Open in desktop
Expand Up @@ -63,7 +63,6 @@
import numpy.linalg as la
import polyutils as pu
import warnings
from polytemplate import polytemplate

__all__ = ['lagzero', 'lagone', 'lagx', 'lagdomain', 'lagline',
'lagadd', 'lagsub', 'lagmulx', 'lagmul', 'lagdiv', 'lagpow',
Expand Down Expand Up @@ -159,7 +158,7 @@ def lag2poly(c) :
array([ 0., 1., 2., 3.])

"""
from polynomial import polyadd, polysub, polymulx
from numpy.polynomial.polynomial import polyadd, polysub, polymulx

[c] = pu.as_series([c])
n = len(c)
Expand Down Expand Up @@ -1732,5 +1731,8 @@ def lagweight(x):
#
# Laguerre series class
#
# Code below this point is generated at build time from the template
# in polytemplate.py and combined with _laguerre.py (see setup.py in
# this directory).
#

exec polytemplate.substitute(name='Laguerre', nick='lag', domain='[-1,1]')
8 changes: 5 additions & 3 deletions numpy/polynomial/legendre.py → numpy/polynomial/_legendre.py
View file
Open in desktop
Expand Up @@ -87,7 +87,6 @@
import numpy.linalg as la
import polyutils as pu
import warnings
from polytemplate import polytemplate

__all__ = ['legzero', 'legone', 'legx', 'legdomain', 'legline',
'legadd', 'legsub', 'legmulx', 'legmul', 'legdiv', 'legpow', 'legval',
Expand Down Expand Up @@ -191,7 +190,7 @@ def leg2poly(c) :


"""
from polynomial import polyadd, polysub, polymulx
from numpy.polynomial.polynomial import polyadd, polysub, polymulx

[c] = pu.as_series([c])
n = len(c)
Expand Down Expand Up @@ -1758,5 +1757,8 @@ def legweight(x):
#
# Legendre series class
#
# Code below this point is generated at build time from the template
# in polytemplate.py and combined with _legendre.py (see setup.py in
# this directory).
#

exec polytemplate.substitute(name='Legendre', nick='leg', domain='[-1,1]')
6 changes: 4 additions & 2 deletions numpy/polynomial/polynomial.py → numpy/polynomial/_polynomial.py
View file
Open in desktop
Expand Up @@ -64,7 +64,6 @@
import numpy.linalg as la
import polyutils as pu
import warnings
from polytemplate import polytemplate

polytrim = pu.trimcoef

Expand Down Expand Up @@ -1482,5 +1481,8 @@ def polyroots(c):
#
# polynomial class
#
# Code below this point is generated at build time from the template
# in polytemplate.py and combined with _polynomial.py (see setup.py in
# this directory).
#

exec polytemplate.substitute(name='Polynomial', nick='poly', domain='[-1,1]')
25 changes: 8 additions & 17 deletions numpy/polynomial/polytemplate.py
View file
Open in desktop
@@ -1,28 +1,19 @@
"""
Template for the Chebyshev and Polynomial classes.

This is an internal module and should not be imported.

This module houses a Python string module Template object (see, e.g.,
http://docs.python.org/library/string.html#template-strings) used by
the `polynomial` and `chebyshev` modules to implement their respective
`Polynomial` and `Chebyshev` classes. It provides a mechanism for easily
creating additional specific polynomial classes (e.g., Legendre, Jacobi,
etc.) in the future, such that all these classes will have a common API.
http://docs.python.org/library/string.html#template-strings) used to
generate the code for the `polynomial`, `chebyshev`, `hermite`,
`hermite_e`, `laguerre`, and `lengendre` modules, to implement their
respective `Polynomial`, `Chebyshev`, `Hermite`, `HermiteE`,
`Laguerre`, and `Lengendre` classes.

"""
import string
import sys

if sys.version_info[0] >= 3:
rel_import = "from . import"
else:
rel_import = "import"

polytemplate = string.Template('''
from __future__ import division
import numpy as np
import warnings
REL_IMPORT polyutils as pu

class $name(pu.PolyBase) :
"""A $name series class.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I bet it's the lack of this code that's causing the import problem on Py3.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That was also my first thought, but I don't understand why and won't be
able to get back to it until the middle of next week.
On Jul 13, 2012 6:09 PM, "njsmith" <
reply@reply.github.com>
wrote:

"""
import string

-import sys

-if sys.version_info[0] >= 3:

  • rel_import = "from . import"
    -else:
  • rel_import = "import"

I bet it's the lack of this code that's causing the import problem on Py3.

Reply to this email directly or view it on GitHub:
https://github.com/numpy/numpy/pull/334/files#r1159923

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

tl;dr We're confusing 2to3. Py3 builds of this branch against Git result in failure. Py3 builds against a source dist work fine. Regenerate numpy/polynomial/__init__.py at build time for Py3, or use explicit imports everywhere.

I've just had another look at this. This code removal is unrelated to the Py3 errors. The problem is related to the 2to3 tool.

When a Py3 build is run directly from the repository, the 2to3 tool doesn't refactor the relative imports (e.g. from polynomial import Polynomial) in numpy/polynomial/__init__.py (presumably because the files don't exist at the time 2to3 is run and the tool must therefore assume they are meant to be external imports).

When a Py3 build is run from a source distribution, the numpy/polynomial/polynomial.py etc. files already exist and the relative imports can be correctly generated in numpy/polynomial/__init__.py. This explains why testing with tox works on my machine - the build and test cycle for each Py version is based on a single sdist created by the system python and cached at ~/.tox/distshare/

Travis CI farms out the build tasks and runs the build and test for each Py version against a fresh Git checkout - hence the reported Py3 failures.

To fix, numpy/polynomial/__init__.py also needs to be regenerated at build time for Py3, or use explicit imports from numpy.polynomial.polynomial import ... etc. everywhere.


Expand Down Expand Up @@ -916,4 +907,4 @@ def cast(series, domain=$domain, window=$domain):
"""
return series.convert(domain, $name, window)

'''.replace('REL_IMPORT', rel_import))
''')
45 changes: 45 additions & 0 deletions numpy/polynomial/setup.py
View file
Open in desktop
@@ -1,6 +1,51 @@
import os
import sys
from distutils import util

from polytemplate import polytemplate

def get_build_dir():
"""Determine build directory."""
plat = util.get_platform()
py_ver = '%s.%s' % (sys.version_info[0], sys.version_info[1])

return os.path.join('build',
'lib.%s-%s' % (plat, py_ver), 'numpy', 'polynomial')

def generate_from_template(base_file, name, nick, domain):
"""Generate polynomial class from the template."""
base_path = os.path.join('numpy', 'polynomial')

fp = open(os.path.join(base_path, base_file))
lines = fp.read()
fp.close()

build_path = get_build_dir()
if not os.path.isdir(build_path):
# Distutils doesn't create the build directory until after
# this code has been executed.
os.makedirs(build_path)

fp = open(os.path.join(build_path, base_file[1:]), 'w')
fp.write(lines)
fp.write(polytemplate.substitute(name=name,
nick=nick, domain=domain))
fp.close()

def configuration(parent_package='',top_path=None):
generate_from_template('_chebyshev.py',
name='Chebyshev', nick='cheb', domain='[-1,1]')
generate_from_template('_hermite.py',
name='Hermite', nick='herm', domain='[-1,1]')
generate_from_template('_hermite_e.py',
name='HermiteE', nick='herme', domain='[-1,1]')
generate_from_template('_laguerre.py',
name='Laguerre', nick='lag', domain='[-1,1]')
generate_from_template('_legendre.py',
name='Legendre', nick='leg', domain='[-1,1]')
generate_from_template('_polynomial.py',
name='Polynomial', nick='poly', domain='[-1,1]')

from numpy.distutils.misc_util import Configuration
config = Configuration('polynomial',parent_package,top_path)
config.add_data_dir('tests')
Expand Down