Permalink
Switch branches/tags
webstorm/191.361 webstorm/191.354 webstorm/191.345 webstorm/191.332 webstorm/191.296 webstorm/191.269 webstorm/191.234 webstorm/191.207 webstorm/191.198 webstorm/191.187 webstorm/191.179 webstorm/191.165 webstorm/191.150 webstorm/191.140 webstorm/191.129 webstorm/191.124 webstorm/191.71 webstorm/191.21 webstorm/191.4 webstorm/183.3807 webstorm/183.3797 webstorm/183.3795.1 webstorm/183.3775 webstorm/183.3768 webstorm/183.3755 webstorm/183.3708 webstorm/183.3698 webstorm/183.3693 webstorm/183.3668 webstorm/183.3647.15 webstorm/183.3647.6 webstorm/183.3647.1 webstorm/183.3646 webstorm/183.3584 webstorm/183.3543 webstorm/183.3515 webstorm/183.3463 webstorm/183.3445 webstorm/183.3414 webstorm/183.3384 webstorm/183.3360 webstorm/183.3348 webstorm/183.3340 webstorm/183.3338 webstorm/183.3303 webstorm/183.3285 webstorm/183.3283.12 webstorm/183.3283.3 webstorm/183.3250 webstorm/183.3232 webstorm/183.3205 webstorm/183.3180 webstorm/183.3151 webstorm/183.3136 webstorm/183.3131 webstorm/183.3128 webstorm/183.3125 webstorm/183.3120 webstorm/183.3115 webstorm/183.3103 webstorm/183.3078 webstorm/183.3074 webstorm/183.3056 webstorm/183.3053 webstorm/183.3030 webstorm/183.3020 webstorm/183.2990 webstorm/183.2950 webstorm/183.2943 webstorm/183.2940.11 webstorm/183.2940.4 webstorm/183.2940.1 webstorm/183.2932 webstorm/183.2925 webstorm/183.2918 webstorm/183.2914 webstorm/183.2899 webstorm/183.2894 webstorm/183.2872 webstorm/183.2852 webstorm/183.2825 webstorm/183.2823 webstorm/183.2820 webstorm/183.2815 webstorm/183.2808 webstorm/183.2802 webstorm/183.2793 webstorm/183.2787 webstorm/183.2778 webstorm/183.2772 webstorm/183.2771 webstorm/183.2769 webstorm/183.2759 webstorm/183.2750 webstorm/183.2736 webstorm/183.2723 webstorm/183.2721 webstorm/183.2705 webstorm/183.2703 webstorm/183.2695
Nothing to show
Find file Copy path
2d347f2 Oct 10, 2017
5 contributors

Users who have contributed to this file

@yole @vlasovskikh @traff @ktisha @IlyaKazakevich
519 lines (434 sloc) 18.3 KB
# encoding: utf-8
import atexit
import zipfile
from pycharm_generator_utils.clr_tools import *
from pycharm_generator_utils.module_redeclarator import *
from pycharm_generator_utils.util_methods import *
# TODO: Move all CLR-specific functions to clr_tools
debug_mode = False
def redo_module(module_name, outfile, module_file_name, doing_builtins):
# gobject does 'del _gobject' in its __init__.py, so the chained attribute lookup code
# fails to find 'gobject._gobject'. thus we need to pull the module directly out of
# sys.modules
mod = sys.modules.get(module_name)
mod_path = module_name.split('.')
if not mod and sys.platform == 'cli':
# "import System.Collections" in IronPython 2.7 doesn't actually put System.Collections in sys.modules
# instead, sys.modules['System'] get set to a Microsoft.Scripting.Actions.NamespaceTracker and Collections can be
# accessed as its attribute
mod = sys.modules[mod_path[0]]
for component in mod_path[1:]:
try:
mod = getattr(mod, component)
except AttributeError:
mod = None
report("Failed to find CLR module " + module_name)
break
if mod:
action("restoring")
r = ModuleRedeclarator(mod, outfile, module_file_name, doing_builtins=doing_builtins)
r.redo(module_name, ".".join(mod_path[:-1]) in MODULES_INSPECT_DIR)
action("flushing")
r.flush()
else:
report("Failed to find imported module in sys.modules " + module_name)
# find_binaries functionality
def cut_binary_lib_suffix(path, f):
"""
@param path where f lives
@param f file name of a possible binary lib file (no path)
@return f without a binary suffix (that is, an importable name) if path+f is indeed a binary lib, or None.
Note: if for .pyc or .pyo file a .py is found, None is returned.
"""
if not f.endswith(".pyc") and not f.endswith(".typelib") and not f.endswith(".pyo") and not f.endswith(".so") and not f.endswith(
".pyd"):
return None
ret = None
match = BIN_MODULE_FNAME_PAT.match(f)
if match:
ret = match.group(1)
modlen = len('module')
retlen = len(ret)
if ret.endswith('module') and retlen > modlen and f.endswith('.so'): # what for?
ret = ret[:(retlen - modlen)]
if f.endswith('.pyc') or f.endswith('.pyo'):
fullname = os.path.join(path, f[:-1]) # check for __pycache__ is made outside
if os.path.exists(fullname):
ret = None
pat_match = TYPELIB_MODULE_FNAME_PAT.match(f)
if pat_match:
ret = "gi.repository." + pat_match.group(1)
return ret
def is_posix_skipped_module(path, f):
if os.name == 'posix':
name = os.path.join(path, f)
for mod in POSIX_SKIP_MODULES:
if name.endswith(mod):
return True
return False
def is_mac_skipped_module(path, f):
fullname = os.path.join(path, f)
m = MAC_STDLIB_PATTERN.match(fullname)
if not m: return 0
relpath = m.group(2)
for module in MAC_SKIP_MODULES:
if relpath.startswith(module): return 1
return 0
def is_skipped_module(path, f):
return is_mac_skipped_module(path, f) or is_posix_skipped_module(path, f[:f.rindex('.')]) or 'pynestkernel' in f
def is_module(d, root):
return (os.path.exists(os.path.join(root, d, "__init__.py")) or
os.path.exists(os.path.join(root, d, "__init__.pyc")) or
os.path.exists(os.path.join(root, d, "__init__.pyo")) or
is_valid_implicit_namespace_package_name(d))
def walk_python_path(path):
for root, dirs, files in os.walk(path):
if root.endswith('__pycache__'):
continue
dirs_copy = list(dirs)
for d in dirs_copy:
if d.endswith('__pycache__') or not is_module(d, root):
dirs.remove(d)
# some files show up but are actually non-existent symlinks
yield root, [f for f in files if os.path.exists(os.path.join(root, f))]
def list_binaries(paths):
"""
Finds binaries in the given list of paths.
Understands nested paths, as sys.paths have it (both "a/b" and "a/b/c").
Tries to be case-insensitive, but case-preserving.
@param paths: list of paths.
@return: dict[module_name, full_path]
"""
SEP = os.path.sep
res = {} # {name.upper(): (name, full_path)} # b/c windows is case-oblivious
if not paths:
return {}
if IS_JAVA: # jython can't have binary modules
return {}
paths = sorted_no_case(paths)
for path in paths:
if path == os.path.dirname(sys.argv[0]): continue
for root, files in walk_python_path(path):
cutpoint = path.rfind(SEP)
if cutpoint > 0:
preprefix = path[(cutpoint + len(SEP)):] + '.'
else:
preprefix = ''
prefix = root[(len(path) + len(SEP)):].replace(SEP, '.')
if prefix:
prefix += '.'
note("root: %s path: %s prefix: %s preprefix: %s", root, path, prefix, preprefix)
for f in files:
name = cut_binary_lib_suffix(root, f)
if name and not is_skipped_module(root, f):
note("cutout: %s", name)
if preprefix:
note("prefixes: %s %s", prefix, preprefix)
pre_name = (preprefix + prefix + name).upper()
if pre_name in res:
res.pop(pre_name) # there might be a dupe, if paths got both a/b and a/b/c
note("done with %s", name)
the_name = prefix + name
file_path = os.path.join(root, f)
res[the_name.upper()] = (the_name, file_path, os.path.getsize(file_path), int(os.stat(file_path).st_mtime))
return list(res.values())
def list_sources(paths):
# noinspection PyBroadException
try:
for path in paths:
if path == os.path.dirname(sys.argv[0]): continue
path = os.path.normpath(path)
if path.endswith('.egg') and os.path.isfile(path):
say("%s\t%s\t%d", path, path, os.path.getsize(path))
for root, files in walk_python_path(path):
for name in files:
if name.endswith('.py') or name.endswith('-nspkg.pth'):
file_path = os.path.join(root, name)
say("%s\t%s\t%d", os.path.normpath(file_path), path, os.path.getsize(file_path))
say('END')
sys.stdout.flush()
except:
import traceback
traceback.print_exc()
sys.exit(1)
# noinspection PyBroadException
def zip_sources(zip_path):
if not os.path.exists(zip_path):
os.makedirs(zip_path)
zip_filename = os.path.normpath(os.path.sep.join([zip_path, "skeletons.zip"]))
try:
zip = zipfile.ZipFile(zip_filename, 'w', zipfile.ZIP_DEFLATED)
except:
zip = zipfile.ZipFile(zip_filename, 'w')
try:
try:
while True:
line = sys.stdin.readline()
line = line.strip()
if line == '-':
break
if line:
# This line will break the split:
# /.../dist-packages/setuptools/script template (dev).py setuptools/script template (dev).py
split_items = line.split()
if len(split_items) > 2:
match_two_files = re.match(r'^(.+\.py)\s+(.+\.py)$', line)
if not match_two_files:
report("Error(zip_sources): invalid line '%s'" % line)
continue
split_items = match_two_files.group(1, 2)
(path, arcpath) = split_items
zip.write(path, arcpath)
else:
# busy waiting for input from PyCharm...
time.sleep(0.10)
say('OK: ' + zip_filename)
sys.stdout.flush()
except:
import traceback
traceback.print_exc()
say('Error creating archive.')
sys.exit(1)
finally:
zip.close()
def add_to_zip(zip, paths):
# noinspection PyBroadException
try:
for path in paths:
print("Walking root %s" % path)
if path == os.path.dirname(sys.argv[0]): continue
path = os.path.normpath(path)
if path.endswith('.egg') and os.path.isfile(path):
pass # TODO: handle eggs
for root, files in walk_python_path(path):
for name in files:
if name.endswith('.py') or name.endswith('-nspkg.pth'):
file_path = os.path.join(root, name)
arcpath = os.path.relpath(file_path, path)
zip.write(file_path, os.path.join(str(hash(path)), arcpath))
except:
import traceback
traceback.print_exc()
sys.exit(1)
def zip_stdlib(zip_path):
if not os.path.exists(zip_path):
os.makedirs(zip_path)
import platform
zip_filename = os.path.normpath(os.path.sep.join([zip_path, "%s-%s-stdlib-%s.zip" % (
'Anaconda' if sys.version.find('Anaconda') != -1 else 'Python',
'.'.join(map(str, sys.version_info)),
platform.platform())]))
print("Adding file to %s" % zip_filename)
try:
zip = zipfile.ZipFile(zip_filename, 'w', zipfile.ZIP_DEFLATED)
except:
zip = zipfile.ZipFile(zip_filename, 'w')
try:
add_to_zip(zip, sys.path)
finally:
zip.close()
# command-line interface
# noinspection PyBroadException
def process_one(name, mod_file_name, doing_builtins, subdir):
"""
Processes a single module named name defined in file_name (autodetect if not given).
Returns True on success.
"""
if has_regular_python_ext(name):
report("Ignored a regular Python file %r", name)
return True
if not quiet:
say(name)
sys.stdout.flush()
action("doing nothing")
try:
fname = build_output_name(subdir, name)
action("opening %r", fname)
old_modules = list(sys.modules.keys())
imported_module_names = []
class MyFinder:
# noinspection PyMethodMayBeStatic
def find_module(self, fullname, path=None):
if fullname != name:
imported_module_names.append(fullname)
return None
my_finder = None
if hasattr(sys, 'meta_path'):
my_finder = MyFinder()
sys.meta_path.append(my_finder)
else:
imported_module_names = None
action("importing")
__import__(name) # sys.modules will fill up with what we want
if my_finder:
sys.meta_path.remove(my_finder)
if imported_module_names is None:
imported_module_names = [m for m in sys.modules.keys() if m not in old_modules]
redo_module(name, fname, mod_file_name, doing_builtins)
# The C library may have called Py_InitModule() multiple times to define several modules (gtk._gtk and gtk.gdk);
# restore all of them
path = name.split(".")
redo_imports = not ".".join(path[:-1]) in MODULES_INSPECT_DIR
if imported_module_names and redo_imports:
for m in sys.modules.keys():
if m.startswith("pycharm_generator_utils"): continue
action("looking at possible submodule %r", m)
# if module has __file__ defined, it has Python source code and doesn't need a skeleton
if m not in old_modules and m not in imported_module_names and m != name and not hasattr(
sys.modules[m], '__file__'):
if not quiet:
say(m)
sys.stdout.flush()
fname = build_output_name(subdir, m)
action("opening %r", fname)
try:
redo_module(m, fname, mod_file_name, doing_builtins)
finally:
action("closing %r", fname)
except:
exctype, value = sys.exc_info()[:2]
msg = "Failed to process %r while %s: %s"
args = name, CURRENT_ACTION, str(value)
report(msg, *args)
if debug_mode:
if sys.platform == 'cli':
import traceback
traceback.print_exc(file=sys.stderr)
raise
return False
return True
def get_help_text():
return (
# 01234567890123456789012345678901234567890123456789012345678901234567890123456789
'Generates interface skeletons for python modules.' '\n'
'Usage: ' '\n'
' generator [options] [module_name [file_name]]' '\n'
' generator [options] -L ' '\n'
'module_name is fully qualified, and file_name is where the module is defined.' '\n'
'E.g. foo.bar /usr/lib/python/foo_bar.so' '\n'
'For built-in modules file_name is not provided.' '\n'
'Output files will be named as modules plus ".py" suffix.' '\n'
'Normally every name processed will be printed and stdout flushed.' '\n'
'directory_list is one string separated by OS-specific path separtors.' '\n'
'\n'
'Options are:' '\n'
' -h -- prints this help message.' '\n'
' -d dir -- output dir, must be writable. If not given, current dir is used.' '\n'
' -b -- use names from sys.builtin_module_names' '\n'
' -q -- quiet, do not print anything on stdout. Errors still go to stderr.' '\n'
' -x -- die on exceptions with a stacktrace; only for debugging.' '\n'
' -v -- be verbose, print lots of debug output to stderr' '\n'
' -c modules -- import CLR assemblies with specified names' '\n'
' -p -- run CLR profiler ' '\n'
' -s path_list -- add paths to sys.path before run; path_list lists directories' '\n'
' separated by path separator char, e.g. "c:\\foo;d:\\bar;c:\\with space"' '\n'
' -L -- print version and then a list of binary module files found ' '\n'
' on sys.path and in directories in directory_list;' '\n'
' lines are "qualified.module.name /full/path/to/module_file.{pyd,dll,so}"' '\n'
' -i -- read module_name, file_name and list of imported CLR assemblies from stdin line-by-line' '\n'
' -S -- lists all python sources found in sys.path and in directories in directory_list\n'
' -z archive_name -- zip files to archive_name. Accepts files to be archived from stdin in format <filepath> <name in archive>'
)
if __name__ == "__main__":
from getopt import getopt
helptext = get_help_text()
opts, args = getopt(sys.argv[1:], "d:hbqxvc:ps:LiSzu")
opts = dict(opts)
quiet = '-q' in opts
_is_verbose = '-v' in opts
subdir = opts.get('-d', '')
if not opts or '-h' in opts:
say(helptext)
sys.exit(0)
if '-L' not in opts and '-b' not in opts and '-S' not in opts and '-i' not in opts and '-u' not in opts and not args:
report("Neither -L nor -b nor -S nor any module name given")
sys.exit(1)
if "-x" in opts:
debug_mode = True
# patch sys.path?
extra_path = opts.get('-s', None)
if extra_path:
source_dirs = extra_path.split(os.path.pathsep)
for p in source_dirs:
if p and p not in sys.path:
sys.path.append(p) # we need this to make things in additional dirs importable
note("Altered sys.path: %r", sys.path)
# find binaries?
if "-L" in opts:
if len(args) > 0:
report("Expected no args with -L, got %d args", len(args))
sys.exit(1)
say(VERSION)
results = list(list_binaries(sys.path))
results.sort()
for name, path, size, last_modified in results:
say("%s\t%s\t%d\t%d", name, path, size, last_modified)
sys.exit(0)
if "-S" in opts:
if len(args) > 0:
report("Expected no args with -S, got %d args", len(args))
sys.exit(1)
say(VERSION)
list_sources(sys.path)
sys.exit(0)
if "-z" in opts:
if len(args) != 1:
report("Expected 1 arg with -z, got %d args", len(args))
sys.exit(1)
zip_sources(args[0])
sys.exit(0)
if "-u" in opts:
if len(args) != 1:
report("Expected 1 arg with -u, got %d args", len(args))
sys.exit(1)
zip_stdlib(args[0])
sys.exit(0)
# build skeleton(s)
timer = Timer()
# determine names
if '-b' in opts:
if args:
report("No names should be specified with -b")
sys.exit(1)
names = list(sys.builtin_module_names)
if not BUILTIN_MOD_NAME in names:
names.append(BUILTIN_MOD_NAME)
if '__main__' in names:
names.remove('__main__') # we don't want ourselves processed
ok = True
for name in names:
ok = process_one(name, None, True, subdir) and ok
if not ok:
sys.exit(1)
else:
if '-i' in opts:
if args:
report("No names should be specified with -i")
sys.exit(1)
name = sys.stdin.readline().strip()
mod_file_name = sys.stdin.readline().strip()
if not mod_file_name:
mod_file_name = None
refs = sys.stdin.readline().strip()
else:
if len(args) > 2:
report("Only module_name or module_name and file_name should be specified; got %d args", len(args))
sys.exit(1)
name = args[0]
if len(args) == 2:
mod_file_name = args[1]
else:
mod_file_name = None
refs = opts.get('-c', '')
if sys.platform == 'cli':
# noinspection PyUnresolvedReferences
import clr
if refs:
for ref in refs.split(';'): clr.AddReferenceByPartialName(ref)
if '-p' in opts:
atexit.register(print_profile)
# We take module name from import statement
name = get_namespace_by_name(name)
if not process_one(name, mod_file_name, False, subdir):
sys.exit(1)
say("Generation completed in %d ms", timer.elapsed())